Added workflow scripts for Titan and Eos to example/Workflow

example/Workflow/Eos/eos-Decomp.pbs

@@ -0,0 +1,54 @@
+#!/bin/bash
+#PBS -A GEO106
+#PBS -N Decomp
+#PBS -j oe
+#PBS -l walltime=02:30:00,nodes=9
+##PBS -l walltime=01:00:00,nodes=18
+##PBS -l gres=widow2%widow3
+##PBS -q killable
+##PBS -q debug
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+#LBPM_WIA_INSTALL_DIR=/lustre/atlas/proj-shared/geo106/build-eos-LBPM-WIA
+
+LBPM_WIA_INSTALL_DIR=$PROJWORK/geo106/Eos-LBPM-WIA
+
+#echo "PBS_O_WORKDIR: `echo $PBS_O_WORKDIR`"
+source $MODULESHOME/init/bash
+module swap PrgEnv-intel PrgEnv-gnu
+
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+  # directory structure should be set up 
+  #   directories must include PATTERN within the name (change as needed)
+  #   each directory must contain
+  #      - Domain.in -- file that specifies parallel domain
+  #      - Segmented.in -- file that specifies input file
+  #      - 8-bit binary file with the digital geometry
+  LIST=$(ls | grep PATTERN)
+
+  # integer labels for the solid phase and the non-wetting phase
+  SOLID=0
+  NWP=1
+
+  # loop over directories and generate input files for parallel simulation
+  # performs domain decomposition based on Domain.in
+  # number of processors must match simulation
+  NUMPROCS=144
+  for i in $LIST; do
+     echo $i
+     cd $i
+     aprun -n $NUMPROCS $LBPM_WIA_INSTALL_DIR/bin/lbpm_segmented_decomp $SOLID $NWP
+
+     mkdir -p MEDIA
+     cp ID* MEDIA
+     cd ..
+  done
+
+
+exit;

example/Workflow/Eos/eos-MorphDrain.pbs

@@ -0,0 +1,55 @@
+#!/bin/bash
+#PBS -A GEO106
+#PBS -N MorphDrain
+#PBS -j oe
+##PBS -l walltime=02:00:00,nodes=216
+#PBS -l walltime=05:00:00,nodes=18
+##PBS -l gres=widow2%widow3
+##PBS -q killable
+##PBS -q debug
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+#LBPM_WIA_INSTALL_DIR=/lustre/atlas/proj-shared/geo106/build-eos-LBPM-WIA
+
+LBPM_WIA_INSTALL_DIR=/ccs/proj/geo106/eos/LBPM-WIA
+
+#echo "PBS_O_WORKDIR: `echo $PBS_O_WORKDIR`"
+source $MODULESHOME/init/bash
+module swap PrgEnv-intel PrgEnv-gnu
+module load python_anaconda
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+cp ../MEDIA/ID* ./
+
+LABEL=$(basename $PWD)
+
+# List of saturation values to generate
+SW="0.90 0.85 0.80 0.75 0.70 0.65 0.60 0.55 0.5 0.45 0.4 0.35 0.3 0.25 0.2"
+echo "r sw" > morphdrain.csv
+
+# number of processors to use
+NUMPROCS=288
+
+#for r in `cat radius.csv`; do 
+for sw in $SW; do 
+      echo $sw >> sw.log
+      aprun -n $NUMPROCS $LBPM_WIA_INSTALL_DIR/bin/lbpm_morphdrain_pp $sw > morphdrain.log
+      radius=$(grep "Final critical radius"  morphdrain.log | sed 's/Final critical radius=//g') 
+      saturation=$(grep "Final saturation"  morphdrain.log | sed 's/Final saturation=//g')
+      echo "$radius $saturation" >> morphdrain.csv
+      tag="sw_$sw"
+      DIR=$LABEL"_drain_"$tag
+      mkdir -p $DIR
+      cp ID.* $DIR
+      cp SignDist.* $DIR
+      cp Domain.in $DIR
+      cp Color.in $DIR
+done
+
+exit;
+

example/Workflow/Eos/eos-MorphOpen.pbs

@@ -0,0 +1,59 @@
+#!/bin/bash
+#PBS -A GEO106
+#PBS -N MorphOpen
+#PBS -j oe
+##PBS -l walltime=02:00:00,nodes=216
+#PBS -l walltime=08:00:00,nodes=18
+##PBS -l gres=widow2%widow3
+##PBS -q killable
+##PBS -q debug
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+#LBPM_WIA_INSTALL_DIR=/lustre/atlas/proj-shared/geo106/build-eos-LBPM-WIA
+
+LBPM_WIA_INSTALL_DIR=/ccs/proj/geo106/eos/LBPM-WIA
+
+#echo "PBS_O_WORKDIR: `echo $PBS_O_WORKDIR`"
+source $MODULESHOME/init/bash
+module swap PrgEnv-intel PrgEnv-gnu
+module load python_anaconda
+
+#module swap cray-mpich2 cray-mpich2/5.6.3
+#module load cudatoolkit
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+cp ../MEDIA/ID* ./
+
+LABEL=$(basename $PWD)
+
+rm sw.log
+
+# List of saturations to generate using Morphological opening
+SW="0.85 0.80 0.75 0.70 0.65 0.60 0.55 0.5 0.45 0.4 0.35 0.3 0.25 0.2 0.15"
+
+echo "r sw" > morphopen.csv
+
+NUMPROCS=288
+
+for sw in $SW; do 
+      echo $sw >> sw.log
+      aprun -n $NUMPROCS $LBPM_WIA_INSTALL_DIR/bin/lbpm_morphopen_pp $sw > morphopen.log
+      radius=$(grep "Final critical radius"  morphopen.log | sed 's/Final critical radius=//g') 
+      saturation=$(grep "Final saturation"  morphopen.log | sed 's/Final saturation=//g')
+      echo "$radius $saturation" >> morphopen.csv
+      tag="sw_$sw"
+      DIR=$LABEL"_open_"$tag
+      mkdir -p $DIR
+      cp ID.* $DIR
+      cp SignDist.* $DIR
+      cp Domain.in $DIR
+      cp Color.in $DIR
+done
+
+exit;
+

example/Workflow/Eos/eos-Preprocess.pbs

@@ -0,0 +1,33 @@
+#!/bin/bash
+#PBS -A GEO106
+#PBS -N Dist
+#PBS -j oe
+#PBS -l walltime=02:30:00,nodes=18
+##PBS -l walltime=01:00:00,nodes=18
+##PBS -l gres=widow2%widow3
+##PBS -q killable
+##PBS -q debug
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+#LBPM_WIA_INSTALL_DIR=/lustre/atlas/proj-shared/geo106/build-eos-LBPM-WIA
+
+LBPM_WIA_INSTALL_DIR=/ccs/proj/geo106/eos/LBPM-WIA
+
+#echo "PBS_O_WORKDIR: `echo $PBS_O_WORKDIR`"
+source $MODULESHOME/init/bash
+module swap PrgEnv-intel PrgEnv-gnu
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+    # generate distance map from the binary image
+    # input files are ID.xxxxx
+    # output files are SignDist.xxxxx
+     NUMPROCS=288
+     aprun -n $NUMPROCS $LBPM_WIA_INSTALL_DIR/bin/lbpm_segmented_pp
+
+
+exit;

example/Workflow/Eos/eos-Random.pbs

@@ -0,0 +1,49 @@
+#!/bin/bash
+#PBS -A GEO106
+#PBS -N Random
+#PBS -j oe
+##PBS -l walltime=02:00:00,nodes=216
+#PBS -l walltime=12:00:00,nodes=18
+##PBS -l gres=widow2%widow3
+##PBS -q killable
+##PBS -q debug
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+#LBPM_WIA_INSTALL_DIR=/lustre/atlas/proj-shared/geo106/build-eos-LBPM-WIA
+
+LBPM_WIA_INSTALL_DIR=/ccs/proj/geo106/eos/LBPM-WIA
+
+#echo "PBS_O_WORKDIR: `echo $PBS_O_WORKDIR`"
+source $MODULESHOME/init/bash
+module swap PrgEnv-intel PrgEnv-gnu
+module load python_anaconda
+
+#module swap cray-mpich2 cray-mpich2/5.6.3
+#module load cudatoolkit
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+
+LABEL=$(basename $PWD)
+
+NUMPROCS=288
+# Generate a bunch of random states
+for sat in `seq -w 8 4 92`; do 
+      cp ../MEDIA/ID* ./
+      sw="0.$sat"
+      aprun -n $NUMPROCS $LBPM_WIA_INSTALL_DIR/bin/lbpm_random_pp $sw 1
+  
+      tag="sw_$sw"
+      DIR=$LABEL"_random_"$tag
+      mkdir -p $DIR
+      cp ID.* $DIR
+      cp SignDist.* $DIR
+      cp Domain.in $DIR
+      cp Color.in $DIR
+done
+
+exit;
+

example/Workflow/Titan/TitanRelPerm.pbs

@@ -0,0 +1,53 @@
+#!/bin/bash
+#PBS -A GEO106
+#PBS -N Ryan-Sandstone-RelPerm
+#PBS -j oe
+#PBS -l walltime=6:00:00,nodes=10660
+
+# Number of nodes should be [# directories] x [GPU per directory]
+# 
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+LBPM_WIA_INSTALL_DIR=/ccs/proj/geo106/titan/LBPM-WIA
+
+source $MODULESHOME/init/bash
+module unload PrgEnv-gnu PrgEnv-pgi PrgEnv-cray PrgEnv-intel
+module load PrgEnv-gnu
+module load cudatoolkit
+module load dynamic-link
+module load python wraprun
+
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+export MPICH_RDMA_ENABLED_CUDA=1
+export MPICH_MAX_THREAD_SAFETY=multiple
+
+
+LIST=$(ls | grep "s5_")
+
+NUMSIM=$(ls | grep "s5_" | wc -l)
+
+# Requested number of nodes must be 54 x $NUMSIM !!
+echo "Number of simulations to run is $NUMSIM (288 GPU per simulation)" 
+
+NUMPROCS=288
+
+# BUIlD THE LAUNCH COMMAND 
+for dir in $LIST; do
+    ARGS=$ARGS" : -n $NUMPROCS -N 1 -d 16 --w-cd $dir $LBPM_WIA_INSTALL_DIR/bin/lbpm_color_simulator"
+done
+
+echo $ARGS | sed 's/./wraprun/' > launch.log
+
+LAUNCH=$(cat launch.log)
+
+echo "running job "
+$LAUNCH
+
+exit;
+
+
+#    aprun -n 512 -N 1 -d 16 $LBPM_WIA_INSTALL_DIR/bin/lbpm_color_simulator >> drainage.log