#!/bin/bash
ncpus="32"    #hardcoded for conrad
mpiprocs="32" #set from 1-32 for conrad

whichrun=2    #1=15km, 2=02km
sim="02km.mnt2.dst0.sst1"

if [ $whichrun -eq 1 ]; then #For Initial/Hist Run, DX=15km, 432x432x50 Grid points
 RAMSIN="RAMSIN.${sim}.init"
 walltimerams="005:00:00"
 walltimerpck="001:45:00"
 walltimexfer="002:00:00"
 nodes="7"
elif [ $whichrun -eq 2 ]; then #For Initial/Hist Run, DX=2km, 950x950x50 Grid points
 RAMSIN="RAMSIN.${sim}.histinitre"
 walltimerams="023:00:00"  #24 hours for full run
 walltimerpck="011:00:00"
 walltimexfer="010:00:00"
 nodes="14"
fi

runrams="x.runrams."${sim}
runrepk="x.runrepk."${sim}
runxfer="x.runxfer."${sim}
base_d="/p/home/smsaleeb/rams_20170906_6.2.06_ONR_MURI"
current_d="${base_d}/run.conrad"
exe_d="${base_d}/bin.rams"
ramsexe="rams-6.2.06.parallel"
ramsin_d="A.RAMSINs.dust.6.2.06"
outfile="stdout.rams.${sim}.txt"
outrpck="stdout.repack.${sim}.txt"
outdir="a.${sim}"

###################################################################################
#Run Simulation
###################################################################################
cores=$(bc -q <<< "${nodes} * ${mpiprocs}")
if [ -f ${runrams} ];then rm -f ${runrams}; fi
cat > ${runrams} << END
#!/bin/bash
#PBS -A ONRDC39385461
#PBS -l select=${nodes}:ncpus=${ncpus}:mpiprocs=${mpiprocs}
#PBS -l walltime=${walltimerams}
#PBS -q standard
#PBS -N y.runrams.${sim}
#PBS -j oe
#PBS -m be
#PBS -M smsaleeb@gmail.com
cp -r ${exe_d}/${ramsexe} ${current_d}/${ramsin_d}/${RAMSIN} ${base_d}/etc ${WORKDIR}
cd ${WORKDIR}
aprun -n ${cores} ./${ramsexe} -f ./${RAMSIN} > ${current_d}/${outfile}
END
qsub ${runrams}
runramsid=`qselect -N y.runrams.${sim} | cut -d '.' -f 1`

####################################################################################
#Repack HDF5 output files
###################################################################################
if [ -f ${runrepk} ];then rm -f ${runrepk}; fi
cat > ${runrepk} << END
#!/bin/bash
#PBS -A ONRDC39385461
#PBS -l select=1:ncpus=${ncpus}:mpiprocs=1
#PBS -l walltime=${walltimerpck}
#PBS -q standard
#PBS -N y.repack.${sim}
#PBS -j oe
#PBS -m be
#PBS -M smsaleeb@gmail.com
#PBS -W depend=afterok:$runramsid
#module load hdf5
cp ${current_d}/h*repack* ${WORKDIR}/${outdir}
cd ${WORKDIR}/${outdir}
./hdf5_repack.sh > ${current_d}/stdout/${outrpck}
END
qsub ${runrepk}
repackid=`qselect -N y.repack.${sim} | cut -d '.' -f 1`

####################################################################################
#Transfer HDF5 output files
###################################################################################
if [ -f ${runxfer} ]; then rm -f ${runxfer}; fi
cat > ${runxfer} << END
#!/bin/bash
#PBS -A ONRDC39385461
#PBS -l select=1:ncpus=1
#PBS -l walltime=${walltimexfer}
#PBS -q transfer
#PBS -N y.transfer.${sim}
#PBS -j oe
#PBS -m be
#PBS -M smsaleeb@gmail.com
#PBS -W depend=afterok:$repackid
cd ${WORKDIR}
rsync -avz ${outdir} smsaleeb@floyd.atmos.colostate.edu:/blizzard/smsaleeb/run.dust
END
qsub ${runxfer}
