#!/bin/bash
# 
# script to run CPTEC Global Model on PC Clusters under MPI Scali
# and Sun Grid Engine without OpenMP
#
# assumptions: assume present at the same directory:
#              ParModel_MPI (Global Model Executable file)
#              MODELIN (Global Model input Namelist file)
#
# usage: run_multi_UNA cpu_mpi cpu_node name hold
# where:
# cpu_mpi: integer, the desired number of mpi processes
# cpu_node: integer, the desired number of mpi processes per shared memory node
# name: character, the job name (for SGE)
# hold: any, present or not;
#            if absent, script finishes after queueing job;
#            if present, script holds till job completion
if [ "$#" == 4 ]
then hold="-sync y"
else hold=
fi
export FEXE=`pwd`
export cpu_mpi=$1
export cpu_node=$2
export RES=$3
num=$(($cpu_mpi+$cpu_node-1))
fra=$(($num/$cpu_node))
cpu_tot=$(($fra*$cpu_node))
echo fila=mpi-npn${cpu_node} total cpus=${cpu_tot}
cat <<EOF0>${FEXE}/qsub.sh
#!/bin/bash
#$ -pe mpi-npn${cpu_node} ${cpu_tot}
#$ -o una1:${FEXE}/Out.MPI${cpu_mpi}
#$ -j y
#$ -V
#$ -S /bin/bash
#$ -N $RES
#
cd ${FEXE}
mpirun -np ${cpu_mpi} -machinefile \${TMPDIR}/machines mpisep.bash
EOF0
chmod +x qsub.sh
cat <<EOF1>${FEXE}/mpisep.bash
#!/bin/bash
#export F_UFMTENDIAN=18,10,11,22,23,24,49,83,82,81,27,25,26,33,99,61,45,42,52,50,80,51,53,92,93,43,77,91,66,20,74,88,89,36,37,38,39
export F_UFMTENDIAN=2,7,9,10,11,14,15,18,19,20,22,23,24,25,26,27,29,31,32,33,36,37,38,39,41,42,43,45,47,49,50,51,52,53,55,61,66,67,71,77,80,81,82,83,88,91,92,93,99
export KMP_STACKSIZE=128m
ulimit -s unlimited
export MPID_RSH
cd ${FEXE}
ParModel_MPI < MODELIN
EOF1
#
chmod +x mpisep.bash
qsub $hold qsub.sh
