#!/bin/bash -x
#help#
#*********************************************************************************************************#
#                                                                                                         #
# script to run CPTEC Global Model on PC Clusters under MPI Scali                                         #
# and Sun Grid Engine without OpenMP                                                                      #
#                                                                                                         #
# assumptions: assume present at the same directory:                                                      #
#              ParModel_MPI (Global Model Executable file)                                                #
#              MODELIN (Global Model input Namelist file)                                                 #
#                                                                                                         #
# usage: runModel_clima cpu_mpi cpu_node  task_omp name TRC LV LABELI LABELW LABELF NMC NMSST initlz hold #
# where:                                                                                                  #
# cpu_mpi: integer, the desired number of mpi processes                                                   #
# cpu_node: integer, the desired number of mpi processes per shared memory node                           #
# name: character, the job name (for SGE)                                                                 #
# initlz  =2,     ! initlz =2 diabatic initialization and normal mode initialization                      #
#                 !        =1 diabatic initialization and without normal mode initialization              #
#                 !        =0 without diabatic initialization and without normal mode initialization      #
#                 !           [TOTAL RESTART ](adiabatic with no normal mode initialization)              #
#                 !        <0 same as >0 with sib variables read in instead of  initialized               #
#                 !        =-1 diabatic initialization and without normal mode initialization             #
#                 !            with sib variables read in instead of  initialized                         #
#                 !        =-2 diabatic initialization and normal mode initialization                     #
#                 !            with sib variables read in instead of  initialized                         #
# hold: any, present or not;                                                                              #
#            if absent, script finishes after queueing job;                                               #
#            if present, script holds till job completion                                                 #
#*********************************************************************************************************#
#help#
#
#       Help:
#
if [ "${1}" = "help" -o -z "${1}" ]
then
  cat < ${0} | sed -n '/^#help#/,/^#help#/p'
  exit 1
else
  TRC=`echo ${5} | awk '{print $1/1}'`   
fi
if [ -z "${6}" ]
then
  echo "LV is not set" 
  exit 2
else
  LV=`echo ${6} | awk '{print $1/1}'`    
fi

if [ -z "${7}" ]
then
  echo "LABELI is not set" 
  exit 3
else
  export LABELI=${7}  
fi

if [ -z "${8}" ]
then
  echo "LABELW is not set" 
  exit 3
else
  export LABELW=${8}  
fi

if [ -z "${9}" ]
then
  echo "LABELF is not set" 
  exit 3
else
  export LABELF=${9}  
fi
if [ -z "${10}" ]
then
  echo "PREFIC is not set" 
  exit 3
else
  export PREFIC=${10}  
fi

if [ -z "${11}" ]
then 
  echo "NMSST is not set" 
  exit 3
else 
  export NMSST=${11}  
fi
if [ -z "${12}" ]
then 
  echo "initlz is not set" 
  exit 3
else 
  export initlz=${12}  
fi
if [ "$#" == 13 ]
then 
  export hold=${13}  
else 
  export hold=""
fi
echo $hold
#

if [ ${TRC} = 21 ]; then
 export timestep=3600
fi 
if [ ${TRC} = 31 ]; then
 export timestep=1800
fi 
if [ ${TRC} = 42 ]; then
 export timestep=1800
fi 
if [ ${TRC} = 62 ]; then
 export timestep=900
fi
if [ ${TRC} = 106 ]; then
 export timestep=900
fi
if [ ${TRC} = 126 ]; then
 export timestep=600
fi
if [ ${TRC} = 133 ]; then
 export timestep=600
fi
if [ ${TRC} = 159 ]; then
 export timestep=600
fi
if [ ${TRC} = 170 ]; then
 export timestep=450
fi
if [ ${TRC} = 213 ]; then
 export timestep=450
fi
if [ ${TRC} = 213 ]; then
 export timestep=300
fi
if [ ${TRC} = 254 ]; then
 export timestep=225
fi
if [ ${TRC} = 299 ]; then
 export timestep=225
fi
if [ ${TRC} = 319 ]; then
 export timestep=225
fi
if [ ${TRC} = 341 ]; then
 export timestep=200
fi
if [ ${TRC} = 382 ]; then
 export timestep=180
fi
if [ ${TRC} = 511 ]; then
 export timestep=150
fi
if [ ${TRC} = 533 ]; then
 export timestep=150
fi
if [ ${TRC} = 666 ]; then
 export timestep=150
fi
if [ ${TRC} = 863 ]; then
 export timestep=150
fi
if [ ${TRC} = 1279 ]; then
 export timestep=20
fi

#
# SETTING THE APPROPRIATED ENVIRONMENT
#
CASE=`echo ${TRC} ${LV} |awk '{ printf("TQ%4.4dL%3.3d\n",$1,$2)  }' `
PATHA=`pwd`
export FILEENV=`find ${PATHA} -name EnvironmentalVariablesMCGA -print`
export PATHENV=`dirname ${FILEENV}`
export PATHBASE=`cd ${PATHENV};cd ../;pwd`
. ${FILEENV} ${CASE} ${PREFIC}
cd ${HOME_suite}/run
#
#   Set nproc, resol, host, machine, NQS Queue and Run time
#
DIRRESOL=`echo ${TRC} ${LV} |awk '{ printf("TQ%4.4dL%3.3d\n",$1,$2)  }' `
HSTMAQ=`hostname`
MAQUI=`hostname -s`
QUEUE=${QUEUE}
RUNTM=`date +'%Y%m%d%T'`
yi=`awk 'BEGIN {print substr("'${LABELI}'",1,4)}'` ; export yi
mi=`awk 'BEGIN {print substr("'${LABELI}'",5,2)}'` ; export mi
di=`awk 'BEGIN {print substr("'${LABELI}'",7,2)}'` ; export di
hi=`awk 'BEGIN {print substr("'${LABELI}'",9,2)}'` ; export hi

yw=`awk 'BEGIN {print substr("'${LABELW}'",1,4)}'` ; export yw
mw=`awk 'BEGIN {print substr("'${LABELW}'",5,2)}'` ; export mw
dw=`awk 'BEGIN {print substr("'${LABELW}'",7,2)}'` ; export dw
hw=`awk 'BEGIN {print substr("'${LABELW}'",9,2)}'` ; export hw

yf=`awk 'BEGIN {print substr("'${LABELF}'",1,4)}'` ; export yf
mf=`awk 'BEGIN {print substr("'${LABELF}'",5,2)}'` ; export mf
df=`awk 'BEGIN {print substr("'${LABELF}'",7,2)}'` ; export df
hf=`awk 'BEGIN {print substr("'${LABELF}'",9,2)}'` ; export hf
NEXTDATE=`${caldate} ${LABELI} + 6h yyyymmddhh`
echo $yi $mi $di $hi  $yf $mf $df $hf
mkdir -p ${DK_HSM}/GFCT/${yi}${mi}${di}${hi} 
mkdir -p ${DK_HSM}/GFGH/${yi}${mi}${di}${hi} 
mkdir -p ${HOME_suite}/run/setout
#
#########################################################
#
#      SCRIPT FOR GLOBAL MODEL PRODUCTION RUNS 
#
#########################################################

#
# Step 1: Set Directories and files:
#
#   DIRBASE is the root directory path; 
#           all files belong to subdirectories of root;
#   EXECFILEPATH is the executable filename (with path)
#   SCRIPTFILEPATH is the script file that submits executable (with path)
#   NAMELISTFILEPATH contains the namelist file read by the executable (with path)
#   OUTPUTFILEPATH is the executable output file (with path)
#
EXECFILEPATH=${DK_suite}/model/exec
EXECFILEPATH2=${DK_suite}/model/exec_${PREFIC}.${LABELI}${LABELF};mkdir -p ${EXECFILEPATH2}/setout
SCRIPTFILEPATH=${EXECFILEPATH2}/modg.${PREFIC}.${DIRRESOL}.${MAQUI}
NAMELISTFILEPATH=${HOME_suite}/run
OUTPUTFILEPATH=${HOME_suite}/run/setout/modg.${PREFIC}.${LABELI}${LABELF}.${DIRRESOL}.${MAQUI}.${RUNTM}.out
FSCR=${HOME_suite}/run
mkdir -p ${EXECFILEPATH2}
export cpu_mpi=$1
export cpu_node=$2
export ThreadsperMPITASK=$3
export RES=$4
num=$(($cpu_mpi+$cpu_node-1))
fra=$(($num/$cpu_node))
cpu_tot=$(($fra*$cpu_node))
echo fila=mpi-npn${cpu_node} total cpus=${cpu_tot}
#
# build Nanmelist to model MCGA
#
if [ -z "${PREFXO}" ] ;then
export PREFX="NMC" 
else
export PREFX="${PREFXO}" 
fi 
if [ -z "${PREFXI}" ] ;then
export PREFY="NMC"
else
export PREFY="${PREFXI}"
fi 
export eigeninit=".FALSE."    #eigeninit=".TRUE."
export mgiven=".FALSE."      #,   ! mgiven       --> .FALSE.
export gaussgiven=".FALSE."      #,   ! gaussgiven       --> .FALSE.

#export eigeninit=".TRUE."    #eigeninit=".TRUE."
#export mgiven=".FALSE."      #,   ! mgiven       --> .FALSE.
#export gaussgiven=".FALSE."      #,   ! gaussgiven       --> .FALSE.

export aspa="'"
export path_in=${DK_suite}/model/datain; mkdir -p ${path_in}

#export path_in=/scratch1/oper/tempo/MCGA/model/datain

export dirfNameOutput=${DK_suite2}/model/dataout/${DIRRESOL}/${yi}${mi}${di}${hi}_P${cpu_mpi}; mkdir -p ${dirfNameOutput}
export dirfNameOutputNEXT=${DK_suite2}/model/dataout/${DIRRESOL}/${NEXTDATE}; mkdir -p ${dirfNameOutputNEXT}

if [ `echo $PREFIC | grep "R" | wc -l` -ge 1 ]; then
export DHFCT=3
else
export DHFCT=24
fi


cat ${NAMELISTFILEPATH}/MODELIN | awk '{  
 if (substr($1,1,5) == "trunc")
  {
   "echo ${TRC}" | getline TRC	   
    printf(" trunc    =%4.4d,		      !TRC   : three-digit triangular truncation\n",TRC)
  }
 else if (substr($1,1,4) == "vert")
  {
   "echo ${LV}" | getline LV	 
    printf(" vert     =%3.3d,		      !LV    : two-digit number of vertical sigma-layers\n",LV)
  }
 else if (substr($1,1,2) == "dt")
  {
   "echo $timestep" | getline timestep       
    printf(" dt       =%.1f,	     !      : delta t\n",timestep)
  }
 else if (substr($1,1,6) == "IDATEI")
  {
   "echo $yi" | getline yi	 
   "echo $mi" | getline mi	 
   "echo $di" | getline di	 
   "echo $hi" | getline hi	 
    printf(" IDATEI   = %2.2d,%2.2d,%2.2d,%4.4d, !LABELI: initial forecasting label\n",hi,di,mi,yi)
  }
 else if (substr($1,1,6) == "IDATEW")
  {
   "echo $yw" | getline yw	 
   "echo $mw" | getline mw	 
   "echo $dw" | getline dw	 
   "echo $hw" | getline hw	 
    printf(" IDATEW   = %2.2d,%2.2d,%2.2d,%4.4d, !LABELC: final forecasting label for cold\n",hw,dw,mw,yw)
  }
 else if (substr($1,1,6) == "IDATEF")
  {
   "echo $yf" | getline yf	 
   "echo $mf" | getline mf	 
   "echo $df" | getline df	 
   "echo $hf" | getline hf	 
    printf(" IDATEF   = %2.2d,%2.2d,%2.2d,%4.4d, !LABELF: final forecasting label for warm\n",hf,df,mf,yf)
  }
 else if (substr($1,1,9) == "eigeninit")
  { 
   "echo $eigeninit" | getline eigeninit
    printf(" eigeninit     =%s,  ! eigenInit  --> .FALSE.\n",eigeninit)
  }
 else if (substr($1,1,6) == "mgiven")
  { 
   "echo $mgiven" | getline mgiven
    printf(" mgiven	   =%s,  ! mgiven  --> .FALSE.\n",mgiven)
  }
 else if (substr($1,1,10) == "gaussgiven")
  { 
   "echo $gaussgiven" | getline gaussgiven
    printf(" gaussgiven    =%s,  ! gaussgiven  --> .FALSE.\n",gaussgiven)
  }
 else if (substr($1,1,5) == "DHFCT")
  { 
   "echo $DHFCT" | getline DHFCT
    printf(" DHFCT    =%2.2d,             !DHFCT : > 0 interval in hours to output diagnostics\n",DHFCT)
  }
 else if (substr($1,1,5) == "PREFX")
  { 
   "echo $aspa" | getline aspa
   "echo $PREFX" | getline PREFX
    printf(" PREFX    =%s%s%s , 	 !PREFX : preffix for name of output files\n",aspa,PREFX,aspa)
  }
 else if (substr($1,1,5) == "NMSST")
  { 
   "echo $aspa" | getline aspa
   "echo $NMSST" | getline NMSST
    printf(" NMSST     =%s%s%s,       !NMSST : sst file name\n",aspa,NMSST,aspa)
  }
 else if (substr($1,1,5) == "PREFY")
  { 
   "echo $aspa" | getline aspa
   "echo $PREFY" | getline PREFY
    printf(" PREFY    =%s%s%s , 	 !PREFY : preffix for name of input files\n",aspa,PREFY,aspa)
  }
 else if (substr($1,1,7) == "path_in")
  { 
   "echo $aspa" | getline aspa
   "echo $path_in" | getline path_in
    printf(" path_in=%s%s%s , \n",aspa,path_in,aspa)
  }
 else if (substr($1,1,14) == "dirfNameOutput")
  { 
   "echo $aspa" | getline aspa
   "echo $dirfNameOutput" | getline dirfNameOutput
    printf(" dirfNameOutput=%s%s%s , \n",aspa,dirfNameOutput,aspa)
  }
 else if (substr($1,1,6) == "initlz")
  { 
   "echo $initlz" | getline initlz
    printf(" initlz = %d , ! initlz =2 diabatic initialization and normal mode initialization\n",initlz)
  }
  else
  {
    print $0
  }
 }'    > ${EXECFILEPATH2}/MODELIN
export PBS_SERVER=${pbs_server1}
optserver=`printf "$PBS_SERVER \n" | cut -c1-3`
if [[ (${optserver} = "aux") ]]; then
export MPPBS="#"
else
export MPPBS="#PBS -l mppwidth=${cpu_mpi}"
fi

#
# Step 2: Build script that runs the AGCM executable getting information
#         from namelist 
#
cat <<EOF1>${EXECFILEPATH2}/mpisep.bash
#!/bin/bash
export F_UFMTENDIAN=18,19,20,22,23,24,25,26,27,31,32,33,36,37,38,39,42,43,44,45,49,50,51,52,53,55,61,66,71,77,80,81,82,83,88,91,92,93,99
export GFORTRAN_CONVERT_UNIT=big_endian:18,19,20,22,23,24,25,26,27,31,32,33,36,37,38,39,42,43,44,45,49,50,51,52,53,55,61,66,71,77,80,81,82,83,88,91,92,93,99
export KMP_STACKSIZE=128m
ulimit -s unlimited
export MPID_RSH
cd ${EXECFILEPATH2}
${EXECFILEPATH2}/ParModel_MPI < ${EXECFILEPATH2}/MODELIN>> ${EXECFILEPATH2}/setout/Print.model.${LABELI}.${tmstp}.%s.MPI${cpu_mpi}.out
EOF1
chmod +x ${EXECFILEPATH2}/mpisep.bash
#
# Step 3: Build script to submit the script above in the UNA
#
cat <<EOF0>${SCRIPTFILEPATH}
#!/bin/bash
#PBS -o ${HSTMAQ}:${EXECFILEPATH2}/setout/Out.model.${PREFIC}.${LABELI}.${tmstp}.%s.MPI${cpu_mpi}.out
#PBS -j oe
#PBS -l walltime=${WALLTIME}
${MPPBS}
#PBS -l mppnppn=${cpu_node}
#PBS -l mppdepth=${ThreadsperMPITASK}
#PBS -V
#PBS -S /bin/bash
#PBS -N $RES
#PBS -q ${QUEUE}
#PBS -A ${QUOTA}

export PBS_SERVER=${pbs_server1}
export HUGETLB_MORECORE=yes
export HUGETLB_ELFMAP=W
export HUGETLB_FORCE_ELFMAP=yes+
#
cp -f ${EXECFILEPATH}/ParModel_MPI ${EXECFILEPATH2}/
cd ${EXECFILEPATH2}
mkdir -p setout
date
export OMP_NUM_THREADS=${ThreadsperMPITASK}
ulimit -s unlimited
optserver=`printf "$PBS_SERVER \n" | cut -c1-3`
if [[ (\${optserver} = "aux") ]]; then
${EXECFILEPATH2}/mpisep.bash
else
aprun -m500h   -n  ${cpu_mpi} -N  ${cpu_node} -d ${ThreadsperMPITASK}  ${EXECFILEPATH2}/ParModel_MPI < ${EXECFILEPATH2}/MODELIN>> ${EXECFILEPATH2}/setout/Print.model.${LABELI}.${tmstp}.%s.MPI${cpu_mpi}.out

fi
date
sleep 60
if [[ ${initlz} -eq -2 ]];then
cp -f ${dirfNameOutput}/GFCTNMC${NEXTDATE}${NEXTDATE}F.unf.${DIRRESOL}.sibprgP*   ${dirfNameOutputNEXT}/
fi
mkdir -p ${dirfNameOutput}/RESTAT_${LABELF}
cp ${dirfNameOutput}/*unf*  ${dirfNameOutput}/RESTAT_${LABELF}/
EOF0
chmod +x ${SCRIPTFILEPATH}
cd ${EXECFILEPATH2}
if [[ ${it} -eq 1 ]];then
FIRST=`qsub ${SCRIPTFILEPATH}`
export FIRST
echo $FIRST
else
SECOND=`qsub -W depend=afterok:$FIRST ${SCRIPTFILEPATH}`
echo $SECOND
fi
#qsub ${hold} ${SCRIPTFILEPATH}
echo  ${hold} ${SCRIPTFILEPATH}

if [ "$hold" == "" ]
then
echo "$hold = NO"
else 
echo "$hold = YES"
itt=2
while [ ${itt} -gt 0 ];do
itt=`qstat @aux20 |grep $RES| wc -l`
itt2=`qstat @eslogin13 |grep $RES| wc -l`
let itt=${itt}+${itt2}
sleep 60
done
fi
