#!/bin/tcsh
#
# DART software - Copyright UCAR. This open source software is provided
# by UCAR, "as is", without charge, subject to all terms of use at
# http://www.image.ucar.edu/DAReS/DART/DART_download
#
# DART $Id$
#
# This script submits the jobs to advance the ROMS ensemble members using
# a JOB ARRAY. EACH ENSEMBLE MEMBER GETS EXECUTED ON ITS OWN NODE(s).
# The run-time control file for ROMS gets updated with the next filename
# and DSTART appropriate for the next forecast cycle.
#
# The prerequisite for this script is that advance_ensemble.csh has been run.
# Things to note: many strings are intended to be replaced when this
# template gets copied and ultimately submitted. Anything that starts
# with 'My' is a string that gets replaced in the "normal" ROMs fashion.
# The next few are not standard.
#
# Myens_size             gets replaced in the 'stage_experiment.csh' script
# SXXXXX                 gets replaced if this is a dependent job (with SBATCH)
# FILTERJOBID            gets replaced by the job ID that must finish first
# EXPERIMENT_DIRECTORY   gets replaced in the 'stage_experiment.csh' script
#
#==========================================================================
# SLURM directives 			sbatch advance_ensemble.csh
# 					squeue -u $USER
# 					scancel <jobnumber>
#SBATCH --ignore-pbs
#SBATCH --array=1-Myens_size
#SBATCH --ntasks=16
#SBATCH --time=10:00
#SBATCH --exclude=node04
#SBATCH --job-name=roms_advance
#SBATCH --output=roms_advance-%A-%a.out
#SBATCH --error=roms_advance-%A-%a.err
#
#==========================================================================
# PBS directives                	qsub test_batch.csh
# 					qstat -u $USER
# 					qdel <jobnumber>
#PBS -J 1-Myens_size
#PBS -l walltime=00:10:00
#PBS -q economy
#PBS -l select=1:ncpus=16:mpiprocs=16
#PBS -A P8685nnnn
#PBS -N roms_advance
#
#==========================================================================
# LSF directives 			bsub < advance_ensemble.csh
# 					bjobs
# 					bkill <jobnumber>
#BSUB -J roms_advance[1-Myens_size]
#BSUB -o roms_advance.%J.%I.log
#BSUB -N -u ${USER}@ucar.edu
#BSUB -q small
#BSUB -n 16
#BSUB -W 0:10
#BSUB -P P8685nnnn

#==========================================================================
# Figure out the platform-specific nuances.
#==========================================================================
if ($?SLURM_JOB_ID) then

   set     JOBNAME = $SLURM_JOB_NAME
   set       JOBID = $SLURM_JOBID
   set ARRAY_INDEX = $SLURM_ARRAY_TASK_ID
   set    NODELIST = $SLURM_NODELIST
   set   LAUNCHCMD = "mpirun -np $SLURM_NTASKS -bind-to core"

else if ($?PBS_O_WORKDIR) then

   env | grep PBS | sort

   set     JOBNAME = $PBS_JOBNAME
   set       JOBID = "$PBS_JOBID"
   set ARRAY_INDEX = $PBS_ARRAY_INDEX
   set    NODELIST = `cat "${PBS_NODEFILE}"`
   set   LAUNCHCMD = "mpiexec_mpt"

   # CISL suggests users set TMPDIR when running batch jobs on Cheyenne
   mkdir -p /glade/scratch/$USER/temp
   setenv TMPDIR /glade/scratch/$USER/temp

else if ($?LS_SUBCWD) then

   set     JOBNAME = $LSB_JOBNAME
   set       JOBID = $LSB_JOBID
   set ARRAY_INDEX = $LSB_JOBINDEX
   set    NODELIST = $LSB_SUB_HOST
   set   LAUNCHCMD = "mpirun.lsf"

else
   echo "This script must be run with a queueing system that supports job arrays."
   exit 1
endif

#--------------------------------------------------------------------------
# Just an echo of job attributes
#--------------------------------------------------------------------------

set instance = $ARRAY_INDEX
set INSTANCE_DIRECTORY = `printf "instance_%04d" $instance`
cd EXPERIMENT_DIRECTORY/$INSTANCE_DIRECTORY

echo
echo "#---------------------------------------------------------------------"
echo "# ROMS $instance started at "`date`
echo "#---------------------------------------------------------------------"
echo
echo "${JOBNAME} ($JOBID) working directory "`pwd`
echo "${JOBNAME} ($JOBID) running       on ${NODELIST}"
echo "${JOBNAME} ($JOBID) started at "`date`
echo

rm -f log_$instance.txt

#--------------------------------------------------------------------------
echo "advancing instance $instance at ..."`date`
#--------------------------------------------------------------------------

# sometimes we need the full name, sometimes we need it without the extension
set RST_FILE = MyRSTNAME
set DAI_FILE = MyDAINAME
set OBS_FILE = MyMODname
set RST_ROOT = $RST_FILE:r
set DAI_ROOT = $DAI_FILE:r
set OBS_ROOT = $OBS_FILE:r

# get the DSTART time so we can get the right observation file
set INPUT_FILE = `sed -n -e '/ININAME/s/.*== *//p' MyROMS_STDIN` || exit 1
set     DSTART = `../get_ocean_time.csh ${INPUT_FILE}`       || exit 1

# Set ROMS standard input parameters needed in template scripts.

\cp ../s4dvar.in.template MyAPARNAM
set OBS_PREF = ../Obs/obs
set  NEW_OBS = `printf %s_%d.nc ${OBS_PREF} $DSTART`
MySUBSTITUTE MyAPARNAM     MyOBSname   $NEW_OBS

# Advance the model
${LAUNCHCMD} ../MyROMS_EXE MyROMS_STDIN >& log_$instance.txt || exit 2

# Check for successful completion - log file should NOT have something like:
# Blowing-up: Saving latest model state into  RESTART file
grep -i blow log_$instance.txt > /dev/null
if ($status == 0) then
   echo "ROMS instance $instance FAILED."
   echo "ROMS instance $instance FAILED."
   echo "ROMS instance $instance FAILED."
   exit 1
endif

# The ROMS restart file will be treated as the DART prior.
# DART will create an updated ROMS posterior.
# Tag the output with the current ocean time.

set DSTART = `../get_ocean_time.csh ${DAI_FILE}` || exit 3

set ROMS_PRIOR     = `printf %s_%04d_%d.nc ${RST_ROOT} $instance $DSTART`
set ROMS_POSTERIOR = `printf roms_posterior_%04d_%d.nc $instance $DSTART`
set ROMS_OBSFILE   = `printf %s_%04d_%d.nc ${OBS_ROOT} $instance $DSTART`
set SAFETY         = `printf roms_dai_original_%04d_%d.nc $instance $DSTART`

\cp -v ${DAI_FILE} ${SAFETY}          || exit 1
\mv -v ${DAI_FILE} ${ROMS_POSTERIOR}  || exit 1
\mv -v ${RST_FILE} ${ROMS_PRIOR}      || exit 1
\mv -v ${OBS_FILE} ${ROMS_OBSFILE}    || exit 1

# Update the ocean.in file with the new DSTART value

\cp ../ocean.in.template  MyROMS_STDIN

MySUBSTITUTE MyROMS_STDIN  MyDSTART   ${DSTART}
MySUBSTITUTE MyROMS_STDIN  MyININAME  ${ROMS_POSTERIOR}

echo
echo "#---------------------------------------------------------------------"
echo "# ROMS instance $instance completed at "`date`
echo "#---------------------------------------------------------------------"
echo

exit 0

# <next few lines under version control, do not edit>
# $URL$
# $Revision$
# $Date$

