#!/bin/tcsh
#
# DART software - Copyright UCAR. This open source software is provided
# by UCAR, "as is", without charge, subject to all terms of use at
# http://www.image.ucar.edu/DAReS/DART/DART_download
#
# DART $Id$

#=============================================================================
# This block of directives constitutes the preamble for the LSF queuing system
#
# the normal way to submit to the queue is:    bsub < runmodel_1x
#
# an explanation of the most common directives follows:
# -J Job name
# -o STDOUT filename
# -e STDERR filename
# -P account
# -q queue    cheapest == [standby, economy, (regular,debug), premium] == $$$$
# -n number of processors  (really)
##=============================================================================
#BSUB -J fortnight
#BSUB -o fortnight.%J.log
#BSUB -q regular
#BSUB -N -u ${USER}@ucar.edu
#BSUB -n 20
#BSUB -W 12:00
#BXXX -P nnnnnnnn
#
##=============================================================================
## This block of directives constitutes the preamble for the PBS queuing system
## PBS is used on the CGD   Linux cluster 'bangkok'
## PBS is used on the CGD   Linux cluster 'calgary'
## 
## the normal way to submit to the queue is:    qsub runmodel_1x
## 
## an explanation of the most common directives follows:
## -N     Job name
## -r n   Declare job non-rerunable
## -e <arg>  filename for standard error
## -o <arg>  filename for standard out 
## -q <arg>   Queue name (small, medium, long, verylong)
## -l nodes=xx:ppn=2   requests BOTH processors on the node. On both bangkok 
##                     and calgary, there is no way to 'share' the processors
##                     on the node with another job, so you might as well use
##                     them both.  (ppn == Processors Per Node)
##=============================================================================
#PBS -N testrun
#PBS -r n
#PBS -e testrun.err
#PBS -o testrun.log
#PBS -q medium
#PBS -l nodes=10:ppn=2

# A common strategy for the beginning is to check for the existence of
# some variables that get set by the different queuing mechanisms.
# This way, we know which queuing mechanism we are working with,
# and can set 'queue-independent' variables for use for the remainder
# of the script.

if ($?LS_SUBCWD) then

   # LSF has a list of processors already in a variable (LSB_HOSTS)

   setenv JOBNAME     $LSB_OUTPUTFILE:ar
   setenv CENTRALDIR  $LS_SUBCWD
   setenv TMPDIR      /ptmp/${user}
   setenv SRCDIR      /fs/image/home/${user}/SVN/DART/models/MITgcm_ocean

   set advance_command = 'mpirun.lsf ./mitgcmuv_20p'
       
else if ($?PBS_O_WORKDIR) then

   setenv JOBNAME     $PBS_JOBNAME
   setenv CENTRALDIR  $PBS_O_WORKDIR
   setenv TMPDIR      /ptmp/${user}
   setenv SRCDIR      /fs/image/home/${user}/SVN/DART/models/MITgcm_ocean

   # PBS has a list of processors in a file whose name is (PBS_NODEFILE)

   set advance_command = 'mpirun ./mitgcmuv_20p'
       
else if ($?MYNODEFILE) then

   # If you have a linux cluster with no queuing software, use this
   # section.  The list of computational nodes is given to the mpirun
   # command and it assigns them as they appear in the file.  In some
   # cases it seems to be necessary to wrap the command in a small
   # script that changes to the current directory before running.

   echo "running with no queueing system"

   # before running this script, do this once.  the syntax is
   # node name : how many tasks you can run on it
   #setenv MYNODEFILE  ~/nodelist
   #echo "node7:2" >! $MYNODEFILE
   #echo "node5:2" >> $MYNODEFILE
   #echo "node3:2" >> $MYNODEFILE
   #echo "node1:2" >> $MYNODEFILE

   setenv NUM_PROCS 8
   echo "running with $NUM_PROCS nodes specified from $MYNODEFILE"

   setenv JOBNAME     Oceans
   setenv CENTRALDIR  `pwd`
   setenv TMPDIR      /ptmp/${user}
   setenv SRCDIR      /fs/image/home/${user}/SVN/DART/models/MITgcm_ocean

   set advance_command = "mpirun -np $NUM_PROCS -nolocal -machinefile $MYNODEFILE ./mitgcmuv_20p"
       
else

   # interactive - assume you are using 'lam-mpi' and that you have
   # already run 'lamboot' once to start the lam server, or that you
   # are running with a machine that has mpich installed.

   echo "running interactively ... NOT"
   # mpirun -np 2 ./mitgcmuv_20p

endif

mkdir -p ${TMPDIR}/${JOBNAME}
cd       ${TMPDIR}/${JOBNAME}

# Copy the namelist files - these are small, so we really copy them
foreach FILE ( data data.cal data.exf data.kpp \
               data.obcs data.pkg eedata )
   cp -p ${SRCDIR}/inputs/$FILE .
end

# link the files used by data&PARM05 - input datasets
foreach FILE ( bathymetry.bin gom_H_199601.bin gom_S_199601.bin \
             gom_T_199601.bin gom_U_199601.bin gom_V_199601.bin )
   ln -sf ${SRCDIR}/inputs/$FILE .
end

# link the files used by data.exf&EXF_NML_02 - external forcings
foreach FILE ( lev05_monthly_sss_relax.bin \
               lev05_monthly_sst_relax.bin \
               run-off.bin_1x1             \
               ncep_air_19960101.bin       \
               ncep_dlwrf_19960101.bin     \
               ncep_dswrf_19960101.bin     \
               ncep_nswrs_19960101.bin     \
               ncep_prate_19960101.bin     \
               ncep_shum_19960101.bin      \
               ncep_uwnd_19960101.bin      \
               ncep_vwnd_19960101.bin      )
   ln -sf ${SRCDIR}/inputs/$FILE .
end

# link the files used by data.obcs&OBCS_PARM01 - open boundaries
foreach FILE ( Rs_SobcsE_52_01_nPx1.bin    Rs_SobcsN_52_01_nPy1.bin \
               Rs_TobcsE_52_01_nPx1.bin    Rs_TobcsN_52_01_nPy1.bin \
               Rs_UobcsE_52_01_nPx1_c1.bin Rs_UobcsN_52_01_nPy1.bin \
               Rs_VobcsE_52_01_nPx1.bin    Rs_VobcsN_52_01_nPy1_c1.bin)
   ln -sf ${SRCDIR}/inputs/$FILE .
end

cp -p ${SRCDIR}/work/mitgcmuv_20p .

eval ${advance_command} 

set OUTPUTDIR = ${CENTRALDIR}/work/${JOBNAME}
mkdir -p                 ${OUTPUTDIR}
mv -f *.data *.meta STD* ${OUTPUTDIR}

ls -al

exit 0

# <next few lines under version control, do not edit>
# $URL$
# $Revision$
# $Date$

