#!/bin/tcsh
#
# DART software - Copyright UCAR. This open source software is provided
# by UCAR, "as is", without charge, subject to all terms of use at
# http://www.image.ucar.edu/DAReS/DART/DART_download
#
# DART $Id$
#
# This script converts all the roms verification observation files into
# a single DART observation sequence file and then performs an assimilation.
#
# The prerequisite for this script is that advance_ensemble.csh has been run.
# Things to note: many strings are intended to be replaced when this
# template gets copied and ultimately submitted. Anything that starts
# with 'My' is a string that gets replaced in the "normal" ROMs fashion.
#
#==========================================================================
# SLURM directives                      sbatch advance_ensemble.csh
#                                       squeue -u $USER
#                                       scancel <jobnumber>
#SBATCH --ignore-pbs
#SBATCH --job-name=filter
#SBATCH --output=filter-%A.log
#SBATCH --error=filter-%A.err
#SBATCH --ntasks=48
#SBATCH --ntasks-per-node=16
#SBATCH --time=00:30:00
#SBATCH --error=filter-%A.err
#SBATCH --output=filter-%A.log
#
#==========================================================================
# PBS directives                        qsub test_batch.csh
#                                       qstat -u $USER
#                                       qdel <jobnumber>
#PBS -N filter
#PBS -e filter.err
#PBS -o filter.log
#PBS -l select=3:ncpus=16:mpiprocs=16
#PBS -l walltime=00:10:00
#PBS -A P8685nnnn
#PBS -q economy
#PBS -r n
#
#==========================================================================
# LSF directives                        bsub < advance_ensemble.csh
#                                       bjobs
#                                       bkill <jobnumber>
#BSUB -J filter
#BSUB -o filter.%J.log
#BSUB -P P8685nnnn
#BSUB -q regular
#BSUB -n 48
#BSUB -R "span[ptile=16]"
#BSUB -W 1:00
#BSUB -N -u ${USER}@ucar.edu
#
#==========================================================================

if ($?SLURM_JOB_ID) then

   set ORIGINALDIR = $SLURM_SUBMIT_DIR
   set     JOBNAME = $SLURM_JOB_NAME
   set       JOBID = $SLURM_JOBID
   set     MYQUEUE = $SLURM_JOB_PARTITION
   set      MYHOST = $SLURM_SUBMIT_HOST
   set   LAUNCHCMD = "mpirun -np $SLURM_NTASKS -bind-to core"

else if ($?PBS_O_WORKDIR) then

   set ORIGINALDIR = $PBS_O_WORKDIR
   set     JOBNAME = $PBS_JOBNAME
   set       JOBID = $PBS_JOBID
   set      MYHOST = $PBS_O_HOST
   set     MYQUEUE = $PBS_QUEUE
   set   LAUNCHCMD = "mpiexec_mpt"

else if ($?LS_SUBCWD) then

   set ORIGINALDIR = $LS_SUBCWD
   set     JOBNAME = $LSB_JOBNAME
   set       JOBID = $LSB_JOBID
   set     MYQUEUE = $LSB_QUEUE
   set      MYHOST = $LSB_SUB_HOST
   set   LAUNCHCMD = "mpirun.lsf"

else

   set ORIGINALDIR = `pwd`
   set     JOBNAME = roms_filter
   set       JOBID = $$
   set     MYQUEUE = Interactive
   set      MYHOST = `hostname`
   set   LAUNCHCMD = "aprun -n 1"

endif

#--------------------------------------------------------------------------
# Just an echo of job attributes
#--------------------------------------------------------------------------

echo
echo "${JOBNAME} ($JOBID) submit directory ${ORIGINALDIR}"
echo "${JOBNAME} ($JOBID) submit      host ${MYHOST}"
echo "${JOBNAME} ($JOBID) running in queue ${MYQUEUE}"
echo "${JOBNAME} ($JOBID) started at "`date`
echo

cd EXPERIMENT_DIRECTORY

#==========================================================================
# STEP 1: prepare for DART INFLATION
# This stages the files that contain the inflation values.
# The inflation values change through time and should be archived.
#
# This file is only relevant if 'inflation' is turned on -
# i.e. if inf_flavor(:) /= 0 AND inf_initial_from_restart = .TRUE.
#
# filter_nml
# inf_flavor                  = 2,                 0,
# inf_initial_from_restart    = .true.,            .false.,
#
# NOTICE: the archiving scripts require the names of these
# files to be as listed above. When being archived, the filenames get a
# unique extension (describing the assimilation time) appended to them.
#
# NOTICE: inf_initial_from_restart and inf_sd_initial_from_restart are somewhat
# problematic. During the bulk of an experiment, these should be TRUE, since
# we want to read existing inflation files. However, the first assimilation
# might need these to be FALSE and then subsequently be set to TRUE.
# There is now only one way to handle this:
#
# 1) create a cookie file called EXPERIMENT_DIRECTORY/roms_inflation_cookie
#    The existence of this file will cause this script to set the
#    namelist appropriately. This script will 'eat' the cookie file
#    to prevent this from happening for subsequent executions. If the
#    inflation file does not exist for them, and it needs to, this script
#    should die. The stage_experiment.csh script automatically creates a cookie
#    file to support this option.
#
# The strategy is to use the LATEST (last) inflation file.
#==========================================================================

# The input.nml may be modified differently for each assimilation cycle.
# (for example, inflation files for the first cycle)

\cp input.nml.template input.nml

# These are used to determine IF we are doing inflation.

set  MYSTRING = `grep 'inf_flavor' input.nml`
set  MYSTRING = `echo $MYSTRING | sed -e "s#[=,'\.]# #g"`
set  PRIOR_INF = $MYSTRING[2]
set  POSTE_INF = $MYSTRING[3]

# These are used to determine if we HAVE BEEN doing inflation.
# Do we read from a namelist or a file?

set  MYSTRING = `grep 'inf_initial_from_restart' input.nml`
set  MYSTRING = `echo $MYSTRING | sed -e "s#[=,'\.]# #g"`
set  PRIOR_TF = `echo $MYSTRING[2] | tr '[:upper:]' '[:lower:]'`
set  POSTE_TF = `echo $MYSTRING[3] | tr '[:upper:]' '[:lower:]'`

# IFF we want PRIOR inflation:

if ( $PRIOR_INF > 0 ) then

   if ($PRIOR_TF == false) then
      # We are not using an existing inflation file.
      echo "inf_flavor(1) = $PRIOR_INF, using namelist values."

   else if ( -e roms_inflation_cookie ) then
      # We want to use an existing inflation file, but this is
      # the first assimilation so there is no existing inflation
      # file. This is the signal we need to to coerce the namelist
      # to have different values for this execution ONLY.
      # Since the local namelist comes from EXPERIMENT_DIRECTORY each time, we're golden.

      set PRIOR_TF = FALSE

ex input.nml <<ex_end
g;inf_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
g;inf_sd_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
wq
ex_end

   else

      # Checking for a prior inflation mean file from the previous assimilation.

      (ls -rt1 output_priorinf_mean.* | tail -n 1 >! latestfile) > & /dev/null
      set nfiles = `cat latestfile | wc -l`

      if ( $nfiles > 0 ) then
         set latest = `cat latestfile`
         \ln -vs $latest input_priorinf_mean.nc
      else
         echo "ERROR: Requested PRIOR inflation but specified no incoming inflation MEAN file."
         echo "ERROR: expected something like output_priorinf_mean.SSSSS.nc"
         exit 2
      endif

      # Checking for a prior inflation sd file from the previous assimilation.

      (ls -rt1 output_priorinf_sd.* | tail -n 1 >! latestfile) > & /dev/null
      set nfiles = `cat latestfile | wc -l`

      if ( $nfiles > 0 ) then
         set latest = `cat latestfile`
         \ln -vs $latest input_priorinf_sd.nc
      else
         echo "ERROR: Requested PRIOR inflation but specified no incoming inflation SD file."
         echo "ERROR: expected something like output_priorinf_sd.SSSSS.nc"
         exit 2
      endif

   endif
else
   echo "Prior Inflation           not requested for this assimilation."
endif

# POSTERIOR: We look for the 'newest' and use it - IFF we need it.

if ( $POSTE_INF > 0 ) then

   if ($POSTE_TF == false) then
      # We are not using an existing inflation file.
      echo "inf_flavor(2) = $POSTE_INF, using namelist values."

   else if ( -e roms_inflation_cookie ) then
      # We want to use an existing inflation file, but this is
      # the first assimilation so there is no existing inflation
      # file. This is the signal we need to to coerce the namelist
      # to have different values for this execution ONLY.
      # Since the local namelist comes from EXPERIMENT_DIRECTORY each time, we're golden.

      set POSTE_TF = FALSE

ex input.nml <<ex_end
g;inf_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
g;inf_sd_initial_from_restart ;s;= .*;= .${PRIOR_TF}., .${POSTE_TF}.,;
wq
ex_end

   else

      # Checking for a posterior inflation mean file from the previous assimilation.

      (ls -rt1 output_postinf_mean.* | tail -n 1 >! latestfile) > & /dev/null
      set nfiles = `cat latestfile | wc -l`

      if ( $nfiles > 0 ) then
         set latest = `cat latestfile`
         \ln -vs $latest input_postinf_mean.nc
      else
         echo "ERROR: Requested POSTERIOR inflation but specified no incoming inflation MEAN file."
         echo "ERROR: expected something like output_postinf_mean.SSSSS.nc"
         exit 2
      endif

      # Checking for a posterior inflation sd file from the previous assimilation.

      (ls -rt1 output_postinf_sd.* | tail -n 1 >! latestfile) > & /dev/null
      set nfiles = `cat latestfile | wc -l`

      if ( $nfiles > 0 ) then
         set latest = `cat latestfile`
         \ln -vs $latest input_postinf_sd.nc
      else
         echo "ERROR: Requested POSTERIOR inflation but specified no incoming inflation SD file."
         echo "ERROR: expected something like output_postinf_sd.SSSSS.nc"
         exit 2
      endif

   endif
else
   echo "Posterior Inflation       not requested for this assimilation."
endif

# Eat the cookie whether it exists or not
\rm -f roms_inflation_cookie

#==========================================================================
# STEP 2:  Consolidate all of the ROMS precomputed forward observations into
#          a DART observation sequence file.
#==========================================================================

# Remove the last set of DART run-time logs - if they exist.
\rm -f dart_log.out dart_log.nml

# grab the DSTART from someplace (i.e. ocean.in)
set DSTARTSTRING = `grep 'DSTART =' instance_0001/MyROMS_STDIN | grep -oE '[[:digit:]]+'`
set DSTART = $DSTARTSTRING[1]

# grab the base MODname from someplace. The advance_ensemble.csh script tags
# each of these with the instance number to keep them unique.
set  MODname = MyMODname
set OBS_ROOT = $MODname:r

# Because convert_roms_obs and filter need bits from the ROMS model_mod,
# a (single) ROMS input file is required to satisfy 'static_init_model()'.
# Any one will do.

ln -sf instance_0001/roms_posterior_????_${DSTART}.nc MyDAINAME

ls -1 instance_*/${OBS_ROOT}*_${DSTART}.nc  >! precomputed_files.txt

./convert_roms_obs  || exit 2

#==========================================================================
# STEP 3: Run DART on the ensemble of new states.
#         Collect all the ROMS_RESTARTs into a list of input files. 
#         The io module will error out if restart_files.txt is too short.
#         (make sure all instances of ROMS advanced successfully)
#         DART (filter) will modify these files in-place.
#==========================================================================

ls -1 instance_*/roms_posterior_????_${DSTART}.nc  >! restart_files.txt

${LAUNCHCMD} ./filter || exit 3

# Tag the output with the valid time of the model state.

foreach FILE ( input_*mean.nc     input_*sd.nc \
            preassim_*mean.nc  preassim_*sd.nc \
           postassim_*mean.nc postassim_*sd.nc \
            analysis_*mean.nc  analysis_*sd.nc \
              output_*mean.nc    output_*sd.nc \
           preassim_member_????.nc \
           postassim_member_????.nc )

   if (  -e $FILE ) then
      set FEXT  = $FILE:e
      set FBASE = $FILE:r
      \mv -v $FILE ${FBASE}.${DSTART}.${FEXT}
   else
      echo "$FILE does not exist, no need to take action."
   endif

end

# Tag the DART observation file with the valid time of the model state.

\mv -v obs_seq.final obs_seq.final.${DSTART}

#==========================================================================
echo "${JOBNAME} ($JOBID) finished at "`date`
#==========================================================================

exit 0

# <next few lines under version control, do not edit>
# $URL$
# $Revision$
# $Date$

