#!/bin/csh -f
#
# DART software - Copyright UCAR. This open source software is provided
# by UCAR, "as is", without charge, subject to all terms of use at
# http://www.image.ucar.edu/DAReS/DART/DART_download
#
# DART $Id$

# ---------------------
# Purpose
# ---------------------
#
# This script is designed to set up, stage, and build a multi-instance run of CESM
# using a B compset where CAM, POP, and CLM are all active. The initial states for
# the models come from multiple experiments so a CESM hybrid setup is used.
#
# POP: uses the result of the 'b40.20th.005_ens${instance}' experiments. The POP
#      restart files were saved as binary files, which is somewhat problematic for
#      data assimilation. Consequently, the entire model state must be advanced for
#      several days before a viable netCDF restart file can be produces. We advance
#      for 72 hours.
#
# RTM: uses the result of the 'b40.20th.005_ens${instance}' experiments. These
#      restart files are actually the CLM restart files from this experiment because
#      the RTM was part of CLM when the experiment was run. The standalone version of
#      RTM can read old CLM restart files.
#
# CICE: uses the result of the 'b40.20th.005_ens${instance}' experiments.
#
# CLM: uses the result of the 'b40.20th.005_ens${instance}' experiments - sort of.
#      CLM has changed since then and the CLM restart files needed to be converted
#      to the new format. I ran 'interpinic' to (essentially) reformat the files
#      and changed the CASENAME to 'cesm_test' and used the multi-instance naming
#      convention for the new files.
#
# CAM: We want to use a newer version of CAM than was used for the b40.20th.005
#      experiments. Consequently, we took a SINGLE instance of CAM and replicated
#      it for all the desired instances. After just a few days, the differences in
#      the ocean and land states will induce enough variability in the CAM states.
#
# Much of the complexity comes from ensuring compatibility between the namelists
# for each instance and staging of the files. The original experiments were run
# before the multi-instance capacity was developed and the naming convention decided.
# Consequently, there is a lot of manipulation of the 'instance' portion of the
# filenames.
#
# This script results in a viable setup for a CESM multi-instance experiment. You
# are STRONGLY encouraged to run the multi-instance CESM a few times and experiment
# with different settings BEFORE you try to assimilate observations. The amount of
# data volume is quite large and you should become comfortable using CESM's restart
# capability to re-stage files in your RUN directory
#
# CESM/DART requires some modifications to the CESM source code EVEN IF YOU ARE NOT
# assimilating. The modifications for CAM require a modification to the
# CESM ${CASE}.run script. The modifications will invoke a DART script that will
# either assimilate or 'do nothing'. It is strongly recommended that you configure
# the ${CASE}.run script to 'do nothing' initially.
#
# CASEROOT/CESM_DART_config will augment the CESM case with the required setup
# and configuration to perform a DART assimilation and modify the ${CASE}.run script
# as mentioned previously - after it makes a backup copy.
#
# This script relies heavily on the information in:
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
#
# ---------------------
# How to use this script.
# ---------------------
#
# -- You will have to read and understand the script in its entirety.
#    You will have to modify things outside this script.
#    This script sets up a CESM multi-instance run as we understand them and
#    it has almost nothing to do with DART. This is intentional.
#
# -- Examine the whole script to identify things to change for your experiments.
#
# -- Edit this script in the $DART/models/CESM/shell_scripts directory
#    or copy it to somewhere where it will be preserved.
#
# -- Run this script. When it is executed, it will create:
#    1) a CESM 'CASE' directory, where the model will be built,
#    2) a run directory, where each forecast (and assimilation) will take place,
#    3) a bld directory for the executables.
#
#    This script also executes ${CASEROOT}/CESM_DART_config to modify
#    the ${CASEROOT}/${CASE}.run script so that the SourceMods for CAM
#    are effective. CESM_DART_config will also augment the case with all
#    the pieces necessary to run DART when the time comes.
#
# -- (if running DART) Edit the DART input.nml that appears in the ${CASEROOT}
#    directory.
#
# -- Submit the job using ${CASEROOT}/${CASE}.submit
#
# ---------------------
# Important features
# ---------------------
#
# If you want to change something in your case other than the runtime
# settings, it is safest to delete everything and start the run from scratch.
# For the brave, read
#
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1142.html
#
# and you may be able to salvage something with
# ./cesm_setup -clean
# ./cesm_setup
# ./${case}.clean_build
# ./${case}.build
#
# ==============================================================================
# ====  Set case options
# ==============================================================================

# the value of "case" will be used many ways;
#    directory and file names, both locally and on HPSS, and
#    script names; so consider its length and information content.
# num_instances:  Number of ensemble members

setenv case                 cesm_multi
setenv compset              B_2000_CAM5
setenv resolution           0.9x1.25_gx1v6
setenv cesmtag              cesm1_1_1
setenv num_instances        30

# ==============================================================================
# define machines and directories
#
# mach            Computer name
# cesmroot        Location of the cesm code base
#                 For cesm1_1_1 on yellowstone
# caseroot        Your (future) CESM case directory, where this CESM+DART will be built.
#                    Preferably not a frequently scrubbed location.
#                    This script will delete any existing caseroot, so this script,
#                    and other useful things should be kept elsewhere.
# rundir          (Future) Run-time directory; scrubbable, large amount of space needed.
# exeroot         (Future) directory for executables - scrubbable, large amount of space needed.
# archdir         (Future) Short-term archive directory
#                    until the long-term archiver moves it to permanent storage.
# dartroot        Location of _your_ DART installation
#                    This is passed on to the CESM_DART_config script.
# ==============================================================================

setenv mach         yellowstone

setenv cesmroot     /glade/p/cesm/cseg/collections/$cesmtag
setenv caseroot     /glade/p/work/${USER}/cases/${case}
setenv exeroot      /glade/scratch/${USER}/${case}/bld
setenv rundir       /glade/scratch/${USER}/${case}/run
setenv archdir      /glade/scratch/${USER}/archive/${case}
setenv dartroot     /glade/u/home/${USER}/svn/DART/trunk

# ==============================================================================
# configure settings
# The reference case has dates in it.
# For a 'hybrid' start, these may be unrelated to the refyear, refmon, refday.
# ==============================================================================

setenv run_refcase cesm_spinup
setenv refyear     2004
setenv refmon      01
setenv refday      01
setenv run_reftod  00000
setenv run_refdate $refyear-$refmon-$refday

# The RTM and CLM directories are hpss:~thoar/DART_CESM_ensemble_2004_01_01.tar
set RTM_stagedir = /glade/scratch/thoar/DART_POP_RESTARTS/2004-01-01-00000
set CLM_stagedir = /glade/scratch/thoar/DART_POP_RESTARTS/CLM_2004-01-01-00000
set CAM_stagedir = /glade/p/cesm/cseg/inputdata/atm/cam/inic/fv
set POP_stagedir = /glade/p/work/aliciak/DART_IC/CCSM4_ensembles/rest/2004-01-01-00000

# ==============================================================================
# runtime settings
#
# resubmit      How many job steps to run on continue runs (will be 0 initially)
# stop_option   Units for determining the forecast length between assimilations
# stop_n        Number of time units in the first forecast
# assim_n       Number of time units between assimilations
# clm_dtime     CLM dynamical timestep (in seconds) ... 1800 is the default
# h1nsteps      is the number of time steps to put in a single CLM .h1. file
#               DART needs to know this and the only time it is known is during
#               this configuration step. The run-time settings can lie.
#
# If the long-term archiver is off, you get a chance to examine the files before
# they get moved to long-term storage. You can always submit $CASE.l_archive
# whenever you want to free up space in the short-term archive directory.
#
# ==============================================================================

setenv short_term_archiver on
setenv long_term_archiver  off
setenv resubmit            0
setenv stop_option         nhours
setenv stop_n              72
setenv assim_n             6

# This specifies the number of time steps available for a CLM forward obs. operator.
# with clm_dtime = 1800 and h1nsteps = $assim_n * 3600 / $clm_dtime, the CLM .h1.
# file is created with values every 30 minutes.

@ clm_dtime = 1800
@ h1nsteps = $assim_n * 3600 / $clm_dtime

# ==============================================================================
# job settings
#
# queue      can be changed during a series by changing the ${case}.run
# timewall   can be changed during a series by changing the ${case}.run
#
# TJH: Advancing 30 instances for 72 hours with 900 pes
#      took about 30 minutes on yellowstone.
# ==============================================================================

setenv ACCOUNT      P8685xxxx
setenv queue        economy
setenv timewall     1:00

# ==============================================================================
# set these standard commands based on the machine you are running on.
# ==============================================================================

set nonomatch       # suppress "rm" warnings if wildcard does not match anything

# The FORCE options are not optional.
# The VERBOSE options are useful for debugging though
# some systems don't like the -v option to any of the following
switch ("`hostname`")
   case be*:
      # NCAR "bluefire"
      set   MOVE = '/usr/local/bin/mv -fv'
      set   COPY = '/usr/local/bin/cp -fv --preserve=timestamps'
      set   LINK = '/usr/local/bin/ln -fvs'
      set REMOVE = '/usr/local/bin/rm -fr'

   breaksw
   default:
      # NERSC "hopper", NWSC "yellowstone"
      set   MOVE = '/bin/mv -fv'
      set   COPY = '/bin/cp -fv --preserve=timestamps'
      set   LINK = '/bin/ln -fvs'
      set REMOVE = '/bin/rm -fr'

   breaksw
endsw

# ==============================================================================
# Make sure the CESM directories exist.
# VAR is the shell variable name, DIR is the value
# ==============================================================================

foreach VAR ( cesmroot dartroot RTM_stagedir CLM_stagedir CAM_stagedir POP_stagedir )
   set DIR = `eval echo \${$VAR}`
   if ( ! -d $DIR ) then
      echo "ERROR: directory '$DIR' not found"
      echo " In the setup script check the setting of: $VAR "
      exit -1
   endif
end

# ==============================================================================
# Create the case - this creates the CASEROOT directory.
#
# For list of the pre-defined component sets: ./create_newcase -list
# To create a variant compset, see the CESM documentation and carefully
# incorporate any needed changes into this script.
# ==============================================================================

# fatal idea to make caseroot the same dir as where this setup script is
# since the build process removes all files in the caseroot dir before
# populating it.  try to prevent shooting yourself in the foot.

if ( $caseroot == `dirname $0` ) then
   echo "ERROR: the setup script should not be located in the caseroot"
   echo "directory, because all files in the caseroot dir will be removed"
   echo "before creating the new case.  move the script to a safer place."
   exit -1
endif

echo "removing old files from ${caseroot}"
echo "removing old files from ${exeroot}"
echo "removing old files from ${rundir}"
${REMOVE} ${caseroot}
${REMOVE} ${exeroot}
${REMOVE} ${rundir}

${cesmroot}/scripts/create_newcase -case ${caseroot} -mach ${mach} \
                -res ${resolution} -compset ${compset}

if ( $status != 0 ) then
   echo "ERROR: Case could not be created."
   exit -1
endif

# preserve a copy of this script as it was run
set ThisFileName = $0:t
${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original

# ==============================================================================
# Record the DARTROOT directory and copy the DART setup script to CASEROOT.
# CESM_DART_config can be run at some later date if desired, but it presumes
# to be run from a CASEROOT directory. If CESM_DART_config does not exist locally,
# then it better exist in the expected part of the DARTROOT tree.
# ==============================================================================

if ( ! -e CESM_DART_config ) then
   ${COPY} ${dartroot}/models/CESM/shell_scripts/CESM_DART_config .
endif

if (   -e CESM_DART_config ) then
   sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" \
       -e "s#HISTORY_OUTPUT_INTERVAL#$assim_n#" < CESM_DART_config >! temp.$$
   ${MOVE} temp.$$ ${caseroot}/CESM_DART_config
   chmod 755       ${caseroot}/CESM_DART_config
else
   echo "ERROR: the script to configure for data assimilation is not available."
   echo "       CESM_DART_config MUST be present locally or in"
   echo "       ${dartroot}/models/CESM/shell_scripts/"
   exit -2
endif

# ==============================================================================
# Configure the case.
# ==============================================================================

cd ${caseroot}

source ./Tools/ccsm_getenv || exit -2

@ ptile = $MAX_TASKS_PER_NODE / 2
@ nthreads = 1

# Save a copy for debug purposes
foreach FILE ( *xml )
   if ( ! -e        ${FILE}.original ) then
      ${COPY} $FILE ${FILE}.original
   endif
end

if ($num_instances < 10) then

   # This is only for the purpose of debugging the code.
   @ atm_tasks = $ptile * $num_instances * 4
   @ lnd_tasks = $ptile * $num_instances * 4
   @ ice_tasks = $ptile * $num_instances * 4
   @ ocn_tasks = $ptile * $num_instances * 4
   @ cpl_tasks = $ptile * $num_instances
   @ glc_tasks = $ptile * $num_instances
   @ rof_tasks = $ptile * $num_instances * 4

else

   # This works, but a more efficient layout should be used
   @ atm_tasks = $ptile * $num_instances * 2
   @ lnd_tasks = $ptile * $num_instances * 2
   @ ice_tasks = $ptile * $num_instances * 2
   @ ocn_tasks = $ptile * $num_instances * 2
   @ cpl_tasks = $ptile * $num_instances
   @ glc_tasks = $ptile * $num_instances
   @ rof_tasks = $ptile * $num_instances * 2

endif

# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
# presently, all components run 'serially' - one after another.
echo ""
echo "ATM gets $atm_tasks"
echo "LND gets $lnd_tasks"
echo "ICE gets $ice_tasks"
echo "OCN gets $ocn_tasks"
echo "CPL gets $cpl_tasks"
echo "GLC gets $glc_tasks"
echo "ROF gets $rof_tasks"
echo ""

./xmlchange NTHRDS_ATM=$nthreads,NTASKS_ATM=$atm_tasks,NINST_ATM=$num_instances
./xmlchange NTHRDS_LND=$nthreads,NTASKS_LND=$lnd_tasks,NINST_LND=$num_instances
./xmlchange NTHRDS_ICE=$nthreads,NTASKS_ICE=$ice_tasks,NINST_ICE=$num_instances
./xmlchange NTHRDS_OCN=$nthreads,NTASKS_OCN=$ocn_tasks,NINST_OCN=$num_instances
./xmlchange NTHRDS_CPL=$nthreads,NTASKS_CPL=$cpl_tasks
./xmlchange NTHRDS_GLC=$nthreads,NTASKS_GLC=$glc_tasks,NINST_GLC=1
./xmlchange NTHRDS_ROF=$nthreads,NTASKS_ROF=$rof_tasks,NINST_ROF=$num_instances
./xmlchange ROOTPE_ATM=0
./xmlchange ROOTPE_LND=0
./xmlchange ROOTPE_ICE=0
./xmlchange ROOTPE_OCN=0
./xmlchange ROOTPE_CPL=0
./xmlchange ROOTPE_GLC=0
./xmlchange ROOTPE_ROF=0

# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/c1158.html#run_start_stop
# "A hybrid run indicates that CESM is initialized more like a startup, but uses
# initialization datasets from a previous case. This is somewhat analogous to a
# branch run with relaxed restart constraints. A hybrid run allows users to bring
# together combinations of initial/restart files from a previous case (specified
# by $RUN_REFCASE) at a given model output date (specified by $RUN_REFDATE).
# Unlike a branch run, the starting date of a hybrid run (specified by $RUN_STARTDATE)
# can be modified relative to the reference case. In a hybrid run, the model does not
# continue in a bit-for-bit fashion with respect to the reference case. The resulting
# climate, however, should be continuous provided that no model source code or
# namelists are changed in the hybrid run. In a hybrid initialization, the ocean
# model does not start until the second ocean coupling (normally the second day),
# and the coupler does a "cold start" without a restart file."

# TJH:
# DART's CAM implementation causes a bit more complexity. DART only uses CAM _initial_
# files, not RESTART files, so there are sourcemods to force a hybrid start for CAM to
# read initial files - even when CONTINUE_RUN = TRUE. A hybrid start is better for POP
# because the velocities are used rather than just T,S. A hybrid start is also more
# desirable because initial values can be specified for ROF - as opposed to just zeros.
#
# The RUN_REFCASE/REFDATE/REFTOD  are used by CLM & RTM to specify the namelist input
# filenames - BUT - their buildnml scripts do not use the INSTANCE, so they all specify
# the same (single) filename. This is remedied by using patched [clm,rtm].buildnml.csh
# scripts that exist in the SourceMods directory.

./xmlchange RUN_TYPE=hybrid
./xmlchange RUN_STARTDATE=$run_refdate
./xmlchange START_TOD=$run_reftod
./xmlchange RUN_REFCASE=$run_refcase
./xmlchange RUN_REFDATE=$run_refdate
./xmlchange RUN_REFTOD=$run_reftod
./xmlchange BRNCH_RETAIN_CASENAME=FALSE
./xmlchange GET_REFCASE=FALSE
./xmlchange EXEROOT=${exeroot}

./xmlchange CALENDAR=GREGORIAN

./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
./xmlchange CONTINUE_RUN=FALSE
./xmlchange RESUBMIT=$resubmit

./xmlchange PIO_TYPENAME=pnetcdf

# COUPLING discussion.
# http://bugs.cgd.ucar.edu/show_bug.cgi?id=1740
# "In summary, users should ensure that the following is true,
#  ATM_NCPL = LND_NCPL = ICE_NCPL >= ROF_NCPL >= OCN_NCPL"
#
# OCN_NCPL == 4 sets the ocean coupling time to 6 hours.
# All related namelist settings are based on this value.
# OCN_NCPL is # coupling intervals per NCPL_BASE_PERIOD (default 'day')

./xmlchange ROF_NCPL=8
./xmlchange OCN_NCPL=4

# There is no compset that has everything we want,
# so we are also turning on the CLM biogeochemistry.
./xmlchange CLM_CONFIG_OPTS='-bgc cn'

if ($short_term_archiver == 'off') then
   ./xmlchange DOUT_S=FALSE
else
   ./xmlchange DOUT_S=TRUE
   ./xmlchange DOUT_S_ROOT=${archdir}
   ./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
endif
if ($long_term_archiver == 'off') then
   ./xmlchange DOUT_L_MS=FALSE
else
   ./xmlchange DOUT_L_MS=TRUE
   ./xmlchange DOUT_L_MSROOT="csm/${case}"
   ./xmlchange DOUT_L_HTAR=FALSE
endif

# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)

./xmlchange DEBUG=FALSE
./xmlchange INFO_DBUG=0

# ==============================================================================
# Set up the case.
# This creates the EXEROOT and RUNDIR directories.
# ==============================================================================

./cesm_setup

if ( $status != 0 ) then
   echo "ERROR: Case could not be set up."
   exit -2
endif

# ==============================================================================
# Edit the run script to reflect queue and wallclock
# ==============================================================================

echo ''
echo 'Updating the run script to set wallclock and queue.'
echo ''

if ( ! -e  ${case}.run.original ) then
   ${COPY} ${case}.run ${case}.run.original
endif

source Tools/ccsm_getenv
set BATCH = `echo $BATCHSUBMIT | sed 's/ .*$//'`
switch ( $BATCH )
   case bsub*:
      # NCAR "bluefire", "yellowstone"
      set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
      set    QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
      sed -e "s/$TIMEWALL[3]/$timewall/" \
          -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
          -e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
          ${MOVE} temp.$$ ${case}.run
          chmod 755       ${case}.run
   breaksw

   default:

   breaksw
endsw

# This is the part that modifies the run script to allow CESM to advance
# correctly given the modifications of CAM for DART. It also copies several
# required DART files to the caseroot directory.

./CESM_DART_config

# ==============================================================================
# Update source files.
#    Ideally, using DART would not require any modifications to the model source.
#    Until then, this script accesses sourcemods from a hardwired location.
#    If you have additional sourcemods, they will need to be merged into any DART
#    mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
# ==============================================================================

if (    -d     ~/${cesmtag}/SourceMods ) then
   ${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
else
   echo "ERROR - No SourceMods for this case."
   echo "ERROR - No SourceMods for this case."
   echo "DART requires modifications to several src files."
   echo "These files can be downloaded from:"
   echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1_24Oct2013.tar"
   echo "untar these into your HOME directory - they will create a"
   echo "~/cesm_1_1_1  directory with the appropriate SourceMods structure."
   exit -4
endif

# The CESM multi-instance capability is relatively new and still has a few
# implementation bugs. These are known problems and will be fixed soon.
# this should be removed when the files are fixed:

echo "REPLACING BROKEN CESM FILES HERE - SHOULD BE REMOVED WHEN FIXED"
echo caseroot is ${caseroot}
if ( -d ~/${cesmtag} ) then

   # preserve the original version of the files
   if ( ! -e  ${caseroot}/Buildconf/clm.buildnml.csh.original ) then
      ${COPY} ${caseroot}/Buildconf/clm.buildnml.csh \
              ${caseroot}/Buildconf/clm.buildnml.csh.original
   endif
   if ( ! -e  ${caseroot}/Buildconf/rtm.buildnml.csh.original ) then
      ${COPY} ${caseroot}/Buildconf/rtm.buildnml.csh \
              ${caseroot}/Buildconf/rtm.buildnml.csh.original
   endif
   if ( ! -e  ${caseroot}/preview_namelists.original ) then
      ${COPY} ${caseroot}/preview_namelists \
              ${caseroot}/preview_namelists.original
   endif

   # patch/replace the broken files
   ${COPY} ~/${cesmtag}/clm.buildnml.csh  ${caseroot}/Buildconf/.
   ${COPY} ~/${cesmtag}/rtm.buildnml.csh  ${caseroot}/Buildconf/.
   ${COPY} ~/${cesmtag}/preview_namelists ${caseroot}/.

endif

# ==============================================================================
# Modify namelist templates for each instance. This is a bit of a nuisance in
# that we are pulling in restart and initial files from 'all over the place'
# and each model component has a different strategy.
#
# When CONTINUE_RUN = FALSE (i.e. just starting up):
#
# CAM has been forced to read initial files - specified by namelist var:ncdata
# POP reads from pointer files
# CICE reads from namelist variable 'ice_ic'
# CLM builds its own 'finidat' value from the REFCASE variables but in CESM1_1_1
#     it does not use the instance string. There is a patch for clm.buildnml.csh
# RTM reads from namelist variable 'finidat_rtm', but rtm.buildnml.csh also is buggy.
#
# All of these must later on be staged with these same filenames.
# OR - all these namelists can be changed to match whatever has been staged.
# MAKE SURE THE STAGING SECTION OF THIS SCRIPT MATCHES THESE VALUES.
# ==============================================================================

@ inst = 1
while ($inst <= $num_instances)

   # following the CESM strategy for 'inst_string'
   set inst_string = `printf _%04d $inst`

   # special for some files: no leading underscore, 2 digits
   set instance2  = `printf %02d $inst`

   # ===========================================================================
   set fname = "user_nl_cam${inst_string}"
   # ===========================================================================
   # For a HOP TEST ... empty_htapes = .false.
   # For a HOP TEST ... use a default fincl1
   # inithist == 'ENDOFRUN' ensures that CAM writes an initial file every time it stops.
   # fincl1,nhtfrq,mfilt all control the history file containing a REQUIRED PHIS field.

   echo " inithist      = 'ENDOFRUN'"                     >> ${fname}
   echo " ncdata        = 'cam_initial${inst_string}.nc'" >> ${fname}
   echo " empty_htapes  = .true. "                        >> ${fname}
   echo " fincl1        = 'PHIS:I' "                      >> ${fname}
   echo " nhtfrq        = -$assim_n "                     >> ${fname}
   echo " mfilt         = 1 "                             >> ${fname}

   # ===========================================================================
   set fname = "user_nl_clm${inst_string}"
   # ===========================================================================

   # Customize the land namelists
   # The filename is built using the REFCASE/REFDATE/REFTOD information.
   #
   # This is the time to consider how DART and CESM will interact.  If you intend
   # on assimilating flux tower observations (nominally at 30min intervals),
   # then it is required to create a .h1. file with the instantaneous flux
   # variables every 30 minutes. Despite being in a namelist, these values
   # HAVE NO EFFECT once CONTINUE_RUN = TRUE so now is the time to set these.
   # Check the value of h1nsteps considering STOP_N.
   #
   # DART's forward observation operators for these fluxes just reads them
   # from the .h1. file rather than trying to create them from the subset of
   # CLM variables that are available in the DART state vector.
   #
   # For a HOP TEST ... hist_empty_htapes = .false.
   # For a HOP TEST ... use a default hist_fincl1
   #
   # FIXME ... add documentation for configuring CLM history files

   echo "dtime             = $clm_dtime,"            >> ${fname}
   echo "hist_empty_htapes = .true."                 >> ${fname}
   echo "hist_fincl1 = 'NEP'"                        >> ${fname}
   echo "hist_fincl2 = 'NEP','FSH','EFLX_LH_TOT_R'"  >> ${fname}
   echo "hist_nhtfrq = -$assim_n,1,"                 >> ${fname}
   echo "hist_mfilt  = 1,$h1nsteps"                  >> ${fname}
   echo "hist_avgflag_pertape = 'A','A'"             >> ${fname}

   # ===========================================================================
   set fname = "user_nl_pop2${inst_string}"
   # ===========================================================================

   # POP Namelists
   # init_ts_suboption = 'data_assim'   for non bit-for-bit restarting (assimilation mode)
   # init_ts_suboption = 'rest'         --> default behavior
   #
   # README:
   # Configuring the contents of the history files for POP is best explained in
   # the section marked "POP2: TAVG Settings" in the cesm1_1_1 pop2 namelist documentation
   # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/modelnl/nl_pop2.html
   #
   # and the CESM-specific documentation for the tavg output variables in the pop2
   # online documentation:
   # http://www.cesm.ucar.edu/models/cesm1.1/pop2/doc/users/node78.html
   # No matter the setting of init_ts_suboption, POP always uses the
   # 'data_assim' behavior when RUN_TYPE=hybrid and CONTINUE_RUN=FALSE.
   # As soon as CONTINUE_RUN=TRUE, the value becomes important.
   #
   # In CESM1_1_1 keep the values for tavg_file_freq_opt and tavg_freq_opt identical.
   # pop2/trunk_tags/cesm_pop_2_1_20130412  explains the issue.
   #
   # DEFAULT values for these are:
   # tavg_file_freq_opt = 'nmonth' 'nmonth' 'once'
   # tavg_freq_opt      = 'nmonth' 'nday'   'once'
   # The  first entry indicates we get a monthly average once a month.
   # The second entry indicates we get a monthly average as it is being created.
   # The  third entry indicates  we get a daily timeslice
   #
   # Default copies of SourceMods/src.pop2/ocn.*.tavg.csh files are provided in the
   # DART_SourceMods_cesm1_1_1.tar bundle.
   #
   # IFF values for these are:
   # tavg_file_freq_opt = 'nmonth' 'never' 'never'
   # tavg_freq_opt      = 'nmonth' 'never' 'never'
   # The  first entry indicates we get a monthly average once a month, and thats all we get..
   #
   # For a HOP TEST ... Would like to have restart files every day, not just for end.
   # For a HOP TEST (untested)... tavg_file_freq_opt = 'nmonth' 'nday' 'once'"

   echo "init_ts_suboption  = 'data_assim'" >> ${fname}

   # ===========================================================================
   set fname = "user_nl_cice${inst_string}"
   # ===========================================================================
   # CICE Namelists

   echo "ice_ic = 'b40.20th.005_ens${instance2}.cice.r.2004-01-01-00000.nc'" >> ${fname}

   # ===========================================================================
   set fname = "user_nl_rtm${inst_string}"
   # ===========================================================================
   # RIVER RUNOFF CAN START FROM AN OLD CLM RESTART FILE
   # you can specify the RTM filename here and override the settings from
   # RUN_REFCASE/RUN_REFDATE/RUN_REFTOD (something you cannot do with CLM).

   echo "finidat_rtm = 'b40.20th.005_ens${instance2}.clm2.r.2004-01-01-00000.nc'" >> ${fname}

   @ inst ++
end

./preview_namelists

# ==============================================================================
# Stage the restarts now that the run directory exists
# ==============================================================================

cat << EndOfText >! stage_initial_cesm_files
#!/bin/sh

cd ${rundir}

echo ''
echo "Copying the restart files from the staging directories"
echo 'into the CESM run directory and creating the pointer files.'
echo ''

let inst=1
while ((\$inst <= $num_instances)); do
   n4=\`printf %04d \$inst\`
   n2=\`printf %02d \$inst\`

   echo ''
   echo "Staging restarts for instance \$inst of $num_instances"

   ${LINK} ${POP_stagedir}/b40.20th.005_ens\${n2}.pop.r.2004-01-01-00000      .
   ${LINK} ${POP_stagedir}/b40.20th.005_ens\${n2}.pop.r.2004-01-01-00000.hdr  .
   ${LINK} ${POP_stagedir}/b40.20th.005_ens\${n2}.pop.ro.2004-01-01-00000     .
   ${LINK} ${POP_stagedir}/b40.20th.005_ens\${n2}.cice.r.2004-01-01-00000.nc  .
   ${LINK} ${RTM_stagedir}/b40.20th.005_ens\${n2}.clm2.r.2004-01-01-00000.nc  .

   ${LINK} ${CAM_stagedir}/cami-mam3_0000-01-01_0.9x1.25_L30_c100618.nc  cam_initial_\${n4}.nc

   ${LINK} ${CLM_stagedir}/${run_refcase}.clm2_\${n4}.r.${run_refdate}-${run_reftod}.nc .

   echo "cam_initial_\${n4}.nc"                              >| rpointer.atm_\${n4}
   echo "b40.20th.005_ens\${n2}.cice.r.2004-01-01-00000.nc"  >| rpointer.ice_\${n4}
   echo "b40.20th.005_ens\${n2}.pop.ro.2004-01-01-00000"     >| rpointer.ocn_\${n4}.ovf
   echo "b40.20th.005_ens\${n2}.pop.r.2004-01-01-00000"      >| rpointer.ocn_\${n4}.restart
   echo "RESTART_FMT=bin"                                    >> rpointer.ocn_\${n4}.restart

   let inst+=1
done

exit 0

EndOfText
chmod 0755 stage_initial_cesm_files

./stage_initial_cesm_files

# ==============================================================================
# build
# ==============================================================================

echo ''
echo 'Building the case'
echo ''

./${case}.build

if ( $status != 0 ) then
   echo "ERROR: Case could not be built."
   exit -5
endif

# ==============================================================================
# What to do next
# ==============================================================================

echo ""
echo "Time to check the case."
echo ""
echo "1) cd ${rundir}"
echo "   and check the compatibility between the namelists/pointer"
echo "   files and the files that were staged."
echo ""
echo "2) cd ${caseroot}"
echo "   (on yellowstone) If the ${case}.run script still contains:"
echo '   #BSUB -R "select[scratch_ok > 0]"'
echo "   around line 9, delete it."
echo ""
echo "3) The very first CESM advance (i.e. CONTINUE_RUN=FALSE)"
echo "   STOP_N must be longer than *AT LEAST 2 TIMES* the coupling"
echo "   frequency between the atmosphere and ocean."
echo "   If coupling once a day, the first advance MUST be at least 48 hours."
echo "   If coupling 4 times a day, the first advance MUST be at least 12 hours."
echo "   After that, STOP_N can be as short as a single coupling frequency."
echo ""
echo "4) The default action is to simply advance CESM ... i.e. NO ASSIMILATION."
echo "   If you want to assimilate, make sure the ${case}.run script"
echo "   references the 'assimilate.csh' script."
echo ""
echo "5) Verify the contents of env_run.xml and submit the CESM job:"
echo "   ./${case}.submit"
echo ""
echo "6) After the job has run, check to make sure it worked and that"
echo "   a: POP is creating netCDF restart files,"
echo "   b: the right restart files exist in the run directory,"
echo "   c: (if you're running DART) the archive dart/hist directory has the DART output,"
echo "   d: everything is working correctly ..."
echo ""
echo "7) To extend the run in $assim_n '"$stop_option"' steps,"
echo "   change the env_run.xml variables:"
echo ""
echo "   ./xmlchange CONTINUE_RUN=TRUE"
echo "   ./xmlchange RESUBMIT=<number_of_cycles_to_run>"
echo "   ./xmlchange STOP_N=$assim_n"
echo ""
echo "   and"
echo "   ./${case}.submit"

cat ${caseroot}/DART_instructions.txt

exit 0

# <next few lines under version control, do not edit>
# $URL$
# $Revision$
# $Date$

