#!/bin/csh -f
#
# DART software - Copyright UCAR. This open source software is provided
# by UCAR, "as is", without charge, subject to all terms of use at
# http://www.image.ucar.edu/DAReS/DART/DART_download
#
# DART $Id$

#*******************************************************************************
#
# ---------------------
# Purpose
# ---------------------
#
# This script is designed to set up, stage, and build a multi-instance run
# of CESM using an F compset where CAM, CLM and CICE are active. The initial state
# come from a single multi-instance reference case so a CESM hybrid setup
# is used. Instructions on what to change to use the SE core or WACCM are
# outlined in the models/cam/model_mod.html documentation.
#
# DOCN: We are using a single data ocean.
#
# Because the atmosphere assimilations typically occur every 6 hours,
# the methodology here reflects that. All of CESM stops every 6 hours
# so that a CAM output file would be available for assimilation.
#
# CESM/DART requires some modifications to the CESM source code EVEN IF YOU
# ARE NOT USING DART. The modifications for CAM require a change to the CESM
# ${CASE}.run script to invoke a DART script that will allow CAM to cycle
# correctly with the source code modifications. Changing one line will
# allow you to invoke DART.
#
# This script results in a viable setup for a CESM multi-instance experiment.
# You are STRONGLY encouraged to run the multi-instance CESM a few times and
# experiment with different settings BEFORE you try to assimilate observations.
# The data volume is quite large and you should become comfortable using
# CESM's restart capability to re-stage files in your RUN directory.
#
# ${CASEROOT}/CESM_DART_config is automatically run by this script and will
# augment the CESM case with the required setup and configuration to use DART
# to perform an assimilation. CESM_DART_config will insert a few dozen
# lines into the ${case}.run script after it makes a backup copy.
#
# This script relies heavily on the information in:
# http://www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/book1.html
#
# ---------------------
# How to use this script.
# ---------------------
#
# -- You will have to read and understand the script in its entirety.
#    You will have to modify things outside this script.
#    This script sets up a plain CESM multi-instance run without DART,
#    intentionally.  Once it is running, calls to DART can be added.
#
# -- Examine the whole script to identify things to change for your experiments.
#
# -- Edit this script in the $DART/models/cam/shell_scripts directory
#    or copy it to somewhere where it will be preserved.
#
# -- Locate the initial multi-instance files that CESM will need.
#
# -- Run this script. When it is executed, it will create:
#    1) a CESM 'CASE' directory, where the model will be built,
#    2) a run directory, where each forecast (and assimilation) will take place,
#    3) a bld directory for the executables.
#    4) The short term archiver will use a fourth directory for
#    storage of model output until it can be moved to long term storage (HPSS)
#
#    This script also executes ${CASEROOT}/CESM_DART_config to modify
#    the ${CASEROOT}/${CASE}.run script so that the SourceMods for CAM
#    are effective. CESM_DART_config will also augment the case with all
#    the pieces necessary to run DART when the time comes.
#
# -- (if running DART) Edit the DART input.nml that appears in the ${CASEROOT}
#    directory.
#
# -- Submit the job using ${CASEROOT}/${CASE}.submit
#
# ---------------------
# Important features
# ---------------------
#
# If you want to change something in your case other than the runtime
# settings, it is safest to delete everything and start the run from scratch.
# For the brave, read
#
# http://www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/x1080.html
#
# and you may be able to salvage something with
# ./cesm_setup -clean
# ./cesm_setup
# ./${case}.clean_build
# ./${case}.build
#
#*******************************************************************************

# ==============================================================================
# case options:
#
# case          The value of "case" will be used many ways; directory and file
#               names both locally and on HPSS, and script names; so consider
#               its length and information content.
# compset       Must be one of the CESM standard names, see the CESM documentation
#               for supported strings.
# resolution    Sets the model grid resolution, see the CESM documentation.
#               f09_f09  ... FV core at ~ 1 degree
#               BUG 1384 applies here, so ocean and atm/land must be at same resolution.
# cesmtag       The version of the CESM source code to use when building the code.
# num_instances The number of ensemble members.
#
# Guidelines on what to change for an SE or WACCM run are described in the 
# models/cam/model_mod.html documentation.
# ==============================================================================
# AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV (F_AMIP_CAM5) (FAMIPC5)

setenv case                 cam1_2_1_hybrid
setenv compset              F_AMIP_CAM5
setenv resolution           f09_f09
setenv cesmtag              cesm1_2_1
setenv num_instances        30

# ==============================================================================
# machines and directories:
#
# mach            Computer name
# cesmdata        Location of some supporting CESM data files.
# cesmroot        Location of the CESM code base.  This version of the script
#                 only supports version cesm1_2_1.
# caseroot        Will create the CESM case directory here, where the CESM+DART
#                 configuration files will be stored.  This should probably not
#                 be in scratch (on yellowstone, your 'work' partition is suggested).
#                 This script will delete any existing caseroot, so this script,
#                 and other useful things should be kept elsewhere.
# rundir          Will create the CESM run directory here.  Will need large
#                 amounts of disk space, generally on a scratch partition.
# exeroot         Will create the CESM executable directory here, where the
#                 CESM executables will be built.  Medium amount of space
#                 needed, generally on a scratch partition.
# archdir         Will create the CESM short-term archive directories here.
#                 Large, generally on a scratch partition.  Files will remain
#                 here until the long-term archiver moves it to permanent storage.
# dartroot        Location of the root of _your_ DART installation
# ==============================================================================

setenv mach         yellowstone
setenv cesmdata     /glade/p/cesm/cseg/inputdata
setenv cesmroot     /glade/p/cesm/releases/$cesmtag
setenv caseroot     /glade/p/work/${USER}/cases/${case}
setenv rundir       /glade/scratch/${USER}/${case}/run
setenv exeroot      /glade/scratch/${USER}/${case}/bld
setenv archdir      /glade/scratch/${USER}/archive/${case}
setenv dartroot     /glade/u/home/${USER}/svn/DART

# ==============================================================================
# configure settings:
#
# refcase    The name of the existing reference case that this run will
#            start from.
#
# refyear    The specific date/time-of-day in the reference case that this
# refmon     run will start from.  (Also see 'runtime settings' below for
# refday     start_year, start_mon, start_day and start_tod.)
# reftod
# NOTE:      all the ref* variables must be treated like strings and have
#            the appropriate number of preceeding zeros 
#
# stagedir   The directory location of the reference case files.
# ==============================================================================
# alternative reference case for different times may be available here:
# or on the HPSS:
# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1         for 1-degree FV ensembles
# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}

setenv refcase     cesm_hybrid
setenv refyear     2004
setenv refmon      01
setenv refday      10
setenv reftod      00000

# useful combinations of time that we use below
setenv refdate      $refyear-$refmon-$refday
setenv reftimestamp $refyear-$refmon-$refday-$reftod

setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}

# ==============================================================================
# runtime settings:
#
# start_year     generally this is the same as the reference case date, but it can
# start_month    be different if you want to start this run as if it was a different time.
# start_day
# start_tod
#
# short_term_archiver  Copies the files from each job step to a 'rest' directory.
# long_term_archiver   Puts the files from all completed steps on tape storage.
#
# resubmit      How many job steps to run on continue runs (should be 0 initially)
# stop_option   Units for determining the forecast length between assimilations
# stop_n        Number of time units in each forecast
#
# If the long-term archiver is off, you get a chance to examine the files before
# they get moved to long-term storage. You can always submit $CASE.l_archive
# whenever you want to free up space in the short-term archive directory.
# ==============================================================================

setenv start_year    2004
setenv start_month   01
setenv start_day     10
setenv start_tod     00000

setenv short_term_archiver off
setenv long_term_archiver  off

setenv resubmit            0
setenv stop_option         nhours
setenv stop_n              6

# ==============================================================================
# job settings:
#
# queue      can be changed during a series by changing the ${case}.run
# timewall   can be changed during a series by changing the ${case}.run
#
# TJH: Advancing 30 instances for 6 hours and assimilating took
#      less than 10 minutes on yellowstone using 1800 pes (120 nodes)
# ==============================================================================

setenv ACCOUNT      ZZZZZZZZ
setenv queue        economy
setenv timewall     0:20

# ==============================================================================
# standard commands:
#
# If you are running on a machine where the standard commands are not in the
# expected location, add a case for them below.
# ==============================================================================

set nonomatch       # suppress "rm" warnings if wildcard does not match anything

# The FORCE options are not optional.
# The VERBOSE options are useful for debugging though
# some systems don't like the -v option to any of the following
switch ("`hostname`")
   case be*:
      # NCAR "bluefire"
      set   MOVE = '/usr/local/bin/mv -fv'
      set   COPY = '/usr/local/bin/cp -fv --preserve=timestamps'
      set   LINK = '/usr/local/bin/ln -fvs'
      set REMOVE = '/usr/local/bin/rm -fr'

   breaksw
   default:
      # NERSC "hopper", NWSC "yellowstone"
      set   MOVE = '/bin/mv -fv'
      set   COPY = '/bin/cp -fv --preserve=timestamps'
      set   LINK = '/bin/ln -fvs'
      set REMOVE = '/bin/rm -fr'

   breaksw
endsw

# ==============================================================================
# ==============================================================================
# by setting the values above you should be able to execute this script and
# have it run.  however, for running a real experiment there are still many
# settings below this point - e.g. component namelists, history file options,
# the processor layout, xml file options, etc - that you will almost certainly
# want to change before doing a real science run.
# ==============================================================================
# ==============================================================================


# ==============================================================================
# Make sure the CESM directories exist.
# VAR is the shell variable name, DIR is the value
# ==============================================================================

foreach VAR ( cesmroot dartroot stagedir )
   set DIR = `eval echo \${$VAR}`
   if ( ! -d $DIR ) then
      echo "ERROR: directory '$DIR' not found"
      echo " In the setup script check the setting of: $VAR "
      exit -1
   endif
end

# ==============================================================================
# Create the case - this creates the CASEROOT directory.
#
# For list of the pre-defined component sets: ./create_newcase -list
# To create a variant compset, see the CESM documentation and carefully
# incorporate any needed changes into this script.
# ==============================================================================

# fatal idea to make caseroot the same dir as where this setup script is
# since the build process removes all files in the caseroot dir before
# populating it.  try to prevent shooting yourself in the foot.

if ( $caseroot == `dirname $0` ) then
   echo "ERROR: the setup script should not be located in the caseroot"
   echo "directory, because all files in the caseroot dir will be removed"
   echo "before creating the new case.  move the script to a safer place."
   exit -1
endif

echo "removing old files from ${caseroot}"
echo "removing old files from ${exeroot}"
echo "removing old files from ${rundir}"
${REMOVE} ${caseroot}
${REMOVE} ${exeroot}
${REMOVE} ${rundir}

${cesmroot}/scripts/create_newcase -case ${caseroot} -mach ${mach} \
                -res ${resolution} -compset ${compset}

if ( $status != 0 ) then
   echo "ERROR: Case could not be created."
   exit -1
endif

# Preserve a copy of this script as it was run.
set ThisFileName = $0:t
${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original

# ==============================================================================
# Record the DARTROOT directory and copy the DART setup script to CASEROOT.
# CESM_DART_config can be run at some later date if desired, but it presumes
# to be run from a CASEROOT directory. If CESM_DART_config does not exist locally,
# then it better exist in the expected part of the DARTROOT tree.
# ==============================================================================

if ( ! -e CESM_DART_config ) then
   ${COPY} ${dartroot}/models/cam/shell_scripts/CESM_DART_config .
endif

if (   -e CESM_DART_config ) then
   sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
   ${MOVE} temp.$$ ${caseroot}/CESM_DART_config
   chmod 755       ${caseroot}/CESM_DART_config
else
   echo "ERROR: the script to configure for data assimilation is not available."
   echo "       CESM_DART_config MUST be present locally or in"
   echo "       ${dartroot}/models/cam/shell_scripts/"
   exit -2
endif

# ==============================================================================
# Configure the case.
# ==============================================================================

cd ${caseroot}

source ./Tools/ccsm_getenv || exit -2

# Make sure the case is configured with a data ocean.

if ( ${COMP_OCN} != docn ) then
   echo " "
   echo "ERROR: This setup script is not appropriate for active ocean compsets."
   echo "ERROR: Please use the models/CESM/shell_scripts examples for that case."
   echo " "
   exit -3
endif

# MAX_TASKS_PER_NODE comes from $case/Tools/mkbatch.$machine
@ ptile = $MAX_TASKS_PER_NODE / 2
@ nthreads = 1

# Save a copy for debug purposes
foreach FILE ( *xml )
   if ( ! -e        ${FILE}.original ) then
      ${COPY} $FILE ${FILE}.original
   endif
end

# NOTE: If you require bit-for-bit agreement between different runs,
#  in particular, between pmo (single instance) and assimilations (NINST > 1),
#  or if you need to change the number of nodes/member due to changing memory needs,
#  then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
#  generate bit-for-bit identical results, regardless of the number of tasks
#  given to it.  The time penalty appears to be ~ 0.5% in the forecast.
#  Alternatively, you can set cpl_tasks = same_number in both experiments

# Task layout:
# Set the nodes_per_instance below to match your case.  If you get 'out of memory' 
# errors OR failures without any messages, try increasing the nodes_per_instance.
# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
# By computing task counts like we do below, we guarantee each instance uses 
# a whole number of nodes which is the recommended configuration.

# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
#       Edwards says there's no speed up by running non-active components concurrently,
#       after ATM has run, so just run all components sequentially.
#       BUT, do arrange it so that each member(instance) spans complete nodes:
#       modulo(total pe count / number of instances, 15) == 0.

@ nodes_per_instance = 2

@ atm_tasks = $ptile * $num_instances * $nodes_per_instance
@ lnd_tasks = $ptile * $num_instances * $nodes_per_instance
@ ice_tasks = $ptile * $num_instances * $nodes_per_instance
@ ocn_tasks = $ptile * $num_instances
@ cpl_tasks = $ptile * $num_instances
@ glc_tasks = $ptile * $num_instances
@ rof_tasks = $ptile * $num_instances
@ wav_tasks = $ptile * $num_instances


switch ("`hostname`")
   case ys*:
      if ($atm_tasks > 4100) then
         echo 'WARNING!   Running DART on yellowstone with > 4100 tasks can cause filter'
         echo '           to hang in MPI_finalize.  Either reduce your task request,'
         echo '           or contact DART for a temporary work-around.'
         exit -9
      endif
   breaksw
endsw

echo ""
echo "ATM gets $atm_tasks"
echo "LND gets $lnd_tasks"
echo "ICE gets $ice_tasks"
echo "OCN gets $ocn_tasks"
echo "CPL gets $cpl_tasks"
echo "GLC gets $glc_tasks"
echo "ROF gets $rof_tasks"
echo "WAV gets $wav_tasks"
echo ""

./xmlchange NTHRDS_ATM=$nthreads,NTASKS_ATM=$atm_tasks,NINST_ATM=$num_instances
./xmlchange NTHRDS_LND=$nthreads,NTASKS_LND=$lnd_tasks,NINST_LND=$num_instances
./xmlchange NTHRDS_ICE=$nthreads,NTASKS_ICE=$ice_tasks,NINST_ICE=$num_instances
./xmlchange NTHRDS_OCN=$nthreads,NTASKS_OCN=$ocn_tasks,NINST_OCN=1
./xmlchange NTHRDS_CPL=$nthreads,NTASKS_CPL=$cpl_tasks
./xmlchange NTHRDS_GLC=$nthreads,NTASKS_GLC=$glc_tasks,NINST_GLC=1
./xmlchange NTHRDS_ROF=$nthreads,NTASKS_ROF=$rof_tasks,NINST_ROF=1
./xmlchange NTHRDS_WAV=$nthreads,NTASKS_WAV=$wav_tasks,NINST_WAV=1
./xmlchange ROOTPE_ATM=0
./xmlchange ROOTPE_LND=0
./xmlchange ROOTPE_ICE=0
./xmlchange ROOTPE_OCN=0
./xmlchange ROOTPE_CPL=0
./xmlchange ROOTPE_GLC=0
./xmlchange ROOTPE_ROF=0
./xmlchange ROOTPE_WAV=0

# http://www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/c1096.html#run_start_stop
# "A hybrid run indicates that CESM is initialized more like a startup, but uses
# initialization datasets from a previous case. This is somewhat analogous to a
# branch run with relaxed restart constraints. A hybrid run allows users to bring
# together combinations of initial/restart files from a previous case (specified
# by $RUN_REFCASE) at a given model output date (specified by $RUN_REFDATE).
# Unlike a branch run, the starting date of a hybrid run (specified by $RUN_STARTDATE)
# can be modified relative to the reference case. In a hybrid run, the model does not
# continue in a bit-for-bit fashion with respect to the reference case. The resulting
# climate, however, should be continuous provided that no model source code or
# namelists are changed in the hybrid run. In a hybrid initialization, the ocean
# model does not start until the second ocean coupling (normally the second day),
# and the coupler does a "cold start" without a restart file."
#
# TJH:
# DART's CAM implementation causes a bit more complexity. DART only uses CAM _initial_
# files, not RESTART files, so there are sourcemods to force a hybrid start for CAM to
# read initial files - even when CONTINUE_RUN = TRUE.

./xmlchange RUN_TYPE=hybrid
./xmlchange RUN_STARTDATE=${start_year}-${start_month}-${start_day}
./xmlchange START_TOD=$start_tod
./xmlchange RUN_REFCASE=$refcase
./xmlchange RUN_REFDATE=$refdate
./xmlchange RUN_REFTOD=$reftod
./xmlchange BRNCH_RETAIN_CASENAME=FALSE
./xmlchange GET_REFCASE=FALSE
./xmlchange EXEROOT=${exeroot}
./xmlchange RUNDIR=${rundir}

# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.

./xmlchange CALENDAR=GREGORIAN
./xmlchange CONTINUE_RUN=FALSE

./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
./xmlchange RESUBMIT=$resubmit

./xmlchange PIO_TYPENAME=pnetcdf

set TEST_MPI = `./xmlquery -valonly MPI_RUN_COMMAND | sed -e 's/MPI_RUN_COMMAND = //'`
if (${TEST_MPI} == 'UNSET') then
   ./xmlchange MPI_RUN_COMMAND=mpirun.lsf
endif

# The river transport model ON is useful only when using an active ocean or
# land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
# If you turn it ON, you will have to stage initial files etc.

./xmlchange ROF_GRID='null'
./xmlchange RTM_MODE='NULL'

# COUPLING discussion. F compsets are 'tight' coupling.
# Only change the ATM_NCPL ... everything is based on this one value,
# including CAM physics and dynamics timesteps.
# Default values for coupling are preserved in env_run.xml.original

./xmlchange NCPL_BASE_PERIOD=day
./xmlchange ATM_NCPL=48

# These are archiving options that may be used.
# You can turn the short/long term archivers on or off,
# but these settings should be made in either event.

./xmlchange DOUT_S_ROOT=${archdir}
./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
./xmlchange DOUT_L_MSROOT="csm/${case}"
./xmlchange DOUT_L_HTAR=FALSE

if ($short_term_archiver == 'off') then
   ./xmlchange DOUT_S=FALSE
else
   ./xmlchange DOUT_S=TRUE
endif
if ($long_term_archiver == 'off') then
   ./xmlchange DOUT_L_MS=FALSE
else
   ./xmlchange DOUT_L_MS=TRUE
endif

# DEBUG = TRUE implies turning on run and compile time debugging.
# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
# WARNING: CAM-SE fails if DEBUG=TRUE 
./xmlchange DEBUG=FALSE
./xmlchange INFO_DBUG=0

# ==============================================================================
# Update source files.
#    Ideally, using DART would not require any modifications to the model source.
#    Until then, this script accesses sourcemods from a hardwired location.
#    If you have additional sourcemods, they will need to be merged into any DART
#    mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
# ==============================================================================

if ( ! -d ~/${cesmtag}/SourceMods ) then
   echo "ERROR - No SourceMods for this case."
   echo "ERROR - No SourceMods for this case."
   echo "DART requires modifications to several src files."
   echo "Download the appropriate files for CESM 1_2_1 from:"
   echo "http://www.image.ucar.edu/pub/DART/CESM"
   echo "untar these into your HOME directory - they will create a"
   echo "~/cesm1_2_1  directory with the appropriate SourceMods structure."
   exit -4
endif

# Copy all of the 'generic' SourceMods
${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/   || exit 2

# Each CLM version has some SourceMods. Link to the right version.
# must parse from a variable of the form:
# CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn

set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`

@ iarg = 1
while ($iarg <= $#clm_opts)

   @ iargp1 = $iarg + 1
   set option = $clm_opts[$iarg]
   set  value = $clm_opts[$iargp1]

   switch ( ${option} )
      case "phys":
         if ( -e    SourceMods/src.clm/src/${value} ) then
            cd      SourceMods/src.clm
            ${LINK} src/${value}/*/*F90 .
            cd      ../..
         else
            echo "No SourceMods for CLM <${value}>."
            echo "Got the version from CLM_CONFIG_OPTS ...  <${CLM_CONFIG_OPTS}>"
         endif
      breaksw
   #  case "bgc":  no special action needed here at this time
   #  breaksw

      default:
      breaksw
   endsw

   @ iarg = $iarg + 2
end

# Need to know if we are using WACCM (aka WCCM) for several reasons.
# Mostly file managment issues.
set waccm = `echo $CCSM_COMPSET | grep WCCM`
if ($status == 0) then
   set waccm = true
else
   set waccm = false
   # WACCM benefits from a modified cd_core.F90, but none of the
   # other configurations do. 
   echo "Using the default cd_core.F90"
   ${REMOVE} -v SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/cd_core.F90
endif

# Each CAM dynamical core has its own SourceMods
if ( -e    SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
   cd      SourceMods/src.cam
   ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
   cd      ../..
else
   echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
endif

# The CESM multi-instance capability is relatively new and still has a few
# implementation bugs. These are known problems and will be fixed soon.
# this should be removed when the files are fixed:

echo "REPLACING BROKEN CESM FILES HERE - SHOULD BE REMOVED WHEN FIXED"
echo caseroot is ${caseroot}
if ( -d ~/${cesmtag} ) then

   # preserve the original version of the files
   if ( ! -e  ${caseroot}/Buildconf/rtm.buildnml.csh.original ) then
      ${COPY} ${caseroot}/Buildconf/rtm.buildnml.csh \
              ${caseroot}/Buildconf/rtm.buildnml.csh.original
   endif

   # patch/replace the broken files (first used by cesm_setup)
   ${COPY} ~/${cesmtag}/rtm.buildnml.csh  ${caseroot}/Buildconf/.

endif

# ==============================================================================
# Set up the case.
# This creates the EXEROOT and RUNDIR directories.
# ==============================================================================

echo 'Setting up the case ...'

./cesm_setup

if ( $status != 0 ) then
   echo "ERROR: Case could not be set up."
   exit -2
endif

# ==============================================================================
# Edit the run script to reflect queue and wallclock
# ==============================================================================

echo ''
echo 'Updating the run script to set wallclock and queue.'
echo ''

if ( ! -e  ${case}.run.original ) then
   ${COPY} ${case}.run ${case}.run.original
endif

source Tools/ccsm_getenv
set BATCH = `echo $BATCHSUBMIT | sed 's/ .*$//'`
switch ( $BATCH )
   case bsub*:
      # NCAR "bluefire", "yellowstone"
      set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
      set    QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
      sed -e "/BSUB/s#$TIMEWALL[3]#$timewall#" \
          -e "/BSUB/s#ptile=[0-9][0-9]*#ptile=$ptile#" \
          -e "/BSUB/s#$QUEUE[3]#$queue#" < ${case}.run >! temp.$$
          ${MOVE} temp.$$ ${case}.run
          chmod 755       ${case}.run
   breaksw

   default:
   breaksw
endsw

# This is the part that modifies the run script to allow CESM to advance
# correctly given the modifications of CAM for DART. It also copies several
# required DART files to the caseroot directory.

./CESM_DART_config || exit -3

# ==============================================================================
# Modify namelist templates for each instance.
#
# In a hybrid run with CONTINUE_RUN = FALSE (i.e. just starting up):
#
# CAM has been forced to read initial files - specified by namelist var:ncdata
# CICE reads from namelist variable 'ice_ic'
# CLM builds its own 'finidat' value from the REFCASE variables but in CESM1_2_1
#     it does not use the instance string. There is a patch for clm.buildnml.csh
#
# When CONTINUE_RUN = TRUE, CICE and CLM read from pointer files.
#
# All of these must later on be staged with these same filenames.
# OR - all these namelists can be changed to match whatever has been staged.
# MAKE SURE THE STAGING SECTION OF THIS SCRIPT MATCHES THESE VALUES.
# ==============================================================================

@ inst = 1
while ($inst <= $num_instances)

   # following the CESM strategy for 'inst_string'
   set inst_string = `printf _%04d $inst`

   # ===========================================================================
   set fname = "user_nl_cam${inst_string}"
   # ===========================================================================
   # ATM Namelist
   # DART/CAM requires a PHIS field in a history file at the same frequency as
   # the CAM assimilation. This script sets it to be written to the .h0. every
   # assimilation time. If you want to write it to a different .h?. file, you MUST
   # modify the assimilate.csh script in TWO places. You will need to set
   # 'empty_htapes = .false.' and change 'nhtfrq' and 'mfilt' to get a CAM 
   # default-looking .h0. file.
   #
   # inithist == 'ENDOFRUN' ensures that CAM writes the required initial file 
   # every time it stops.

   echo " inithist      = 'ENDOFRUN'"                     >> ${fname}
   echo " ncdata        = 'cam_initial${inst_string}.nc'" >> ${fname}
   echo " empty_htapes  = .true. "                        >> ${fname}
   echo " fincl1        = 'PHIS:I' "                      >> ${fname}
   echo " nhtfrq        = -$stop_n "                      >> ${fname}
   echo " mfilt         = 1 "                             >> ${fname}

   # CAM forcing files.
   # Some of the files specified here are because the default files only
   # contain data through 2005 and we are interested in timeframes after that.

   # CAM5 does prognostic aerosols by default (which is expensive).
   # Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
   # Prescribed aerosols on the timescales of interest to assimilation are acceptable.
   # WACCM5 should not have these set.

   if ($waccm == false ) then

    set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"

    echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
    echo " tracer_cnst_file      =  'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}

    echo "aerodep_flx_datapath   = '${chem_datapath}/aero'"                                      >> ${fname}
    echo "aerodep_flx_file       = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'"    >> ${fname}
    echo "aerodep_flx_type       = 'CYCLICAL'"                                                   >> ${fname}
    echo "aerodep_flx_cycle_yr   = 2000"                                                         >> ${fname}

    echo " ext_frc_specifier     = "                                                             >> ${fname}
    echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc',"      >> ${fname}
    echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
    echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
    echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
    echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
    echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
    echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'"    >> ${fname}
 
    echo " srf_emis_specifier    = "                                                             >> ${fname}
    echo " 'DMS    -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc',"      >> ${fname}
    echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc',"      >> ${fname}
    echo " 'SOAG   -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
    echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
    echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
    echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
    echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
    echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
    echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'"    >> ${fname}

   endif

   # ===========================================================================
   set fname = "user_nl_clm${inst_string}"
   # ===========================================================================
   # LAND Namelist
   # With a RUN_TYPE=hybrid the finidat is automatically specified
   # using the REFCASE/REFDATE/REFTOD information. i.e.
   # finidat = ${stagedir}/${refcase}.clm2${inst_string}.r.${reftimestamp}.nc
   #
   # TJH ... should make monthly average file or something ...
   # 1.3 MB is 100x bigger than CAM history file. See page 65 of
   # http://www.cesm.ucar.edu/models/cesm1.2/clm/models/lnd/clm/doc/UsersGuide/clm_ug.pdf
   #
   # Making a (compact) .h0. file is a good idea, since the clm restart files
   # do not have all the metadata required to reconstruct a gridded field.
   # 'TSA' is 2m surface air temperature.  This also prevents
   # having truly empty history files, resulting in ntapes = 0,
   # which prevents the hybrid-mode model from restarting.
   #
   # Every stop_n hours
   # echo "hist_mfilt  = 1"              >> ${fname}
   # echo "hist_nhtfrq = -$stop_n"       >> ${fname}
   # Every month
   # echo "hist_mfilt  = 1"              >> ${fname}
   # echo "hist_nhtfrq = 0"              >> ${fname}

   echo "hist_empty_htapes = .true."   >> ${fname}
   echo "hist_fincl1 = 'TSA'"          >> ${fname}
   echo "hist_nhtfrq = -$stop_n"       >> ${fname}
   echo "hist_mfilt  = 1"              >> ${fname}
   echo "hist_avgflag_pertape = 'I'"   >> ${fname}

   # ===========================================================================
   set fname = "user_nl_cice${inst_string}"
   # ===========================================================================
   # CICE Namelist

   echo "ice_ic = '${refcase}.cice${inst_string}.r.${reftimestamp}.nc'" >> ${fname}

   @ inst ++
end

./preview_namelists || exit -3

# ==============================================================================
# Stage the restarts now that the run directory exists
# ==============================================================================

set init_time = ${reftimestamp}

cat << EndOfText >! stage_cesm_files
#!/bin/csh -f
# This script can be used to help restart an experiment from any previous step.
# The appropriate files are copied to the RUN directory.
#
# Before running this script:
#  1) be sure CONTINUE_RUN is set correctly in the env_run.xml file in
#     your CASEROOT directory.
#     CONTINUE_RUN=FALSE => you are starting over at the initial time.
#     CONTINUE_RUN=TRUE  => you are starting from a previous step but not
#                           the very first one.
#  2) be sure 'restart_time' is set to the day and time that you want to
#     restart from if not the initial time.

set restart_time = $init_time

# get the settings for this case from the CESM environment
cd ${caseroot}
source ./Tools/ccsm_getenv || exit -2
cd ${RUNDIR}

echo 'Copying the required CESM files to the run directory to rerun'
echo 'a previous step.  CONTINUE_RUN from env_run.xml is' \${CONTINUE_RUN}
if ( \${CONTINUE_RUN} == TRUE ) then
  echo 'so files for some later step than the initial one will be restaged.'
  echo "Date to reset files to is: \${restart_time}"
else
  echo 'so files for the initial step of this experiment will be restaged.'
  echo "Date to reset files to is: ${init_time}"
endif
echo ''

if ( \${CONTINUE_RUN} == TRUE ) then

   #----------------------------------------------------------------------
   # This block copies over a set of restart files from any previous step of
   # the experiment that is NOT the initial step.
   # After running this script resubmit the job to rerun.
   #----------------------------------------------------------------------

   echo "Staging restart files for run date/time: " \${restart_time}

   #  The short term archiver is on, so the files we want should be in one
   #  of the short term archive 'rest' restart directories.  This assumes
   #  the long term archiver has NOT copied these files to the HPSS yet.

   if (  \${DOUT_S} == TRUE ) then

      # The restarts should be in the short term archive directory.  See
      # www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/x1565.html#running_ccsm_restarts
      # for more help and information.

      set RESTARTDIR = \${DOUT_S_ROOT}/rest/\${restart_time}

      if ( ! -d \${RESTARTDIR} ) then

         echo "restart file directory not found: "
         echo " \${RESTARTDIR}"
         echo "If the long-term archiver is on, you may have to restore this directory first."
         echo "You can also check for either a .sta or a .sta2 hidden subdirectory in"
         echo "\${DOUT_S_ROOT}"
         echo "which may contain the 'rest' directory you need,"
         echo "and then modify RESTARTDIR in this script."
         exit -1

      endif

      ${COPY} \${RESTARTDIR}/* . || exit -1

   else

      # The short term archiver is off, which leaves all the restart files
      # in the run directory.  The rpointer files must still be updated to
      # point to the files with the right day/time.

      @ inst=1
      while (\$inst <= $num_instances)

         set inst_string = \`printf _%04d \$inst\`

         echo "${case}.clm2\${inst_string}.r.\${restart_time}.nc" >! rpointer.lnd\${inst_string}
         echo "${case}.cice\${inst_string}.r.\${restart_time}.nc" >! rpointer.ice\${inst_string}
         echo "${case}.cam\${inst_string}.r.\${restart_time}.nc"  >! rpointer.atm\${inst_string}

         @ inst ++
      end

      # There are no instance numbers in these filenames.
      echo "${case}.cpl.r.\${restart_time}.nc"     >! rpointer.drv
      echo "${case}.docn.r.\${restart_time}.nc"    >! rpointer.ocn
      echo "${case}.docn.rs1.\${restart_time}.bin" >> rpointer.ocn

   endif

   # Relink the CAM initial files back to the hardwired names set in the namelist

   @ inst=1
   while (\$inst <= $num_instances)
      set inst_string = \`printf _%04d \$inst\`
      ${LINK} ${case}.cam\${inst_string}.i.\${restart_time}.nc cam_initial\${inst_string}.nc
      @ inst ++
   end

   echo "All files reset to rerun experiment step for time " \$restart_time

else     # CONTINUE_RUN == FALSE

   #----------------------------------------------------------------------
   # This block links the right files to rerun the initial (very first)
   # step of an experiment.  The names and locations are set during the
   # building of the case; to change them rebuild the case.
   # After running this script resubmit the job to rerun.
   #----------------------------------------------------------------------

   @ inst=1
   while (\$inst <= $num_instances)

      set inst_string = \`printf _%04d \$inst\`

      echo "Staging initial files for instance \$inst of $num_instances"

      ${LINK} ${stagedir}/${refcase}.clm2\${inst_string}.r.${init_time}.nc  .
      ${LINK} ${stagedir}/${refcase}.cice\${inst_string}.r.${init_time}.nc  .
      ${LINK} ${stagedir}/${refcase}.cam\${inst_string}.i.${init_time}.nc   cam_initial\${inst_string}.nc

      # If you are using the river runoff model, you must specify an initial file here
      # ${LINK} ${stagedir}/${refcase}.rtm\${inst_string}.r.${init_time}.nc .

      @ inst ++
   end

   echo "All files set to run the FIRST experiment step at time" $init_time

endif
exit 0

EndOfText
chmod 0755 stage_cesm_files

./stage_cesm_files

# ==============================================================================
# build
# ==============================================================================

echo ''
echo 'Building the case'
echo ''

./${case}.build

if ( $status != 0 ) then
   echo "ERROR: Case could not be built."
   exit -5
endif

# ==============================================================================
# What to do next
# ==============================================================================

echo ""
echo "Time to check the case."
echo ""
echo "1) cd ${rundir}"
echo "   and check the compatibility between the namelists/pointer"
echo "   files and the files that were staged."
echo ""
echo "2) cd ${caseroot}"
echo ""
echo "3) The case is initially configured to do NO ASSIMILATION by calling"
echo "   a script named 'no_assimilate.csh' in the ${case}.run script."
echo "   When you are ready to add data assimilation, change the"
echo "   ${case}.run script to call 'assimiliate.csh' instead of 'no_assimilate.csh'."
echo "   The configuration of CAM requires that 'no_assimilate.csh' be called even"
echo "   if doing a CESM-only run."
echo ""
echo "4) Verify the contents of env_run.xml and submit the CESM job:"
echo "   ./${case}.submit"
echo ""
echo "5) After the job has run, check to make sure it worked."
echo ""
echo "6) To extend the run in $stop_n '"$stop_option"' steps,"
echo "   change the env_run.xml variables:"
echo ""
echo "   ./xmlchange CONTINUE_RUN=TRUE"
echo "   ./xmlchange RESUBMIT=<number_of_cycles_to_run>"
echo ""
echo "   and"
echo "   ./${case}.submit"

cat ${caseroot}/DART_instructions.txt

exit 0

# <next few lines under version control, do not edit>
# $URL$
# $Revision$
# $Date$

