~/DAI/cam2.0.1/models/atm/cam/src/control/restart.F90

  -> write_restart  (in restart.F90)
     -> timemgr_write_restart
        write (nrg, iostat=ioerr) eps, caseid, hyai, hybi, hyam,  &
                                   hybm, aeres
     ->  write_restart_dynamics (nrg)
         dynamics/eul/restart_dynamics.F90  chosen for Eulerian core somewhere
            contains subroutine write_restart_dynamics
         gets prognostic variables from module prognostics in .../src/dynamics/eul
             these have 3 time levels (last dimension), except qminus and
             dpsl...emga have 1 time level

   real(r8), allocatable :: ps(:,:,:)
   real(r8), allocatable :: u3(:,:,:,:)
   real(r8), allocatable :: v3(:,:,:,:)
   real(r8), allocatable :: t3(:,:,:,:)
   real(r8), allocatable :: q3(:,:,:,:,:)
         q3 is specific humidity (water vapor) and other constituents.
   real(r8), allocatable :: qminus(:,:,:,:)



         -> wrtoutr8  (in cam2.0.1/models/atm/cam/src/control/binary_io.F90)

  p = dynamics/eul/prognostic.F90
  s = dynamics/eul/sltb1.F90
  t = dynamics/eul/tfilt_massfix.F90
  b = physics/cam1/buffer.F90

  p NOT     call wrtout_r8 (nrg,vort(1,1,beglat,n3m1), plndlv)
  p NOT     call wrtout_r8 (nrg,vort(1,1,beglat,n3m2), plndlv)
!   NOT IN MASTER LIST; no "vorticity" is 
   real(r8), allocatable :: vort(:,:,:,:)   ! vorticity
      
  p NOT     call wrtout_r8 (nrg,div(1,1,beglat,n3m1) , plndlv)
  p NOT     call wrtout_r8 (nrg,div(1,1,beglat,n3m2) , plndlv)
!   NOT IN MASTER LIST; no "divergence" is 
   real(r8), allocatable :: div(:,:,:,:)    ! divergence
      
  p NOT     call wrtout_r8 (nrg,dpsl  ,plond )
   real(r8), allocatable :: dpsl(:,:)       ! longitudinal pressure gradient
  p NOT     call wrtout_r8 (nrg,dpsm  ,plond )
   real(r8), allocatable :: dpsm(:,:)       ! meridional pressure gradient
  p NOT     call wrtout_r8 (nrg,dps   ,plond )
   real(r8), allocatable :: dps(:,:)        ! pressure gradient
  p phis    call wrtout_r8 (nrg,phis  ,plond )
   real(r8), allocatable :: phis(:,:)       ! surface geopotential
  p omega   call wrtout_r8 (nrg,omga  ,plndlv)
   real(r8), allocatable :: omga(:,:,:)     ! vertical velocity
!   NOT IN MASTER LIST; none of these are
           !
           ! Write fields u3,v3,t3,q3,ps at time indices n3 and n3m1
           !
            begj = beglatex + numbnd
      
  p U       call wrtout_r8 (nrg,u3(1,1,begj,n3m1)  ,plndlv)
  p V       call wrtout_r8 (nrg,v3(1,1,begj,n3m1)  ,plndlv)
  p T       call wrtout_r8 (nrg,t3(1,1,begj,n3m1)  ,plndlv)
  p PS      call wrtout_r8 (nrg,ps(1,beglat,n3m1)  ,plond)
      
  p U       call wrtout_r8 (nrg,u3(1,1,begj,n3m2)  ,plndlv)
  p V       call wrtout_r8 (nrg,v3(1,1,begj,n3m2)  ,plndlv)
  p T       call wrtout_r8 (nrg,t3(1,1,begj,n3m2)  ,plndlv)
  p PS      call wrtout_r8 (nrg,ps(1,beglat,n3m2)  ,plond)
      
  p Q       call wrtout_r8 (nrg,q3(1,1,1,begj,n3m1),plndlv*(pcnst+pnats))
  p Q       call wrtout_r8 (nrg,q3(1,1,1,begj,n3m2),plndlv*(pcnst+pnats))
         pcnst is advected constituents, pnats is non-advected.(prognostics.F90)
           !
? kdr slt = semi lagrangian transport ?
           ! Write slt arrays (trajectory mid-point coordinates and
           ! slt forcast of moisture and constituents
           !
   NOT in master list
 s          call wrtout_r8 (nrg,lammp,plnlv)
 s          call wrtout_r8 (nrg,phimp,plnlv)
 s          call wrtout_r8 (nrg,sigmp,plnlv)
 t          call wrtout_r8 (nrg,qfcst,plndlv*pcnst)
defined in sltb1.F90
!  lammp   Longitude coordinates of the trajectory mid-points of the
!          parcels that correspond to the global grid points contained
!          in the latitude slice being forecasted.  On entry lammp
!          is an initial guess.
!  phimp   Latitude coordinates of the trajectory mid-points of the
!          parcels that correspond to the global grid points contained
!          in the latitude slice being forecasted.  On entry phimp
!          is an initial guess.
!  sigmp   Hybrid value at the trajectory midpoint for each gridpoint
!          in a vertical slice from the global grid.  On entry sigmp is
!          an initial guess.
and tfilt_massfix.F90
   qfcst(plond,plev,pcnst)! slt moisture forecast


xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx      
      
     ->  write_restart_physics (nrg, nrg2)
         in physics/cam1/restart_physics.F90
         -> write_field_from_chunk
         !
         ! Buffer module variables
         !
 b PBLH        call write_field_from_chunk(nrg,1,1,1,pblht)
 b TPERT       call write_field_from_chunk(nrg,1,1,1,tpert)
 b QRS         call write_field_from_chunk(nrg,1,pver,1,qrs)
 b QRL         call write_field_from_chunk(nrg,1,pver,1,qrl)
 b QPERT       call write_field_from_chunk(nrg,1,pcnst+pnats,1,qpert)
         pcnst is advected constituents, pnats is non-advected.(prognostics.F90)
         From cam1/buffer.F90
  real(r8), allocatable :: pblht(:,:)      ! PBL height
  real(r8), allocatable :: tpert(:,:)      ! temperature perturbation (PBL)
      master list; "PBL plume temp perturbation"
  real(r8), allocatable :: qrs(:,:,:)      ! shortwave radiative heating rate
  real(r8), allocatable :: qrl(:,:,:)      ! longwave  radiative heating rate
  real(r8), allocatable :: qpert(:,:,:)    ! moisture/constituent perturb.(PBL)
      master list; QPERT = "PBL plume moisture perturbation"; same?



loops over ptimelevels around the following
 b CLOUD?   call write_field_from_chunk(nrg,1,pver,1,cld(1,1,begchunk,n3tmp))
   = cloud fraction     CLDTOT=total cloud cover
   cldovrlap.F90:   :: cld(pcols,pver)     ! Fractional cloud cover

            call write_field_from_chunk(nrg,1,pver,1,qcwat(1,1,begchunk,n3tmp))
           =Grid average condensed water mixing ratio.
            call write_field_from_chunk (nrg,1,pver,1,tcwat(1,1,begchunk,n3tmp))
            call write_field_from_chunk (nrg,1,pver,1,lcwat(1,1,begchunk,n3tmp))
 b CWAT?    masterlist; Grid average condensed water mixing ratio.
            masterlist; yyCWAT is passive tracer quanitity
                        but yy are not Q,T, or L (q,t,l)
   restart_physics.F90:! cld, qcwat, and tcwat are physics things, 
                         but have dynamics time levels
   tphysbc.F90:   tcwato(pcols,pver)             !cloud water old temperature
   tphysbc.F90:   tcwatn(pcols,pver)            !cloud water new temperature
   tphysbc.F90:   lcwato(pcols,pver)             ! cloud liquid water old q
   tphysbc.F90:   lcwatn(pcols,pver)            ! cloud liq. water new q


  real(r8), allocatable :: qcwat(:,:,:,:)
  real(r8), allocatable :: tcwat(:,:,:,:)
  real(r8), allocatable :: cld(:,:,:,:)
  real(r8), allocatable :: lcwat(:,:,:,:)

         !
         ! Comsrf module variables
           surface fluxes for the subcomponents of cam
         !
         #if (! defined COUP_CSM)
               call write_field_from_chunk(nrg,1,1,1,fsnt)
fsnt(:,:)      ! Net column abs solar flux at model top
         #endif
               call write_field_from_chunk(nrg,1,1,1,fsns)
fsns(:,:)      ! surface absorbed solar flux
         #if (! defined COUP_CSM)
               call write_field_from_chunk(nrg,1,1,1,flnt)
               call write_field_from_chunk(nrg,1,1,1,flns)
flnt(:,:)      ! Net outgoing lw flux at model top
flns(:,:)      ! Srf longwave cooling (up-down) flux
         #endif

type (srfflx_state), allocatable :: srfflx_state2d(:)
     real(r8) :: asdir(pcols)            ! albedo: shortwave, direct
     real(r8) :: asdif(pcols)            ! albedo: shortwave, diffuse
     real(r8) :: aldir(pcols)            ! albedo: longwave, direct
     real(r8) :: aldif(pcols)            ! albedo: longwave, diffuse
master list; no 'albedo' anywhere!
         loops over i=begchunk,endchunk
                 tmpfield(:,i) = srfflx_state2d(i)%asdir(:)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
                 tmpfield(:,i) = srfflx_state2d(i)%asdif(:)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
                 tmpfield(:,i) = srfflx_state2d(i)%aldir(:)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
                 tmpfield(:,i) = srfflx_state2d(i)%aldif(:)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
         
         #if (! defined COUP_CSM)
               call write_field_from_chunk(nrg,1,1,1,asdirice)
               call write_field_from_chunk(nrg,1,1,1,asdifice)
               call write_field_from_chunk(nrg,1,1,1,aldirice)
               call write_field_from_chunk(nrg,1,1,1,aldifice)
    TSICE      call write_field_from_chunk(nrg,1,1,1,tsice)
    master list; Sea-ice surface Temperature
         #endif
 same as asdir..., but for ice?

               do i=begchunk,endchunk
                 tmpfield(:,i) = srfflx_state2d(i)%lwup(:)
               end do
comsrf.F90     real(r8) :: lwup(pcols)             ! longwave up radiative flux
               call write_field_from_chunk(nrg,1,1,1,tmpfield)

               call write_field_from_chunk(nrg,1,1,1,landfrac)
    LANDFRAC  Fraction of grid-square covered by land. 
    comsrf.F90      landfrac(:,:)  ! fraction of sfc area covered by land

               call write_field_from_chunk(nrg,1,1,1,landm)
    LANDM Land-Ocean transition mask:(Output as instantaneous field by default)
    comsrf.F90    landm(:,:)     ! land/ocean/sea ice flag

               call write_field_from_chunk(nrg,1,1,1,sgh)
    SGH Standard deviation of orography
    comsrf.F90    sgh(:,:)       ! land/ocean/sea ice flag
       yes, that's what's in code; looks like accidental copy of landm descr.
        
                 tmpfield(:,i) = srfflx_state2d(i)%ts(:)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
    TS Surface temperature
    comsrf.F90    ts(pcols)               ! sfc temp 

                 tmpfield3d(:,:,i) = surface_state2d(i)%tssub(:,:)
               call write_field_from_chunk(nrg,1,plevmx,1,tmpfield3d)
    comsrf.F90    tssub(pcols,plevmx) ! cam surface/subsurface temperatures

               call write_field_from_chunk(nrg,1,1,1,sicthk)
    comsrf.F90    sicthk(:,:)    ! cam sea-ice thickness (m)

               call write_field_from_chunk(nrg,1,1,1,snowhland)
    comsrf.F90    snowhland(:,:) !snow depth (liquid water) ovr lnd

         #if (! defined COUP_CSM)
               call write_field_from_chunk(nrg,1,1,1,snowhice)
    comsrf.F90    snowhice(:,:)  ! snow depth (liquid water) ovr ice
         #else
               snowhice = 0.
         #endif
loops around indented lines
                ncol = get_ncols_p(i)
                 tmpfield(:ncol,i) = surface_state2d(i)%flwds(:ncol)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
    radclwmx.F90    flwds(pcols)         ! Down longwave flux at surface
                ncol = get_ncols_p(i)
                 tmpfield(:ncol,i) = surface_state2d(i)%sols(:ncol)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
    radcswmx.F90    sols(pcols)      ! Direct solar rad on surface (< 0.7)
                    SOLS               Downward visible direct solar to surface

                ncol = get_ncols_p(i)
                 tmpfield(:ncol,i) = surface_state2d(i)%soll(:ncol)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
    radcswmx.F90    soll(pcols)      ! Direct solar rad on surface (>= 0.7)
                    SOLL               Downward near IR direct solar to surface.

                ncol = get_ncols_p(i)
                 tmpfield(:ncol,i) = surface_state2d(i)%solsd(:ncol)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
    radcswmx.F90    solsd(pcols)     ! Diffuse solar rad on surface (< 0.7)
                    SOLSD              Downward visible diffuse solar to surface

                ncol = get_ncols_p(i)
                 tmpfield(:ncol,i) = surface_state2d(i)%solld(:ncol)
               call write_field_from_chunk(nrg,1,1,1,tmpfield)
    radcswmx.F90    solld(pcols)     ! Diffuse solar rad on surface (>= 0.7)
                    SOLLD              Downward near IR diffuse solar to surface

               call write_field_from_chunk(nrg,1,1,1,trefmxav)
    comsrf.F90  trefmxav(:,:)  ! diagnostic: tref max over the day
    TREFMXAV    Daily maximum reference height
                temperature. The output frequency and
                type of the file determines if it's an
                average or minimum/maximum over
                another time frequency. If the output
                file is a monthly average this will be the
                monthly average of the daily minimum temperature.

               call write_field_from_chunk(nrg,1,1,1,trefmnav)
     comsrf.F90  trefmnav    ! diagnostic: tref min over the day
     TREFMNAV  Daily minimum reference height
               temperature. The output frequency and
               type of the file determines if it's an
               average or minimum/maximum over
               another time frequency. If the output
               file is a monthly average this will be the
               monthly average of the daily minimum
               temperature.

               call write_field_from_chunk(nrg,1,1,1,icefrac)
     comsrf.F90  icefrac(:,:)   ! fraction of sfc area covered by seaice
     ICEFRAC  Fraction of grid-square covered by sea-ice
         
         #if ( defined COUP_CSM )
               call write_restart_ccsm ()
         #endif

---------
      if (aeres) then
         call write_field_from_chunk(nrg2, 1, pverp*pverp,1, abstot_3d(1,1,1,begchunk))
abstot_3d(:,:,:,:) ! Non-adjacent layer absorptivites
         call write_field_from_chunk(nrg2, 1, pver*4,     1, absnxt_3d(1,1,1,begchunk))
absnxt_3d(:,:,:,:) ! Nearest layer absorptivities
         call write_field_from_chunk(nrg2, 1, pverp,      1, emstot_3d(1,1,begchunk))
emstot_3d(:,:,:) ! Total emissivity endif

         
         
--------------------------------------------------------

     ->  write_restart_history (nrg, luhrest)
         cam2.0.1/models/atm/cam/src/control/history.F90

We can avoid having to deal with secondary and history buffer restart files:

"Master and primary restart files are always written during a model run. 
A secondary restart file is written if absorptivity/emissivity is not be 
calculated on the first timestep after restart, and therefore must be saved 
on a restart dataset.  For a stand-alone run this occurs if the primary 
history file write frequency, NHTFRQ(1), is not a multiple of the
absorptivity/emissivity calculation frequency, IRADAE (note that for a 
CCSM flux coupled run, only the flux coupler
determines when the restart files are written). It is advisable to 
avoid this situation if possible, since this dataset is relatively
large even for the standard T42 model. 

A history buffer restart file is written in order to retain the accumulated 
values in the history buffers if restart files are to be
written on a timestep when one or more history file time samples are not 
written. A separate restart dataset is written for each
history file. Each history buffer restart file contains the portion of the 
history buffers pertaining to that history file. "
