Add a globus transfer job. NOAA-EMC#1357

DavidHuber-NOAA · Mar 23, 2023 · 9d04c17 · 9d04c17
1 parent 2e92b7c
commit 9d04c17
Show file tree

Hide file tree

Showing 12 changed files with 349 additions and 19 deletions.
diff --git a/jobs/JGLOBAL_GLOBUS b/jobs/JGLOBAL_GLOBUS
@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base globus"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+export CDATE=${CDATE:-${PDY}${cyc}}
+export CDUMP=${CDUMP:-${RUN:-"gfs"}}
+
+
+###############################################################
+# Run archive script
+###############################################################
+
+${GLOBALGLOBUSXFERSH:-${SCRgfs}/exglobal_globus_xfer.sh}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0
diff --git a/jobs/rocoto/globus.sh b/jobs/rocoto/globus.sh
@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="arch"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGLOBAL_GLOBUS
+status=$?
+
+
+exit "${status}"
diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua
@@ -39,4 +39,7 @@ load(pathJoin("ufswm", "1.0.0"))
 load(pathJoin("met", "9.1"))
 load(pathJoin("metplus", "3.1"))
 
+-- Set the path to globus
+append_path("PATH", "/home/dhuber/local/bin")
+
 whatis("Description: GFS run environment")
diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
@@ -47,6 +47,7 @@ export HOMEDIR="@HOMEDIR@"
 export STMP="@STMP@"
 export PTMP="@PTMP@"
 export NOSCRUB="@NOSCRUB@"
+export NIAGARA_USERNAME="${LOGNAME}"
 
 # Base directories for various builds
 export BASE_GIT="@BASE_GIT@"
@@ -87,8 +88,8 @@ export VERBOSE="YES"
 export KEEPDATA="NO"
 export CHGRP_RSTPROD="@CHGRP_RSTPROD@"
 export CHGRP_CMD="@CHGRP_CMD@"
-export NCDUMP="$NETCDF/bin/ncdump"
-export NCLEN="$HOMEgfs/ush/getncdimlen"
+export NCDUMP="${NETCDF}/bin/ncdump"
+export NCLEN="${HOMEgfs}/ush/getncdimlen"
 
 # Machine environment, jobs, and other utility scripts
 export BASE_ENV="${HOMEgfs}/env"
@@ -111,6 +112,7 @@ export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
 export RUNDIR="${DATAROOT}"  # TODO: Should be removed; use DATAROOT instead
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
+export ATARDIRloc="@ATARDIRloc@"
 
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
@@ -378,8 +380,10 @@ export binary_diag=".false."
 export DO_METP="YES"         # Run METPLUS jobs - set METPLUS settings in config.metp
 
 # Archiving options
-export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive
+export HPSSARCH="@HPSSARCH@"          # save data to HPSS archive
 export LOCALARCH="@LOCALARCH@"        # save data to local archive
+export DO_GLOBUS="NO"                 # transfer local archives to Niagara for HPSS archive; only valid if LOCALARCH=YES
+export REMOTE_USERNAME=${REMOTE_USERNAME:-${LOGNAME}}
 if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
    echo "Both HPSS and local archiving selected.  Please choose one or the other."
    exit 2

diff --git a/parm/config/config.globus b/parm/config/config.globus
@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+########## config.globus ##########
+# Globus transfer specific
+
+echo "BEGIN: config.globus"
+
+# Get task specific resources
+. ${EXPDIR}/config.resources globus
+
+export GLOBUS_XFR=${GLOBUS_XFR:-"globus transfer --notify failed --preserve-mtime --sync-level mtime"}
+export GLOBUS_RM=${GLOBUS_RM:-"globus rm --notify failed -f"}
+export GLOBUS_WAIT=${GLOBUS_WAIT-"globus task wait"}
+export GLOBUS_LS=${GLOBUS_LS:-"globus ls"}
+
+export TARGET_DIR=${TARGET_DIR:-"/collab1/data/${NIAGARA_USERNAME:-$LOGNAME}/${PSLOT}/${CDATE}"}
+
+if [[ ${machine} == "HERA" ]]; then
+   export LOCAL_GLOBUS_ADDR="82109590-c090-11ea-bef9-0e716405a293"
+elif [[ ${machine} == "ORION" ]]; then
+   export LOCAL_GLOBUS_ADDR="84bad22e-cb80-11ea-9a44-0255d23c44ef"
+elif [[ ${machine} == "JET" ]]; then
+   export LOCAL_GLOBUS_ADDR="34ea8506-1882-11eb-81b5-0e2f230cc907"
+fi
+
+# Default is Niagara
+export REMOTE_GLOBUS_ADDR=${REMOTE_GLOBUS_ADDR:-"21467dd0-afd6-11ea-8f12-0a21f750d19b"}
+
+echo "END: config.arch"
diff --git a/parm/config/config.resources b/parm/config/config.resources
@@ -13,7 +13,7 @@ if [ $# -ne 1 ]; then
     echo "atmensanalprep atmensanalrun atmensanalpost"
     echo "aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres"
-    echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
+    echo "eobs ediag eomg eupd ecen esfc efcs epos earc globus"
     echo "init_chem mom6ic ocnpost"
     echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
     echo "wavegempak waveawipsbulls waveawipsgridded"
@@ -651,7 +651,7 @@ elif [ ${step} = "mom6ic" ]; then
     export npe_node_mom6ic=24
     export is_exclusive=True
 
-elif [[ ${step} = "arch" || ${step} = "earc" || ${step} = "getic" ]]; then
+elif [[ ${step} = "arch" || ${step} = "earc" || ${step} = "getic" || ${step} = "globus" ]]; then
 
     eval "export wtime_${step}='06:00:00'"
     eval "export npe_${step}=1"

diff --git a/scripts/exgdas_enkf_earc.sh b/scripts/exgdas_enkf_earc.sh
@@ -9,6 +9,12 @@ export n=$((10#${ENSGRP}))
 export CDUMP_ENKF=$(echo "${EUPD_CYC:-"gdas"}" | tr a-z A-Z)
 export ARCH_LIST="${ROTDIR}/${RUN}.${PDY}/${cyc}/earc${ENSGRP}"
 
+if [[ ${LOCALARCH} = "YES" ]]; then
+   tar_dir=${tar_dir:-${ATARDIRloc}}
+elif [[ ${HPSSARCH} = "YES" ]]; then
+   tar_dir=${tar_dir:-${ATARDIR}}
+fi
+
 # ICS are restarts and always lag INC by $assim_freq hours.
 EARCINC_CYC=${ARCH_CYC}
 EARCICS_CYC=$((ARCH_CYC-assim_freq))
@@ -40,7 +46,7 @@ if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]];
    TARCMD="htar"
    if [[ ${LOCALARCH} = "YES" ]]; then
        TARCMD="tar"
-       [ ! -d "${ATARDIR}"/"${CDATE}" ] && mkdir -p "${ATARDIR}"/"${CDATE}"
+       [ ! -d "${tar_dir}"/"${CDATE}" ] && mkdir -p "${tar_dir}"/"${CDATE}"
    fi
 
 #--determine when to save ICs for warm start
@@ -65,15 +71,15 @@ if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]];
 
    if [ "${CDATE}" -gt "${SDATE}" ]; then # Don't run for first half cycle
 
-     ${TARCMD} -P -cvf "${ATARDIR}/${CDATE}/${RUN}_grp${ENSGRP}.tar" $(cat "${ARCH_LIST}/${RUN}_grp${n}.txt")
+     ${TARCMD} -P -cvf "${tar_dir}/${CDATE}/${RUN}_grp${ENSGRP}.tar" $(cat "${ARCH_LIST}/${RUN}_grp${n}.txt")
      status=$?
      if [ "${status}" -ne 0 ] && [ "${CDATE}" -ge "${firstday}" ]; then
          echo "$(echo "${TARCMD}" | tr 'a-z' 'A-Z') ${CDATE} ${RUN}_grp${ENSGRP}.tar failed"
          exit "${status}"
      fi
 
      if [ "${SAVEWARMICA}" = "YES" ] && [ "${cyc}" -eq "${EARCINC_CYC}" ]; then
-       ${TARCMD} -P -cvf "${ATARDIR}/${CDATE}/${RUN}_restarta_grp${ENSGRP}.tar" $(cat "${ARCH_LIST}/${RUN}_restarta_grp${n}.txt")
+       ${TARCMD} -P -cvf "${tar_dir}/${CDATE}/${RUN}_restarta_grp${ENSGRP}.tar" $(cat "${ARCH_LIST}/${RUN}_restarta_grp${n}.txt")
        status=$?
        if [ "${status}" -ne 0 ]; then
            echo "$(echo "${TARCMD}" | tr 'a-z' 'A-Z') ${CDATE} ${RUN}_restarta_grp${ENSGRP}.tar failed"
@@ -82,7 +88,7 @@ if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]];
      fi
 
      if [ "${SAVEWARMICB}" = "YES" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ]; then
-       ${TARCMD} -P -cvf "${ATARDIR}/${CDATE}/${RUN}_restartb_grp${ENSGRP}.tar" $(cat "${ARCH_LIST}/${RUN}_restartb_grp{n}.txt")
+       ${TARCMD} -P -cvf "${tar_dir}/${CDATE}/${RUN}_restartb_grp${ENSGRP}.tar" $(cat "${ARCH_LIST}/${RUN}_restartb_grp{n}.txt")
        status=$?
        if [ "${status}" -ne 0 ]; then
            echo "$(echo "${TARCMD}" | tr 'a-z' 'A-Z') ${CDATE} ${RUN}_restartb_grp${ENSGRP}.tar failed"
@@ -105,11 +111,11 @@ if [ "${ENSGRP}" -eq 0 ]; then
         TARCMD="htar"
         if [[ ${LOCALARCH} = "YES" ]]; then
             TARCMD="tar"
-            [ ! -d "${ATARDIR}"/"${CDATE}" ] && mkdir -p "${ATARDIR}"/"${CDATE}"
+            [ ! -d "${tar_dir}"/"${CDATE}" ] && mkdir -p "${tar_dir}"/"${CDATE}"
         fi
 
         set +e
-        ${TARCMD} -P -cvf "${ATARDIR}/${CDATE}/${RUN}.tar" $(cat "${ARCH_LIST}/${RUN}.txt")
+        ${TARCMD} -P -cvf "${tar_dir}/${CDATE}/${RUN}.tar" $(cat "${ARCH_LIST}/${RUN}.txt")
         status=$?
         if [ "${status}" -ne 0 ] && [ "${CDATE}" -ge "${firstday}" ]; then
             echo "$(echo "${TARCMD}" | tr 'a-z' 'A-Z') ${CDATE} ${RUN}.tar failed"

diff --git a/scripts/exglobal_archive.sh b/scripts/exglobal_archive.sh
@@ -13,6 +13,12 @@ if [ "${ARCHICS_CYC}" -lt 0 ]; then
     ARCHICS_CYC=$((ARCHICS_CYC+24))
 fi
 
+if [[ ${LOCALARCH} = "YES" ]]; then
+   tar_dir=${tar_dir:-${ATARDIRloc}}
+elif [[ ${HPSSARCH} = "YES" ]]; then
+   tar_dir=${tar_dir:-${ATARDIR}}
+fi
+
 # CURRENT CYCLE
 APREFIX="${CDUMP}.t${cyc}z."
 
@@ -111,8 +117,8 @@ if [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]]; then
 TARCMD="htar"
 if [[ ${LOCALARCH} = "YES" ]]; then
    TARCMD="tar"
-   [ ! -d "${ATARDIR}"/"${CDATE}" ] && mkdir -p "${ATARDIR}"/"${CDATE}"
-   [ ! -d "${ATARDIR}"/"${CDATE_MOS}" ] && [ -d "${ROTDIR}"/gfsmos."${PDY_MOS}" ] && [ "${cyc}" -eq 18 ] && mkdir -p "${ATARDIR}"/"${CDATE_MOS}"
+   [ ! -d "${tar_dir}"/"${CDATE}" ] && mkdir -p "${tar_dir}"/"${CDATE}"
+   [ ! -d "${tar_dir}"/"${CDATE_MOS}" ] && [ -d "${ROTDIR}"/gfsmos."${PDY_MOS}" ] && [ "${cyc}" -eq 18 ] && mkdir -p "${tar_dir}"/"${CDATE_MOS}"
 fi
 
 #--determine when to save ICs for warm start and forecast-only runs
@@ -181,7 +187,7 @@ if [ "${CDUMP}" = "gfs" ]; then
     # Aerosols
     if [ "${DO_AERO}" = "YES" ]; then
         for targrp in chem; do
-            ${TARCMD} -P -cvf "${ATARDIR}"/"${CDATE}"/"${targrp}".tar $(cat "${ARCH_LIST}"/"${targrp}".txt)
+            ${TARCMD} -P -cvf "${tar_dir}"/"${CDATE}"/"${targrp}".tar $(cat "${ARCH_LIST}"/"${targrp}".txt)
             status=$?
             if [ "${status}" -ne 0 ] && [ "${CDATE}" -ge "${firstday}" ]; then
                 echo "HTAR ${CDATE} ${targrp}.tar failed"
@@ -203,7 +209,7 @@ if [ "${CDUMP}" = "gfs" ]; then
     #--save mdl gfsmos output from all cycles in the 18Z archive directory
     if [ -d gfsmos."${PDY_MOS}" ] && [ "${cyc}" -eq 18 ]; then
         set +e
-        ${TARCMD} -P -cvf "${ATARDIR}"/"${CDATE_MOS}"/gfsmos.tar ./gfsmos."${PDY_MOS}"
+        ${TARCMD} -P -cvf "${tar_dir}"/"${CDATE_MOS}"/gfsmos.tar ./gfsmos."${PDY_MOS}"
         status=$?
         if [ "${status}" -ne 0 ] && [ "${CDATE}" -ge "${firstday}" ]; then
             echo "$(echo "${TARCMD}" | tr 'a-z' 'A-Z') ${CDATE} gfsmos.tar failed"
@@ -237,7 +243,7 @@ fi
 shopt -s extglob
 for targrp in ${targrp_list}; do
     set +e
-    ${TARCMD} -P -cvf "${ATARDIR}"/"${CDATE}"/"${targrp}".tar $(cat "${ARCH_LIST}"/"${targrp}".txt)
+    ${TARCMD} -P -cvf "${tar_dir}"/"${CDATE}"/"${targrp}".tar $(cat "${ARCH_LIST}"/"${targrp}".txt)
     status=$?
     if [ "${status}" -ne 0 ] && [ "${CDATE}" -ge "${firstday}" ]; then
         echo "$(echo "${TARCMD}" | tr 'a-z' 'A-Z') ${CDATE} ${targrp}.tar failed"