justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 2664, Stage 1

Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes8387559424 (7999 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio usertests:calcuttj_PDHD_MC_neg5GeV_stage1-fnal-w2664s1p1*ionscint_pdext.root2592000False
2https://fndcadoor.fnal.gov:2880/dune/scratch/users/calcuttj/PDHD_MC_Prod_Test_neg5GeV_ggi/fnal/02664/1prod_beam*log

Environment variables

NameValue
DUNESW_TAR/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/6862534214c2d14e6efd2b5670af25f8bf3eae46
DUNESW_VERSIONv10_10_02d00
NEVENTS10
NSKIP10
pipyaml1
UTIL_TAR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/6e55c9e87c4af087e8cc3f712f8154ba8102f235
YAMLFILEpdhd_beam_gen_5GeV_gen_g4_ion.yaml

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
169000016900

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
2890000289000000
Files processed00101020203030404050506060707080809090100100110110120120Sep-16 15:00Sep-16 16:00Sep-16 17:00Sep-16 18:00Files processedBin start timesNumber per binUK_BristolNL_SURFsaraUK_DurhamUK_OxfordUK_ManchesterUK_RAL-PPDUS_UChicagoUS_WisconsinCZ_FZUCERNIT_CNAFUS_UCSD
Replicas per RSE169470.11753865245214178.8439830232947169360.94638766857037322.2536859166461342279.4271722131298222.8389886895497520287.92406993540214183.0357722458553817300.37645390077716162.0555410831461817315.9726091240379145.9382578675751714332.9905436563669134.587449483911149346.915913825194128.522245409896047357.06793538481946125.574257534164266365.50826608126727123.97452450237355372.73515565114377123.18772427271483378.01708027929317122.94454935518795Replicas per RSEDUNE_US_FNAL_DISK_S…DUNE_US_FNAL_DISK_STAGE (35%)FNAL_DCACHE (35%)DUNE_CERN_EOS (8%)NIKHEF (4%)DUNE_UK_MANCHESTER_…DUNE_UK_MANCHESTER_CEPH (3%)RAL-PP (3%)DUNE_UK_GLASGOW (2%)DUNE_FR_CCIN2P3_DIS…DUNE_FR_CCIN2P3_DISK (1%)SURFSARA (1%)DUNE_UK_LANCASTER_C…DUNE_UK_LANCASTER_CEPH (1%)RAL_ECHO (1%)DUNE_US_BNL_SDCC (0…DUNE_US_BNL_SDCC (0%)

RSEs used

NameInputsOutputs
DUNE_CERN_EOS4243
DUNE_US_FNAL_DISK_STAGE310
NIKHEF190
DUNE_UK_MANCHESTER_CEPH1736
RAL-PP170
DUNE_UK_GLASGOW140
DUNE_FR_CCIN2P3_DISK90
SURFSARA723
DUNE_UK_LANCASTER_CEPH60
RAL_ECHO50
DUNE_US_BNL_SDCC233
None0507
PRAGUE027
DUNE_IT_INFN_CNAF07

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash
#

#These must be defined
if [ -z $UTIL_TAR ]; then
  echo "Fatal must provide UTIL_TAR env var"
  exit 1
fi
export PYTHONPATH=$UTIL_TAR:$PYTHONPATH

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "Justin specific env vars"
env | grep JUSTIN
now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}
a=1
 if [ -n "$DUNESW_TAR" ]; then
   stat ${DUNESW_TAR}
   if [ $? -ne 0 ]; then
     echo "Failed to stat $DUNESW_TAR. Exiting safely"
     exit 0
   fi
 
   export PRODUCTS=$DUNESW_TAR:$PRODUCTS 
   echo "Set dunesw to $DUNESW_TAR"
 fi

 if [ -n "$WIRECELL_TAR" ]; then
   stat ${WIRECELL_TAR}
   if [ $? -ne 0 ]; then
     echo "Failed to stat $WIRECELL_TAR. Exiting safely"
     exit 0
   fi
 
   export PRODUCTS=$WIRECELL_TAR:$PRODUCTS 
   echo "Set wirecell to $WIRECELL_TAR"
 fi

 #Setup recent lar software suite
 DUNESW_VERSION=${DUNESW_VERSION:-v10_02_02d00}
 setup dunesw \
    "${DUNESW_VERSION}" \
    -q "${DUNE_QUALIFIER:-e26:prof}"
 
 setup_exit=$?
 if [ $? -ne 0 ]; then
   echo "Failed to setup dunesw $DUNESW_VERSION $DUNE_QUALIFIER"
   exit $setup_exit
 fi
 
 echo "DUNESW loc:"
 ups active | grep dunesw

 echo "WIRECELL loc:"
 ups active | grep wirecell
 

 if [ -n "$extra_fcl_path" ]; then
   export FHICL_FILE_PATH=${extra_fcl_path}:${FHICL_FILE_PATH}
   echo "Added ${extra_fcl_path} to fhicl file path"
   echo "${FHICL_FILE_PATH}"
 fi
 
 echo "printing env"
 env
 echo "DUNE specific env vars"
 env | grep DUNE

# User can specify to pip install pyyaml
if [ -n "$pipyaml" ]; then
  echo "Installing yaml"
  pip install --user pyyaml
  echo "Done"
# elif [ -n "${YAML_TAR}" ]; then
#   echo "Using venv at ${YAML_TAR}"
#   source ${YAML_TAR}/yaml_venv/bin/activate
fi


 echo "Will use justin-get-file"
 DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
 if [ "${DID_PFN_RSE}" == "" ] ; then
   echo "Could not get file"
   exit 0
 fi
 pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
 did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 
 input_filename=`echo $did | cut -f2 -d':'`
 echo "input file: $input_filename"

 echo "jobsub_id: ${JUSTIN_JOBSUB_ID:-1}"
 
 #Get run/subrun from parent if using input -- else from justin job ID/process"
 if [ -z "$SetRun" ]; then
   echo "Getting run subrun from $did"
   run_subrun=`python -m beam_job_utils get_run_subrun -i $did`
   get_run_ret=$?
   if [ $get_run_ret -ne 0 ]; then
     echo "error in get_run_subrun"
   fi
 
   echo $run_subrun
   run=`echo $run_subrun | cut -f1 -d' '`
   subrun=`echo $run_subrun | cut -f2 -d' '`
 else
   run=`echo $jobid | cut -f1 -d'_'`  ##TODO -- check order here
   subrun=`echo $jobid | cut -f2 -d'_'`
 fi
 nevents=${NEVENTS:--1} #Set from env override or -1 
 NSKIP=${NSKIP:-0} 
 echo "nevents: $nevents"
 
 overrides="core.data_tier=full-reconstructed \
  core.application.version=${DUNESW_VERSION} \
  core.application.name=reco \
  core.application.family=dunesw \
  core.application=dunesw.reco \
 "

#User specifies if they'd like to use an input file or not 
if [ -n "$MCJob" ]; then
  inputflag=""
else
  inputflag="-i $pfn --parent $did"
fi

echo "input flag: $inputflag"

if [[ $pfn == *.hdf5 ]]; then
  echo "Setting LD_PRELOAD"  
  export LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so
fi

# TODO -- metadata file configurable
python -m beam_job_utils run_job \
      --json ${UTIL_TAR}/${JSONFILE:-pdsp_beam_base_meta.json} \
      --overrides ${overrides} \
      --run $run \
      --subrun $subrun \
      --yaml ${UTIL_TAR}/${YAMLFILE:-pdsp_beam_gen.yaml} \
      --nevents $nevents \
      $(if [ ${NSKIP} -ne 0 ]; then echo "--event ${NSKIP}"; fi) \
      $inputflag

exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error running. Exiting with ${exitcode}"
  exit $exitcode
fi

echo "lsing"
ls -lhS
echo "disk usage"
du -sh .

echo "$pfn" > justin-processed-pfns.txt
justIN time: 2025-09-18 16:03:36 UTC       justIN version: 01.05.00
</