justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 8806, Stage 1

Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes6291456000 (6000 MiB)
Max distance for inputs100.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopehd-protodune
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio hd-protodune:bsmMC_2025a_reco2-fnal-w8806s1p1*_protodunehd_*_reco_stage2.root7776000FalseDUNE_US_FNAL_DISK_STAGE

Environment variables

NameValue
DUNESW_VERSIONv10_05_00d00
JSONFILEpdhd_wnp04_base_meta.json
MCJob1
NEVENTS10
pipyaml1
UTIL_TAR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/8903d6be21c5250084ac1854692983819112b027
YAMLFILEpdhd_wnp04_spsneutrino_mc.yaml

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
10000000100000

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
143600001434001100
Files processed0040408080120120160160200200240240280280320320Oct-09 19:00Oct-09 20:00Oct-09 21:00Oct-09 22:00Oct-09 23:00Oct-10 00:00Oct-10 01:00Oct-10 02:00Oct-10 03:00Oct-10 04:00Oct-10 05:00Oct-10 06:00Oct-10 07:00Oct-10 08:00Oct-10 09:00Oct-10 10:00Oct-10 11:00Files processedBin start timesNumber per binNL_SURFsaraUK_QMULUK_SheffieldUK_LancasterNL_NIKHEFUS_WisconsinUS_FNAL-FermiG…US_FNAL-FermiGridCERNUK_OxfordUK_ManchesterUK_RAL-Tier1UK_RAL-PPDCZ_FZUUK_DurhamES_PIC

RSEs used

NameInputsOutputs
MONTECARLO10020
QMUL0408
DUNE_US_FNAL_DISK_STAGE0299
NIKHEF0169
DUNE_UK_MANCHESTER_CEPH028
SURFSARA027
DUNE_US_BNL_SDCC024
PRAGUE018
DUNE_CERN_EOS018
DUNE_UK_LANCASTER_CEPH08
DUNE_ES_PIC01

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
FR_CCIN2P310

Jobscript

#!/bin/bash
#

#These must be defined
if [ -z $UTIL_TAR ]; then
  echo "Fatal must provide UTIL_TAR env var"
  exit 1
fi
export PYTHONPATH=$UTIL_TAR:$PYTHONPATH
# FHICL files in tarball can be found by appending tarball to fhicl path
export FHICL_FILE_PATH=$UTIL_TAR:$FHICL_FILE_PATH
# May want to append util_tar to FW_SEARCH_PATH in the future (for data processing)
#export FW_SEARCH_PATH=$UTIL_TAR:$FW_SEARCH_PATH

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "Justin specific env vars"
env | grep JUSTIN
now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}
a=1
if [ -n "$DUNESW_TAR" ]; then
  stat ${DUNESW_TAR}
  if [ $? -ne 0 ]; then
    echo "Failed to stat $DUNESW_TAR. Exiting safely"
    exit 0
  fi
 
  export PRODUCTS=$DUNESW_TAR:$PRODUCTS 
  echo "Set dunesw to $DUNESW_TAR"
fi

if [ -n "$WIRECELL_TAR" ]; then
  stat ${WIRECELL_TAR}
  if [ $? -ne 0 ]; then
    echo "Failed to stat $WIRECELL_TAR. Exiting safely"
    exit 0
  fi
 
  export PRODUCTS=$WIRECELL_TAR:$PRODUCTS 
  echo "Set wirecell to $WIRECELL_TAR"
fi

#Setup recent lar software suite
DUNESW_VERSION=${DUNESW_VERSION:-v10_05_00d00}
setup dunesw \
  "${DUNESW_VERSION}" \
  -q "${DUNE_QUALIFIER:-e26:prof}"
 
setup_exit=$?
if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNESW_VERSION $DUNE_QUALIFIER"
  exit $setup_exit
fi
 
echo "DUNESW loc:"
ups active | grep dunesw

echo "WIRECELL loc:"
ups active | grep wirecell
 
 
echo "printing env"
env
echo "DUNE specific env vars"
env | grep DUNE

# User can specify to pip install pyyaml
if [ -n "$pipyaml" ]; then
  echo "Installing yaml"
  pip install --user pyyaml
  echo "Done"
# elif [ -n "${YAML_TAR}" ]; then
#   echo "Using venv at ${YAML_TAR}"
#   source ${YAML_TAR}/yaml_venv/bin/activate
fi

echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 
input_filename=`echo $did | cut -f2 -d':'`
echo "input file: $input_filename"

echo "jobsub_id: ${JUSTIN_JOBSUB_ID:-1}"
 
#Get run/subrun from parent if using input -- else from justin job ID/process"
if [ -z "$MCJob" ]; then
  echo "Getting run subrun from $did"
  run_subrun=`python -m beam_job_utils get_run_subrun -i $did`
  get_run_ret=$?
  if [ $get_run_ret -ne 0 ]; then
    echo "error in get_run_subrun"
  fi
 
  echo $run_subrun
  run=`echo $run_subrun | cut -f1 -d' '`
  subrun=`echo $run_subrun | cut -f2 -d' '`
else
  run=`echo $jobid | cut -f1 -d'_'`  ##TODO -- check order here
  subrun=`echo $jobid | cut -f2 -d'_'`
fi
nevents=${NEVENTS:--1} #Set from env override or -1 
NSKIP=${NSKIP:-0} 
echo "nevents: $nevents"
 
overrides="core.data_tier=full-reconstructed \
  core.application.version=${DUNESW_VERSION} \
  core.application.name=reco \
  core.application.family=dunesw \
  core.application=dunesw.reco \
"

#User specifies if they'd like to use an input file or not 
if [ -n "$MCJob" ]; then
  inputflag=""
else
  inputflag="-i $pfn --parent $did"
fi

echo "input flag: $inputflag"

# TODO -- metadata file configurable
python -m beam_job_utils run_job \
      --json ${UTIL_TAR}/${JSONFILE:-pdhd_base_meta.json} \
      --overrides ${overrides} \
      --run $run \
      --subrun $subrun \
      --yaml ${UTIL_TAR}/${YAMLFILE:-pdhd_spsneutrino_mc.yaml} \
      --nevents $nevents \
      $inputflag

exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error running. Exiting with ${exitcode}"
  exit $exitcode
fi

echo "$pfn" > justin-processed-pfns.txt
justIN time: 2025-11-04 01:27:21 UTC       justIN version: 01.05.01