justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 12729, Stage 1

Workflow12729
Campaign1060
Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4194304000 (4000 MiB)
Max distance for inputs100.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1https://fndcadoor.fnal.gov:2880/dune/scratch/users/jierans/hd_cosmic/ntuple/fnal/12729/1triggerAna_*.ntuple.root
2https://fndcadoor.fnal.gov:2880/dune/scratch/users/jierans/hd_cosmic/artroot/fnal/12729/1tpg*.root

Environment variables

NameValue
DUNE_VERSIONv10_14_01d00
INPUT_TAR_DIR_LOCAL/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/95643e8c255e927b03b7ab9ce3995ad879dfbb29
NUM_EVENTS300

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
1000039970000

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
2078619973432016281000

RSEs used

NameInputsOutputs
MONTECARLO16290

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
CERN2490
UK_RAL-PPD710
US_UChicago650
UK_Manchester470
UK_RAL-Tier1400
NL_NIKHEF330
ES_PIC290
UK_QMUL210
US_FNAL-FermiGrid190
IT_CNAF160
UK_Oxford130
UK_Lancaster120
UK_Imperial50
BR_CBPF20

Jobscript

#!/bin/bash
# fcl file and DUNE software version/qualifier to be used
# FCL_FILE=${FCL_FILE:-$INPUT_TAR_DIR_LOCAL/my_code/fcls/my_reco.fcl}
DUNE_VERSION=${DUNE_VERSION:-v10_14_01d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
echo "sw version: ${DUNE_VERSION} ${DUNE_QUALIFIER}"
echo "input tarball location:" ${INPUT_TAR_DIR_LOCAL}

# number of events to process from the input file
if [ "$NUM_EVENTS" != "" ]; then
  events_option="-n $NUM_EVENTS"
fi

# First get an unprocessed file from this stage
did_pfn_rse=$($JUSTIN_PATH/justin-get-file)
did=`echo $did_pfn_rse | cut -f1 -d' '`
pfn=`echo $did_pfn_rse | cut -f2 -d' '`
rse=`echo $did_pfn_rse | cut -f3 -d' '`
echo "Input DID = $did"
echo "Input PFN = $pfn"
echo "Input RSE = $rse"

if [ "$did_pfn_rse" = "" ]; then
  echo "Nothing to process - exit jobscript"
  exit 0
fi

if [[ $did != monte-carlo-* ]]; then
  echo "Error: did must start with 'monte-carlo-'. Got: $did" >&2
  exit 1
fi

# Convert DID to a run:subrun:event triplet for larsoft
tmp=${did#monte-carlo-}      # strip leading "monte-carlo-"
workflow_id=${tmp%-*}        # everything before last '-'
mcnum=${tmp##*-}            # everything after last '-'
runspec="${workflow_id}:${mcnum}:1"
echo "Run:SubRun:Event = $runspec"


# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
export PRODUCTS="$(echo ${INPUT_TAR_DIR_LOCAL}/dunesw-${DUNE_VERSION}/localProducts_larsoft*_prof/):$PRODUCTS"
# Then we can set up our local products
setup duneana "$DUNE_VERSION" -q "$DUNE_QUALIFIER"
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

# Construct outFile from input $pfn
now=$(date -u +"%Y-%m-%dT_%H%M%SZ")
Ffname=$(echo $pfn | awk -F/ '{print $NF}')
fname=$(echo $Ffname | awk -F. '{print $1}')
campaign="justIN.w${JUSTIN_WORKFLOW_ID}s${JUSTIN_STAGE_ID}"

outFile=tpg_${JUSTIN_WORKFLOW_ID}_${mcnum}.root
anaFile=triggerAna_${JUSTIN_WORKFLOW_ID}_${mcnum}.ntuple.root

# Here is where the LArSoft command is call it
(
  # Do the scary preload stuff in a subshell!
  export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
  echo "$LD_PRELOAD"
  # Gen
  # lar -c prodbackground_radiological_decay0_dunevd10kt_1x8x6_patched_wall_gammas.fcl $events_option -e $runspec -o tmp_gen.root
  lar -c prodMUSUN_dune10kt_1x2x6.fcl $events_option -e $runspec -o gen1.root
  lar -c filterMUSUN_dune10kt_1x2x6.fcl $events_option -s gen1.root -o tmp_gen.root
  # G4
  lar -c standard_g4_dune10kt_1x2x6.fcl $events_option -s tmp_gen.root -o tmp_g4.root
  # Detsim
  lar -c detsim_dune10kt_1x2x6_notpcsigproc.fcl $events_option -s tmp_g4.root -o tmp_detsim.root
  # TPG - tpc
  lar -c tpg_dune10kt_1x2x6.fcl $events_option -s tmp_detsim.root -o tmp_tpg_tpc.root
  # TPG - PDS + Ana
  lar -c triggersim_triggerana_dunefd_1x2x6_pds.fcl $events_option -s tmp_tpg_tpc.root -o $outFile -T $anaFile 
)

# Subshell exits with exit code of last command
larExit=$?
echo "lar exit code $larExit"

if [ $larExit -eq 0 ]; then
  # Success !
  echo "$pfn" >justin-processed-pfns.txt
  jobscriptExit=0
else
  # Oh :(
  jobscriptExit=1
fi

# Create compressed tar file with all log files
tar zcf $(echo "$JUSTIN_JOBSUB_ID.logs.tgz" | sed 's/@/_/g') *.log
exit $jobscriptExit
justIN time: 2026-02-04 13:30:56 UTC       justIN version: 01.06.00