Workflow 11399, Stage 1

Workflow	11399
Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/jierans/vd_radiologicals_with_pds/ntuple/fnal/11399/1	triggerAna_*.ntuple.root
2	https://fndcadoor.fnal.gov:2880/dune/scratch/users/jierans/vd_radiologicals_with_pds/artroot/fnal/11399/1	tpg*.root

Environment variables

Name	Value
DUNE_VERSION	v10_14_01d00
INPUT_TAR_DIR_LOCAL	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/fdeb0d994c9cd59b7d1e308c08f5a3a9fad4bc0e
NUM_EVENTS	20

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
1000	0	0	0	0	999	0	1

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
1570	0	0	0	0	1510	0	9	24	27	0	0

RSEs used

Name	Inputs	Outputs
MONTECARLO	1059	0
None	0	1998

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
CA_SFU	23	0
NL_NIKHEF	3	0
NL_SURFsara	2	0
US_UChicago	1	1
UK_QMUL	1	0
CA_Victoria	1	0

Jobscript

#!/bin/bash
# fcl file and DUNE software version/qualifier to be used
# FCL_FILE=${FCL_FILE:-$INPUT_TAR_DIR_LOCAL/my_code/fcls/my_reco.fcl}
DUNE_VERSION=${DUNE_VERSION:-v10_14_01d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
echo "sw version: ${DUNE_VERSION} ${DUNE_QUALIFIER}"
echo "input tarball location:" ${INPUT_TAR_DIR_LOCAL}

# number of events to process from the input file
if [ "$NUM_EVENTS" != "" ]; then
  events_option="-n $NUM_EVENTS"
fi

# First get an unprocessed file from this stage
did_pfn_rse=$($JUSTIN_PATH/justin-get-file)
did=`echo $did_pfn_rse | cut -f1 -d' '`
pfn=`echo $did_pfn_rse | cut -f2 -d' '`
rse=`echo $did_pfn_rse | cut -f3 -d' '`
echo "Input DID = $did"
echo "Input PFN = $pfn"
echo "Input RSE = $rse"

if [ "$did_pfn_rse" = "" ]; then
  echo "Nothing to process - exit jobscript"
  exit 0
fi

if [[ $did != monte-carlo-* ]]; then
  echo "Error: did must start with 'monte-carlo-'. Got: $did" >&2
  exit 1
fi

# Convert DID to a run:subrun:event triplet for larsoft
tmp=${did#monte-carlo-}      # strip leading "monte-carlo-"
workflow_id=${tmp%-*}        # everything before last '-'
mcnum=${tmp##*-}            # everything after last '-'
runspec="${workflow_id}:${mcnum}:1"
echo "Run:SubRun:Event = $runspec"


# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
export PRODUCTS="$(echo ${INPUT_TAR_DIR_LOCAL}/dunesw-${DUNE_VERSION}/localProducts_larsoft*_prof/):$PRODUCTS"
# Then we can set up our local products
setup duneana "$DUNE_VERSION" -q "$DUNE_QUALIFIER"
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

# Construct outFile from input $pfn
now=$(date -u +"%Y-%m-%dT_%H%M%SZ")
Ffname=$(echo $pfn | awk -F/ '{print $NF}')
fname=$(echo $Ffname | awk -F. '{print $1}')
campaign="justIN.w${JUSTIN_WORKFLOW_ID}s${JUSTIN_STAGE_ID}"

outFile=tpg_${JUSTIN_WORKFLOW_ID}_${mcnum}.root
anaFile=triggerAna_${JUSTIN_WORKFLOW_ID}_${mcnum}.ntuple.root

# Here is where the LArSoft command is call it
(
  # Do the scary preload stuff in a subshell!
  export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
  echo "$LD_PRELOAD"
  # Gen
  lar -c prodbackground_radiological_decay0_dunevd10kt_1x8x6_patched_wall_gammas.fcl $events_option -e $runspec -o tmp_gen.root
  # G4
  lar -c standard_g4_dunevd10kt_1x8x6_3view_30deg.fcl $events_option -s tmp_gen.root -o tmp_g4.root
  # Detsim
  lar -c detsim_dunevd10kt_1x8x6_3view_30deg_notpcsigproc.fcl $events_option -s tmp_g4.root -o tmp_detsim.root
  # TPG - tpc
  lar -c triggersim_tpg_vd_1x8x6.fcl $events_option -s tmp_detsim.root -o tmp_tpg_tpc.root
  # TPG - PDS + Ana
  lar -c triggersim_triggerana_dunefdvd_1x8x6_pds.fcl $events_option -s tmp_tpg_tpc.root -o $outFile -T $anaFile 
)

# Subshell exits with exit code of last command
larExit=$?
echo "lar exit code $larExit"

if [ $larExit -eq 0 ]; then
  # Success !
  echo "$pfn" >justin-processed-pfns.txt
  jobscriptExit=0
else
  # Oh :(
  jobscriptExit=1
fi

# Create compressed tar file with all log files
tar zcf $(echo "$JUSTIN_JOBSUB_ID.logs.tgz" | sed 's/@/_/g') *.log
exit $jobscriptExit