justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 178, Stage 1

Priority50
Processors1
Wall seconds36000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4194304000 (4000 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio usertests:pdvd-tdebde-test-fnal-w178s1p1np02vd_raw_run038131*.*2592000False

Environment variables

NameValue
FHICL_TAR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/d0cd8f293e7716f3710e79fd7fb9e1c56dc2d7ef
INPUT_TAR_DIR_LOCAL/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/d672b14eb7839876bbd543a30533c5f2278d8a12

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
5800005800

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
93000090010200
Files processed0010102020303040405050Jul-31 11:00Jul-31 12:00Jul-31 13:00Jul-31 14:00Jul-31 15:00Files processedBin start timesNumber per binNL_SURFsaraUK_QMULUK_ManchesterUK_RAL-Tier1NL_NIKHEFES_PIC
Replicas per RSE58490.025244.558269.975244.50000000000003Replicas per RSEDUNE_CERN_EOS (50%)FNAL_DCACHE (50%)

RSEs used

NameInputsOutputs
DUNE_CERN_EOS610
SURFSARA060
DUNE_UK_MANCHESTER_CEPH053
RAL_ECHO030
NIKHEF024
DUNE_UK_GLASGOW04
QMUL03

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
UK_Manchester01

Jobscript

#!/bin/bash
:<<'EOF'

To use this jobscript to process 10 files from the dc4-vd-coldbox-bottom
data and put the output in the usertests namespace (MetaCat) and 
scope (Rucio), and in the usertests:output-test-01 dataset in MetaCat and
Rucio, use this command to create the workflow:

justin simple-workflow \
--mql \
"files from dune:all where core.run_type='dc4-vd-coldbox-bottom' and dune.campaign='dc4' limit 10" \
--jobscript dc4-vd-coldbox-bottom.jobscript --max-distance 30 --rss-mb 4000 \
--scope usertests --output-pattern '*_reco_data_*.root:output-test-01' 

The following optional environment variables can be set when creating the
workflow/stage: FCL_FILE, NUM_EVENTS, DUNE_VERSION, DUNE_QUALIFIER 

EOF

# fcl file and DUNE software version/qualifier to be used
FCL_FILE=standard_reco_stage1_protodunevd_keepup.fcl
DUNE_VERSION=${DUNE_VERSION:-v10_08_01d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}

# number of events to process from the input file
if [ "$NUM_EVENTS" != "" ] ; then
 events_option="-n $NUM_EVENTS"
fi

# First get an unprocessed file from this stage
did_pfn_rse=`$JUSTIN_PATH/justin-get-file`

if [ "$did_pfn_rse" = "" ] ; then
  echo "Nothing to process - exit jobscript"
  exit 0
fi

# Keep a record of all input DIDs, for pdjson2meta file -> DID mapping
echo "$did_pfn_rse" | cut -f1 -d' ' >>all-input-dids.txt

# pfn is also needed when creating justin-processed-pfns.txt
pfn=`echo $did_pfn_rse | cut -f2 -d' '`
echo "Input PFN = $pfn"

# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh

export PRODUCTS=$INPUT_TAR_DIR_LOCAL:${PRODUCTS}

echo "PRODUCTS: ${PRODUCTS}"

setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

echo "DUNESW_DIR: ${DUNESW_DIR}"

# Properly setup custom code with INPUT_TAR_DIR_LOCAL

export DUNESW_DIR=${INPUT_TAR_DIR_LOCAL}/dunesw/${DUNE_VERSION}
export DUNESW_FQ_DIR=${DUNESW_DIR}/slf7.x86_64.e26.prof
export DUNESW_LIB=${DUNESW_FQ_DIR}/lib

export DUNECORE_DIR=${INPUT_TAR_DIR_LOCAL}/dunecore/${DUNE_VERSION}
export DUNECORE_LIB=${DUNECORE_DIR}/include
export DUNECORE_FQ_DIR=${DUNECORE_DIR}/slf7.x86_64.e26.prof
export DUNECORE_LIB=${DUNECORE_FQ_DIR}/lib

$export DUNERECO_DIR=${INPUT_TAR_DIR_LOCAL}/dunereco/${DUNE_VERSION}
$export DUNERECO_LIB=${DUNERECO_DIR}/include
$export DUNERECO_FQ_DIR=${DUNERECO_DIR}/slf7.x86_64.e26.prof
$export DUNERECO_LIB=${DUNERECO_FQ_DIR}/lib

$export WIRECELL_PATH=${DUNE_PARDATA_DIR}/WireCellData
$export WIRECELL_PATH=${WIRECELL_DIR}/Linux64bit+3.10-2.17-e26-prof/share/wirecell:${WIRECELL_PATH}
$export WIRECELL_PATH=${DUNERECO_DIR}/wire-cell-cfg:${WIRECELL_PATH}

export DUNEPROTOTYPES_DIR=${INPUT_TAR_DIR_LOCAL}/duneprototypes/${DUNE_VERSION}
export DUNEPROTOTYPES_INC=${DUNEPROTOTYPES_DIR}/include
export DUNEPROTOTYPES_FQ_DIR=${DUNEPROTOTYPES_DIR}/slf7.x86_64.e26.prof
export DUNEPROTOTYPES_LIB=${DUNEPROTOTYPES_FQ_DIR}/lib

export FHICL_FILE_PATH=${DUNESW_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${DUNEPROTOTYPES_DIR}/fcl:${FHICL_FILE_PATH}
$export FHICL_FILE_PATH=${DUNERECO_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${INPUT_TAR_DIR_LOCAL}:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${FHICL_TAR}:${FHICL_FILE_PATH}

export FW_SEARCH_PATH=${INPUT_TAR_DIR_LOCAL}/config_data:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${INPUT_TAR_DIR_LOCAL}:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${FHICL_TAR}:$FW_SEARCH_PATH

#echo "DUNESW_DIR: ${DUNESW_DIR}"
#echo "DUNERECO_DIR: ${DUNERECO_DIR}"
#echo "DUNEPROTOTYPES_DIR: ${DUNEPROTOTYPES_DIR}"
#echo "WIRECELL_PATH: ${WIRECELL_PATH}"
echo "FHICL_FILE_PATH: ${FHICL_FILE_PATH}"


export OMP_NUM_THREADS=${JUSTIN_PROCESSORS} 

# Construct outFile from input $pfn 
now=$(date -u +"%Y-%m-%dT_%H%M%SZ")
Ffname=`echo $pfn | awk -F/ '{print $NF}'`
fname=`echo $Ffname | awk -F. '{print $1}'`
outFile=${fname}_tde_bde_stage1_dev_${now}

campaign="justIN.r${JUSTIN_WORKFLOW_ID}s${JUSTIN_STAGE_ID}"

(
# Do the scary preload stuff in a subshell!
export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
echo "$LD_PRELOAD"
lar -c $FCL_FILE $events_option -T ${outFile}_hist.root -o ${outFile}.root "$pfn" > ${outFile}.log 2>&1
)

# Subshell exits with exit code of last command
larExit=$?
echo "lar exit code $larExit"
echo "$pfn" > justin-processed-pfns.txt

ls -lRS

# Create compressed tar file with all log files 
tar zcf `echo "$JUSTIN_JOBSUB_ID.logs.tgz" | sed 's/@/_/g'` *.log
#exit $jobscriptExit
exit $larExit

justIN time: 2025-08-04 18:26:03 UTC       justIN version: 01.04.00