Workflow 10823, Stage 1

Workflow	10823
Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	6290407424 (5999 MiB)
Max distance for inputs	30.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio usertests:calcuttj_pdhd_data_test-fnal-w10823s1p1	pdhd_prod_data*root	2592000	False

Environment variables

Name	Value
extra_fcl_path	/exp/dune/app/users/calcuttj/dunesw_standalone/fcl/protodunehd/reco/

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10	0	0	0	0	10	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
17	0	0	0	0	16	0	0	1	0	0	0

RSEs used

Name	Inputs	Outputs
DUNE_CERN_EOS	11	0
SURFSARA	0	10

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
UK_RAL-Tier1	0	1

Jobscript

#!/bin/sh
#
# Example jobscript that runs lar for all the files
# referred to by the MQL expression given on the justin command line.
#
# Submit with something like this:
#
# ./justin simple-workflow \
#  --max-distance 30 \
#  --mql "rucio-dataset protodune-sp:np04_raw_run_number_5769" \
#  --jobscript lar.jobscript
#
# Then monitor with dashboard or ./justin show-jobs --workflow-id ID
# where ID is the value printed by the first command
#


# the xroot lib for streaming non-root files is in testproducts, 
# so add it to the start of the path
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

# From jobsub
export CLUSTER=${CLUSTER:-1}
export PROCESS=${PROCESS:-1}

if [ -n "${extra_fcl_path}" ]; then
  echo "Adding ${extra_fcl_path} to fhicl file path"
  export FHICL_FILE_PATH=${extra_fcl_path}:${FHICL_FILE_PATH}
  echo $FHICL_FILE_PATH
fi

setup dunesw ${DUNESW_VERSION:-v10_10_04d00} -q e26:prof
setup_exit=$?
if [ "$setup_exit" != 0 ]; then
  echo "Error setting up dunesw ${DUNESW_VERSION}"
  exit
fi

# Get an unprocessed file from this stage
did_pfn_rse=`$JUSTIN_PATH/justin-get-file`
if [ "${did_pfn_rse}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

did=`echo $did_pfn_rse | cut -f1 -d' '`
pfn=`echo $did_pfn_rse | cut -f2 -d' '`
rse=`echo $did_pfn_rse | cut -f3 -d' '`
name_only=$(echo $did | cut -f2 -d':')
echo "name" $name_only


echo "Found input file URL $pfn at $rse"

if [ -n "${test_path}" ]; then
  pfn=${test_path}
  echo "Set pfn to test path ${pfn}"
fi

#Get the stage and momentum 
export stage=${stage:-"Gen"}
export momentum=${momentum:-1}


echo "Using gen fcl: ${gen_fcl}"

now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}
output_preamble="pdhd_prod_data_"

nevents=${nevents:--1}
nskip=${nskip:-0}
LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so lar \
  -c  standard_reco_stage1_protodunehd_keepup.fcl \
  -o temp.root \
  -n ${nevents} \
  --nskip ${nskip} \
  ${pfn}

larReturnCode=$?
echo "stage1 lar returns $larReturnCode"
if [ "$larReturnCode" != 0 ] ; then
  exit $larReturnCode
fi

#IonScintPDExt
lar \
    -c  standard_reco_stage2_calibration_protodunehd_keepup.fcl \
    -o ${output_preamble}_${jobid}_${stageid}_${now}.root \
    temp.root

larReturnCode=$?
echo "stage2 lar returns $larReturnCode"
if [ "$larReturnCode" != 0 ] ; then
  exit $larReturnCode
fi
#---------------------------------------------


####EDIT METADATA FOR THE FOLLOWING FILE
#${name_only}_${jobid}_${stageid}_${now}_detsim_reco1.root

# Record that we processed the input file ok (did we???)
echo "$pfn" > justin-processed-pfns.txt

# For debugging
for i in *.json
do
  echo "==== Start $i ===="
  cat $i
  echo "==== End $i ===="
done

ls -ltR

exit 0