Workflow 111, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	90.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	fardet-vd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio fardet-vd:fardet-vd-reco2-ritm2440929_111-fnal-w111s1p1	*reco2.root	51840000	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/d8dadc139bff0f0047b9fbf848af00bb7bad89eb

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10000	0	0	0	0	9790	0	210

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
14348	0	0	0	0	11667	0	222	1264	1169	8	18

RSEs used

Name	Inputs	Outputs
DUNE_US_FNAL_DISK_STAGE	11491	9690
DUNE_UK_GLASGOW	0	22
DUNE_UK_MANCHESTER_CEPH	0	14
QMUL	0	14
SURFSARA	0	9
DUNE_US_BNL_SDCC	0	8
DUNE_FR_CCIN2P3_DISK	0	4
DUNE_UK_LANCASTER_CEPH	0	4
PRAGUE	0	4
RAL_ECHO	0	4
NIKHEF	0	1
RAL-PP	0	1

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
UK_Manchester	138	27
UK_RAL-PPD	70	1
CERN	44	1
UK_QMUL	39	17
US_FNAL-FermiGrid	32	6
CZ_FZU	29	5
UK_Lancaster	15	0
NL_SURFsara	14	7
NL_NIKHEF	13	0
ES_PIC	10	4
FR_CCIN2P3	10	1
US_UChicago	9	0
UK_Edinburgh	8	0
IT_CNAF	7	0
BR_CBPF	6	1
UK_RAL-Tier1	4	3
US_UCSD	2	0
UK_Durham	2	0
UK_Oxford	2	0
UK_Sheffield	1	0

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_04d01}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"
 
#metacat file show -mj ${DID} > old_md.json
#mcExit=$?
#if [ $mcExit -eq 0 ] ; then
#  echo "old metadata:"
#  cat old_md.json
#else
#  echo "Could not retrieve old metadata"
#  exit 1 
#fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============reco2=========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco2="${infile%.root}" 
reco2_name=${reco2}_${now}_reco2 

echo "check  output filename "  ${reco2_name}.root

#lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n 1 #-n -1 
lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n -1 
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
OUTFILE=`ls *reco2.root`
echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco2 --appversion  v09_91_04d01  --requestid ritm2440929  --input_json ${INPUT_DIR}/le_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-vd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.json 
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
  echo "$FILE" > justin-processed-pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls
exit 0