Workflow 163, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	90.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	fardet-vd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio fardet-vd:fardet-vd-reco2-ritm2440929_163-fnal-w163s1p1	*reco2.root	51840000	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/d8dadc139bff0f0047b9fbf848af00bb7bad89eb

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10000	0	0	0	0	9997	0	3

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
13884	0	0	0	0	12348	0	150	1312	20	24	30

RSEs used

Name	Inputs	Outputs
DUNE_US_FNAL_DISK_STAGE	10696	9826
DUNE_UK_GLASGOW	0	27
DUNE_UK_MANCHESTER_CEPH	0	20
RAL_ECHO	0	17
SURFSARA	0	16
RAL-PP	0	13
DUNE_UK_LANCASTER_CEPH	0	10
DUNE_US_BNL_SDCC	0	8
PRAGUE	0	7
QMUL	0	7
DUNE_CERN_EOS	0	5
NIKHEF	0	3
DUNE_CA_SFU	0	1

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
UK_Manchester	118	62
CERN	49	15
UK_RAL-PPD	42	37
US_UChicago	36	13
NL_SURFsara	35	19
UK_RAL-Tier1	23	37
UK_Lancaster	18	13
ES_PIC	16	4
UK_QMUL	15	10
US_FNAL-FermiGrid	14	13
UK_Edinburgh	12	1
CZ_FZU	8	12
US_Wisconsin	6	3
US_Colorado	6	2
US_UCSD	4	0
IT_CNAF	3	4
US_PuertoRico	3	0
NL_NIKHEF	2	1
BR_CBPF	2	3
UK_Durham	2	6
UK_Oxford	2	8

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_04d01}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"
 
#metacat file show -mj ${DID} > old_md.json
#mcExit=$?
#if [ $mcExit -eq 0 ] ; then
#  echo "old metadata:"
#  cat old_md.json
#else
#  echo "Could not retrieve old metadata"
#  exit 1 
#fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============reco2=========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco2="${infile%.root}" 
reco2_name=${reco2}_${now}_reco2 

echo "check  output filename "  ${reco2_name}.root

#lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n 1 #-n -1 
lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n -1 
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
OUTFILE=`ls *reco2.root`
echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco2 --appversion  v09_91_04d01  --requestid ritm2440929  --input_json ${INPUT_DIR}/le_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-vd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.json 
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
  echo "$FILE" > justin-processed-pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls
exit 0