Workflow 523, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	90.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	fardet-vd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio fardet-vd:fardet-vd-reco2-ritm2440929_523-fnal-w523s1p1	*reco2.root	51840000	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/67fa70d88651a634e48981ba9183a72ae15d3923

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10000	0	0	0	0	9993	0	7

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
11051	0	0	0	0	10753	0	1	254	42	1	0

RSEs used

Name	Inputs	Outputs
DUNE_US_FNAL_DISK_STAGE	9699	9970
PRAGUE	514	1
SURFSARA	17	3
DUNE_US_BNL_SDCC	14	6
RAL-PP	8	0
DUNE_UK_GLASGOW	7	1
DUNE_CERN_EOS	6	0
QMUL	5	1
DUNE_UK_MANCHESTER_CEPH	4	0
DUNE_UK_LANCASTER_CEPH	1	0
NIKHEF	1	0
DUNE_CA_SFU	1	1
RAL_ECHO	0	8

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
US_FNAL-T1	125	0
ES_PIC	76	0
UK_Edinburgh	20	0
US_UChicago	9	1
UK_Manchester	8	0
US_Colorado	1	0
BR_CBPF	1	0
CZ_FZU	0	1

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_04d01}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"
 
#metacat file show -mj ${DID} > old_md.json
#mcExit=$?
#if [ $mcExit -eq 0 ] ; then
#  echo "old metadata:"
#  cat old_md.json
#else
#  echo "Could not retrieve old metadata"
#  exit 1 
#fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============reco2=========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco2="${infile%.root}" 
reco2_name=${reco2}_${now}_reco2 

echo "check  output filename "  ${reco2_name}.root

#lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n 1 #-n -1 
lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n -1 
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
OUTFILE=`ls *reco2.root`
echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco2 --appversion  v09_91_04d01  --requestid ritm2440929  --input_json ${INPUT_DIR}/le_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-vd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.json 
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
  echo "$FILE" > justin-processed-pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls
exit 0