justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 10225, Stage 1

Workflow10225
Campaign69
Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4194304000 (4000 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopefardet-hd
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio fardet-hd:fardet-hd-reco-ritm2612698_10225-fnal-w10225s1p1*reco.root51840000False

Environment variables

NameValue
INPUT_DIR/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/63063150ce9e5c360a96921b1b37b12fd17d1619

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
99250000992500

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
537000004638067665000
Files processed00400400800800120012001600160020002000240024002800280032003200Nov-15 16:00Nov-15 17:00Nov-15 18:00Nov-15 19:00Nov-15 20:00Nov-15 21:00Nov-15 22:00Nov-15 23:00Nov-16 00:00Files processedBin start timesNumber per binUK_DurhamUK_QMULUK_OxfordUK_SheffieldCA_SFUUK_ManchesterUK_RAL-Tier1UK_LancasterNL_NIKHEFUK_BrunelUK_RAL-PPDES_PICCZ_FZUCERNUK_BristolUK_EdinburghIT_CNAF
Replicas per RSE9925489.77244608236384237.049449275756479925272.24103396460623266.71499187447193894365.11659383721826135.486308905744921379.98333797395077134.47500126163652Replicas per RSEDUNE_FR_CCIN2P3_DISK (47%)FNAL_DCACHE (47%)PRAGUE (4%)DUNE_US_FNAL_DISK_STAGE (0%)

RSEs used

NameInputsOutputs
DUNE_FR_CCIN2P3_DISK121849842
PRAGUE2630
DUNE_US_FNAL_DISK_STAGE10
NIKHEF016
RAL_ECHO013
QMUL012
DUNE_UK_GLASGOW010
DUNE_UK_MANCHESTER_CEPH04
RAL-PP02
DUNE_CERN_EOS01
DUNE_ES_PIC01
DUNE_IT_INFN_CNAF01
SURFSARA01

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
UK_Lancaster7616
IT_CNAF2048
ES_PIC888
UK_Sheffield4184
UK_Manchester4102
NL_NIKHEF0816
UK_RAL-Tier10628
UK_QMUL0222
UK_RAL-PPD092
CZ_FZU072
CERN051
UK_Brunel028
UK_Oxford024
UK_Edinburgh016
UK_Bristol012
UK_Durham012

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite

#setup dunesw v10_11_00d01 -q e26:prof
setup dunesw v10_12_01d01 -q e26:prof
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 
 

for nf in {1..4}
  do
  DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 
 ##Check that any file was returned
 if [ "${DID_PFN_RSE}" == "" ] ; then
   echo "Could not get file"
#   exit 0
   continue
 fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 echo ${DID} >> did.list
 echo ${FILE} >> file.list 
done
 
while read inFile; do

temp=`echo "$inFile" | awk -F'/' '{print $NF}'`
DID="fardet-hd:"$temp
FILE=$inFile 
 
 
metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============TPG========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco="${infile%.root}" 
reco_name=${reco}_${now}_reco

echo "check  output filename "  ${reco_name}.root

lar -c  tpg_dune10kt_1x2x6.fcl $FILE -o ${reco_name}.root -n -1
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
#OUTFILE=`ls *reco.root`
OUTFILE=${reco_name}.root

#mv triggerAnaTree_hist.root triggerAnaTree_${reco_name}_hist.root


echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco --appversion  v10_12_01d01  --requestid ritm2612698  --input_json ${INPUT_DIR}/trg_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-hd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.temp.json 


sed '/DUNE/ s/.*/\L&/'  ${OUTFILE}.temp.json >  ${OUTFILE}.json

idrun=`jq -r '.. | objects | select(has("core.runs")) | .["core.runs"][0]' old_md.json`
idsubrun=`jq -r '.. | objects | select(has("core.runs_subruns")) | .["core.runs_subruns"][0]' old_md.json`

sed -i "s/-99999/$idrun/g" ${OUTFILE}.json
sed -i "s/-88888/$idsubrun/g" ${OUTFILE}.json

# cp ${OUTFILE}.json triggerAnaTree_${reco_name}_hist.root.json
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
#  echo "$FILE" > justin-processed-pfns.txt
   echo "$FILE" >> temp_pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls

done < file.list
cp temp_pfns.txt justin-processed-pfns.txt
exit 0
justIN time: 2026-02-06 13:05:07 UTC       justIN version: 01.06.00