justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 10258, Stage 1

Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4194304000 (4000 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopefardet-hd
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio fardet-hd:fardet-hd-reco-ritm2612698_10258-fnal-w10258s1p1*reco.root51840000False

Environment variables

NameValue
INPUT_DIR/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/b47f01ac0d05214f74f7313134f879aeb35c59b7

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
9040000086600380

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
109200000606921012104222413110
Files processed002002004004006006008008001000100012001200140014001600160018001800200020002200220024002400Nov-16 12:00Nov-16 13:00Nov-16 14:00Nov-16 15:00Nov-16 16:00Nov-16 17:00Nov-16 18:00Nov-16 19:00Nov-16 20:00Nov-16 21:00Nov-16 22:00Nov-16 23:00Nov-17 00:00Nov-17 01:00Nov-17 02:00Files processedBin start timesNumber per binUK_BristolNL_SURFsaraUK_QMULUK_EdinburghUK_OxfordUK_SheffieldCA_SFUUK_ManchesterUK_RAL-Tier1UK_LancasterNL_NIKHEFUK_BrunelUK_RAL-PPDUS_UChicagoES_PICUS_WisconsinCZ_FZUUS_FNAL-FermiG…US_FNAL-FermiGridCERNIT_CNAFUS_UCSD
Replicas per RSE9040484.97382519490867225.718332055149749040277.62346154746484258.64009688448056952363.45364135645514130.008134858349681379.9478601934057128.700012908640081379.9826200638373128.70000143429337Replicas per RSEFNAL_DCACHE (47%)DUNE_FR_CCIN2P3_DISK (47%)DUNE_US_FNAL_DISK_STAGE (5%)QMUL (0%)DUNE_CERN_EOS (0%)

RSEs used

NameInputsOutputs
DUNE_FR_CCIN2P3_DISK180028613
DUNE_US_FNAL_DISK_STAGE8170
QMUL57
DUNE_CERN_EOS32
RAL_ECHO05
NIKHEF04
DUNE_ES_PIC04
DUNE_IT_INFN_CNAF03
DUNE_US_BNL_SDCC03
RAL-PP02
DUNE_UK_GLASGOW01
SURFSARA01

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
UK_QMUL1190113
UK_RAL-Tier11097217
NL_NIKHEF958184
CERN59883
UK_Lancaster5520
UK_Sheffield53436
NL_SURFsara46451
UK_RAL-PPD45520
ES_PIC41728
CZ_FZU28629
UK_Manchester1778
UK_Edinburgh16416
UK_Bristol1174
IT_CNAF11520
UK_Brunel510
UK_Oxford474
CA_SFU1668
US_UCSD40
US_Colorado40
US_UChicago065
US_FNAL-FermiGrid01

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite

#setup dunesw v10_11_00d01 -q e26:prof
setup dunesw v10_12_01d01 -q e26:prof
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 
 

for nf in {1..4}
  do
  DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 
 ##Check that any file was returned
 if [ "${DID_PFN_RSE}" == "" ] ; then
   echo "Could not get file"
#   exit 0
   continue
 fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 echo ${DID} >> did.list
 echo ${FILE} >> file.list 
done
 
while read inFile; do

temp=`echo "$inFile" | awk -F'/' '{print $NF}'`
DID="fardet-hd:"$temp
FILE=$inFile 
 
 
metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============TPG========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco="${infile%.root}" 
reco_name=${reco}_${now}_reco

echo "check  output filename "  ${reco_name}.root

lar -c  tpg_dune10kt_1x2x6.fcl $FILE -o ${reco_name}.root -n -1
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
#OUTFILE=`ls *reco.root`
OUTFILE=${reco_name}.root

#mv triggerAnaTree_hist.root triggerAnaTree_${reco_name}_hist.root


echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco --appversion  v10_12_01d01  --requestid ritm2612698  --input_json ${INPUT_DIR}/trg_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-hd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.temp.json 


sed '/DUNE/ s/.*/\L&/'  ${OUTFILE}.temp.json >  ${OUTFILE}.json

idrun=`jq -r '.. | objects | select(has("core.runs")) | .["core.runs"][0]' old_md.json`
idsubrun=`jq -r '.. | objects | select(has("core.runs_subruns")) | .["core.runs_subruns"][0]' old_md.json`

sed -i "s/-99999/$idrun/g" ${OUTFILE}.json
sed -i "s/-88888/$idsubrun/g" ${OUTFILE}.json

# cp ${OUTFILE}.json triggerAnaTree_${reco_name}_hist.root.json
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
#  echo "$FILE" > justin-processed-pfns.txt
   echo "$FILE" >> temp_pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls

done < file.list
cp temp_pfns.txt justin-processed-pfns.txt
exit 0
justIN time: 2025-12-19 12:13:08 UTC       justIN version: 01.05.03