justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 10250, Stage 1

Workflow10250
Campaign69
Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4194304000 (4000 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopefardet-hd
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio fardet-hd:fardet-hd-reco-ritm2612698_10250-fnal-w10250s1p1*reco.root51840000False

Environment variables

NameValue
INPUT_DIR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/b4e19b6c2a60f80ce0114a216f461c819425a95c

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
59810400597700

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
33470000330230411000
Files processed0010001000200020003000300040004000Nov-16 08:00Nov-16 09:00Nov-16 10:00Nov-16 11:00Nov-16 12:00Nov-16 13:00Files processedBin start timesNumber per binUK_BristolNL_SURFsaraUK_QMULUK_EdinburghCA_SFUUK_RAL-Tier1NL_NIKHEFUK_RAL-PPDES_PICCZ_FZUUS_FNAL-FermiG…US_FNAL-FermiGridCERNIT_CNAFUK_OxfordUK_SheffieldUK_ManchesterUK_Lancaster
Replicas per RSE5981485.09889920751135227.495279762405725981276.5053013450691253.4148744607078475366.6200381554925129.553522696846379.3624008642348128.7019303721955379.6546323102253128.70056637778694379.89373284604216128.70005362160913Replicas per RSEDUNE_FR_CCIN2P3_DISK (48%)FNAL_DCACHE (48%)PRAGUE (3%)QMUL (0%)DUNE_US_FNAL_DISK_STAGE (0%)DUNE_UK_MANCHESTER_CEPH (0%)

RSEs used

NameInputsOutputs
DUNE_FR_CCIN2P3_DISK58325953
PRAGUE1691
QMUL61
DUNE_US_FNAL_DISK_STAGE50
DUNE_UK_MANCHESTER_CEPH40
RAL_ECHO011
NIKHEF06
RAL-PP02
DUNE_CERN_EOS01
DUNE_ES_PIC01

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
ES_PIC110
UK_QMUL80
UK_Sheffield48
NL_NIKHEF40
UK_RAL-Tier140

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite

#setup dunesw v10_11_00d01 -q e26:prof
setup dunesw v10_12_01d01 -q e26:prof
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 
 

for nf in {1..4}
  do
  DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 
 ##Check that any file was returned
 if [ "${DID_PFN_RSE}" == "" ] ; then
   echo "Could not get file"
#   exit 0
   continue
 fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 echo ${DID} >> did.list
 echo ${FILE} >> file.list 
done
 
while read inFile; do

temp=`echo "$inFile" | awk -F'/' '{print $NF}'`
DID="fardet-hd:"$temp
FILE=$inFile 
 
 
metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============TPG========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco="${infile%.root}" 
reco_name=${reco}_${now}_reco

echo "check  output filename "  ${reco_name}.root

lar -c  tpg_dune10kt_1x2x6.fcl $FILE -o ${reco_name}.root -n -1
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
#OUTFILE=`ls *reco.root`
OUTFILE=${reco_name}.root

#mv triggerAnaTree_hist.root triggerAnaTree_${reco_name}_hist.root


echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco --appversion  v10_12_01d01  --requestid ritm2612698  --input_json ${INPUT_DIR}/trg_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-hd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.temp.json 


sed '/DUNE/ s/.*/\L&/'  ${OUTFILE}.temp.json >  ${OUTFILE}.json

idrun=`jq -r '.. | objects | select(has("core.runs")) | .["core.runs"][0]' old_md.json`
idsubrun=`jq -r '.. | objects | select(has("core.runs_subruns")) | .["core.runs_subruns"][0]' old_md.json`

sed -i "s/-99999/$idrun/g" ${OUTFILE}.json
sed -i "s/-88888/$idsubrun/g" ${OUTFILE}.json

# cp ${OUTFILE}.json triggerAnaTree_${reco_name}_hist.root.json
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
#  echo "$FILE" > justin-processed-pfns.txt
   echo "$FILE" >> temp_pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls

done < file.list
cp temp_pfns.txt justin-processed-pfns.txt
exit 0
justIN time: 2026-02-06 13:06:10 UTC       justIN version: 01.06.00