justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 9410, Stage 1

Priority50
Processors1
Wall seconds18000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4193255424 (3999 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopevd-protodune-det-reco
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio vd-protodune-det-reco:fnal-w9410s1p1*_keepup.root7776000False

Environment variables

NameValue
FHICL_TAR/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/0183247c92433907df7c1f561070e5f865254efc
METADATA_DIR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/ff18e11900bfcc2b467fc81eba35d7e7de647a80

Condor Class Ads

NameValue
HAS_CVMFS_dune_osgstorage_orgtrue

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
3430048900293605

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
71450000430413628320811403198
Files processed00100100200200300300400400500500600600700700800800900900Oct-28 10:00Oct-28 12:00Oct-28 14:00Oct-28 16:00Oct-28 18:00Oct-28 20:00Oct-28 22:00Oct-29 00:00Oct-29 02:00Oct-29 04:00Oct-29 06:00Oct-29 08:00Oct-29 10:00Oct-29 12:00Oct-29 14:00Oct-29 16:00Oct-29 18:00Oct-29 20:00Oct-29 22:00Files processedBin start timesNumber per binUK_RAL-Tier1NL_SURFsaraUK_DurhamUK_QMULUK_SheffieldUK_ManchesterNL_NIKHEFUK_BrunelUK_RAL-PPDFR_CCIN2P3ES_PICUK_BristolUK_LancasterCERNIT_CNAF
Replicas per RSE3430490.025244.53430269.975244.50000000000003Replicas per RSEDUNE_CERN_EOS (50%)FNAL_DCACHE (50%)

RSEs used

NameInputsOutputs
DUNE_CERN_EOS50080
DUNE_US_FNAL_DISK_STAGE02914
RAL_ECHO02
SURFSARA01

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
NL_NIKHEF43127
UK_Manchester25922
NL_SURFsara20418
UK_QMUL18811
UK_Sheffield17411
UK_RAL-Tier115725
UK_RAL-PPD13916
ES_PIC909
UK_Lancaster601
UK_Durham335
UK_Brunel140
FR_CCIN2P3141
CZ_FZU110
CERN10
UK_Bristol10

Jobscript

#!/bin/bash
#

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -n "${DUNESW_DIR}" ]; then
  stat ${DUNESW_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat dunesw dir"
    exit 1
  fi

  export PRODUCTS=$DUNESW_DIR:$PRODUCTS
fi

if [ -n "${LARRECO_DIR}" ]; then
  stat ${LARRECO_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat larreco dir"
    exit 1
  fi

  export PRODUCTS=$LARRECO_DIR:$PRODUCTS
fi


if [ -n "${DUNEPROTOTYPES_DIR}" ]; then
  stat ${DUNEPROTOTYPES_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat dunedetdataformats dir"
    exit 1
  fi

  export PRODUCTS=$DUNEPROTOTYPES_DIR:$PRODUCTS
fi

echo "PRODUCTS $PRODUCTS"

DUNE_TAG=v10_11_00d00
#Setup recent lar software suite
DUNE_VERSION=${DUNE_VERSION:-${DUNE_TAG}}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit 1
fi

export FHICL_FILE_PATH=${FHICL_TAR}:${FHICL_FILE_PATH}
export FW_SEARCH_PATH=${FHICL_TAR}:$FW_SEARCH_PATH

echo "FHICL_FILE_PATH: ${FHICL_FILE_PATH}"

if [ -n "${USE_INPUT_FCL}" ]; then
  
  if [ -z ${INPUT_DIR} ]; then
    echo "Error, INPUT_DIR is undefined but user requested USE_INPUT_FCL"
    exit 1
  fi

  stat ${INPUT_DIR}
  if [ $? -ne 0 ]; then
    echo "Failed to stat input dir. Exiting safely"
    exit 0
  fi

  FHICL_FILE_PATH=${INPUT_DIR}:${FHICL_FILE_PATH}
  echo "FCL PATH: $FHICL_FILE_PATH"
fi

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

# Temporary fix to get the propoer PDS map
export FHICL_FILE_PATH=${METADATA_DIR}:${FHICL_FILE_PATH}
export FW_SEARCH_PATH=${METADATA_DIR}:${FW_SEARCH_PATH}

FCL1=${FCL1:-"standard_reco_stage1_protodunevd_keepup_all.fcl"}
#FCL1=${FCL1:-"standard_reco_stage1_protodunevd_keepup.fcl"}
#FCL1=${FCL1:-"standard_reco_protodunevd_keepup.fcl"}
echo "FCL1 dump:" ${FCL1}
fhicl-dump ${FCL1}
if [ $? -ne 0 ]; then
  echo "fhicl-dump ${FCL1} failed"
  exit 1
fi

# not yet setup
#FCL2=${FCL2:-"protodunevd_data_reco_stage2_calibration.fcl"}
#echo "FCL2 dump:" ${FCL2}
#fhicl-dump ${FCL2}
#if [ $? -ne 0 ]; then
#  echo "fhicl-dump ${FCL2} failed"
#  exit 1
#fi


echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "printing env"
env

echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "pfn: ${pfn}"
echo "did: ${did}"
now=$(date -u +"%Y%m%dT%H%M%SZ")

nevents=${NEVENTS:--1}

extra_line=""
if [ -n "${SKIPFCL2}" ]; then
  jobsub_id=`echo ${JUSTIN_JOBSUB_ID:-1.1@1} | cut -f1 -d'@' | sed -e"s/\./_/"`
  extra_line="-T pdvd_${jobsub_id}_${JUSTIN_WORKFLOW_ID}_${now}_decoder.root"
fi
echo "Running reco stage1"
touch reco.log
starttime=`date +"%s"`.0
LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so lar \
    -c ${FCL1} \
    -n ${nevents} \
    ${extra_line} ${pfn} #>reco.log 2>&1
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco1"
  cat reco.log
  exit $larExit
fi

if [ -n "${SKIPFCL2}" ]; then
  echo "$pfn" > justin-processed-pfns.txt
  exit 0
fi

#output_stage1_file=`ls *stage1.root`

starttime=`date +"%s"`.0
#lar -c ${FCL2} \
#    $output_stage1_file #>reco.log 2>&1
#larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco2"
  cat reco.log
  exit $larExit
fi



output_reco_file=`ls *keepup.root`
output_mr_file=`ls *keepup_hists.root`

#new_mr_file=`echo $output_reco_file | sed -e "s/reco/reco_hists/"`
#mv $output_mr_file $new_mr_file
#output_mr_file=$new_mr_file

echo "Output files:"
echo "\tReco: ${output_reco_file}"
echo "\tHists: ${output_mr_file}"

echo "Forming reco metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "artroot" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "full-reconstructed" --get_events -p "$did" \
                     --campaign "vd-protodune-reco-keepup-v0" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_reco_file" -j "${output_reco_file}.json"
if [ $? -ne 0 ]; then
  echo "Error in reco metadata"
  exit 1
fi
echo "Ran successfully"
## TODO -- CHECK
cat ${output_reco_file}.json

echo "Forming hist metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "root-tuple-virtual" -p "$did" \
                     --campaign "vd-protodune-reco-keepup-v0" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_mr_file" -j "${output_mr_file}.json"
 #--parent_as_json \
if [ $? -ne 0 ]; then
  echo "Error in hist metadata"
  exit 1
fi
echo "formed"
cat ${output_mr_file}.json



echo "$pfn" > justin-processed-pfns.txt
justIN time: 2025-11-04 01:33:00 UTC       justIN version: 01.05.01