justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 12719, Stage 1

Workflow12719
Campaign1022
Priority50
Processors1
Wall seconds86400
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes4194304000 (4000 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio usertests:fnal-w12719s1p1*_offline.root15552000True

Environment variables

NameValue
FHICL_FILEstandard_reco_stage0_protodunevd_offline.fcl
METADATA_DIR/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/5de38a4c938bde1a0f76a07a324c79d1ca2a3058
NEVENTS-1

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
10000078691501

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
167200786147707825800
Files processed0040408080120120160160200200240240280280320320360360Feb-03 22:00Feb-03 23:00Feb-04 00:00Feb-04 01:00Feb-04 02:00Feb-04 03:00Files processedBin start timesNumber per binUK_RAL-PPDCERNUK_QMULUK_OxfordUK_ManchesterUK_RAL-Tier1NL_NIKHEFES_PICUK_ImperialIT_CNAF
Replicas per RSE1000490.0138048096969246.06951466826555982269.9861951903031242.9304853317345Replicas per RSEDUNE_CERN_EOS (50%)FNAL_DCACHE (49%)

RSEs used

NameInputsOutputs
DUNE_CERN_EOS11090
DUNE_US_FNAL_DISK_STAGE0915

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
NL_NIKHEF400
CERN130
UK_RAL-Tier1120
UK_Manchester90
UK_RAL-PPD80
UK_QMUL70
IT_CNAF60
UK_Oxford40
UK_Imperial20

Jobscript

#!/bin/bash
#

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

DUNE_TAG=v10_17_00d00
#Setup recent lar software suite
DUNE_VERSION=${DUNE_VERSION:-${DUNE_TAG}}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit 1
fi

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

#export WIRECELL_PATH=${WIRECELL_CFG}:$WIRECELL_PATH
export FHICL_FILE_PATH=${METADATA_DIR}:${FHICL_FILE_PATH}
export FW_SEARCH_PATH=${METADATA_DIR}:$FW_SEARCH_PATH 

FCL1=${FCL1:-${FHICL_FILE}}

echo "FCL1 dump:" ${FCL1}
fhicl-dump ${FCL1}
if [ $? -ne 0 ]; then
  echo "fhicl-dump ${FCL1} failed"
  exit 1
fi

echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "printing env"
env

echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "pfn: ${pfn}"
echo "did: ${did}"
now=$(date -u +"%Y%m%dT%H%M%SZ")

#output_file=$(basename ${pfn%.*})_${now}_singlehit.root
#echo "output_file: $output_file"

nevents=${NEVENTS:--1}

extra_line=""
if [ -n "${SKIPFCL2}" ]; then
  jobsub_id=`echo ${JUSTIN_JOBSUB_ID:-1.1@1} | cut -f1 -d'@' | sed -e"s/\./_/"`
fi

echo -n "Running reco "
touch reco.log
starttime=`date +"%s"`.0
if [[ "${FCL1}" == *"stage0"* ]]; then
    echo "stage0"
    LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so lar -c ${FCL1} \
    -n ${nevents} \
    ${pfn} #>reco.log 2>&1
else
    echo "stage1"
    lar -c ${FCL1} \
    -n ${nevents} \
    ${pfn} #>reco.log 2>&
fi
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco1"
  cat reco.log
  exit $larExit
fi

if [ -n "${SKIPFCL2}" ]; then
  echo "$pfn" > justin-processed-pfns.txt
  exit 0
fi

#output_stage1_file=`ls *stage1.root`

#starttime=`date +"%s"`.0
#lar -c ${FCL2} \
#    $output_stage1_file #>reco.log 2>&1
#larExit=$?
#endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco2"
  cat reco.log
  exit $larExit
fi

output_reco_file=`ls *_offline.root`

echo "Output files:"
echo "\tReco: ${output_reco_file}"
#echo "\tHists: ${output_mr_file}"

echo "Forming reco metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "artroot" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "full-reconstructed" --get_events -p "$did" \
                     --campaign "vd-protodune-det-reco-test" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_reco_file" -j "${output_reco_file}.json"
if [ $? -ne 0 ]; then
  echo "Error in reco metadata"
  exit 1
fi
echo "Ran successfully"
cat ${output_reco_file}.json

if [[ "${FCL1}" == *"stage1"* ]]; then
  larcv_file=`ls larcv_*.root`
  output_larcv_file=${output_reco_file%.*}_larcv.root
  mv ${larcv_file} ${output_larcv_file}

  echo "\tLArCV: ${output_larcv_file}"

  echo "Forming larcv metadata"
  python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
		       --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
		       --data_tier "root-tuple-virtual" -p "$did" \
		       --campaign "vd-protodune-det-reco-test" \
		       --fcl $FCL1 \
		       -f "${JUSTIN_SCOPE}:$output_larcv_file" -j "${output_larcv_file}.json"
  # --parent_as_json \
  if [ $? -ne 0 ]; then
    echo "Error in larcv metadata"
    exit 1
  fi
  echo "formed"
  cat ${output_larcv_file}.json
fi

#echo "Forming hist metadata"
#python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
#                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
#                     --data_tier "root-tuple-virtual" -p "$did" \
#                     --campaign "hd-protodune-sim-reco" \
#                     --fcl $FCL1 \
#                     -f "${JUSTIN_SCOPE}:$output_mr_file" -j "${output_mr_file}.json"
# --parent_as_json \
#if [ $? -ne 0 ]; then
#  echo "Error in hist metadata"
#  exit 1
#fi
#echo "formed"
#cat ${output_mr_file}.json

echo "$pfn" > justin-processed-pfns.txt
justIN time: 2026-02-04 02:58:37 UTC       justIN version: 01.06.00