justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 11278, Stage 1

Priority50
Processors4
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes6815744000 (6500 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio usertests:usertests-fnal-w11278s1p1*.root2592000False

Environment variables

NameValue
FHICL_TAR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/6e1edf004832887b3ec4d0f6daab4c59de5c07c7
INPUT_TAR_DIR_LOCAL/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/d6dc8fee1c7500fa5ab50cc93d16767de6e91bd5
METADATA_DIR/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/273045f43d7c86de1848a0b2b33e0d614d73cf83
NEVENTS1

Condor Class Ads

NameValue
HAS_CVMFS_dune_osgstorage_orgtrue

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
999099600102

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
76800001007531400
Files processed000.10.10.20.20.30.30.40.40.50.50.60.60.70.70.80.80.90.911Dec-19 08:00Dec-19 09:00Dec-19 10:00Files processedBin start timesNumber per binUS_FNAL-FermiG…US_FNAL-FermiGrid
Replicas per RSE418477.2561746317069197.87643426439385158381.89568182437716324.05713309069984119304.348999954777289.774103669213679279.90158447310057233.2794598012453464285.58684226646164188.8375380017229257305.18777606965847156.2804955716241633326.9119376361249138.0775892435169732345.337538001723129.086842266461618360.5337622507848124.8268212081366613370.22054019875475123.401584473100537376.5250692437626122.98504846919862379.3680727956639122.926985264394Replicas per RSEDUNE_US_BNL_SDCC (4…DUNE_US_BNL_SDCC (41%)DUNE_US_FNAL_DISK_S…DUNE_US_FNAL_DISK_STAGE (15%)DUNE_UK_MANCHESTER_…DUNE_UK_MANCHESTER_CEPH (11%)DUNE_CERN_EOS (7%)NIKHEF (6%)QMUL (5%)DUNE_IT_INFN_CNAF (…DUNE_IT_INFN_CNAF (3%)DUNE_CA_SFU (3%)DUNE_ES_PIC (1%)PRAGUE (1%)RAL-PP (0%)DUNE_FR_CCIN2P3_DIS…DUNE_FR_CCIN2P3_DISK (0%)

RSEs used

NameInputsOutputs
DUNE_US_BNL_SDCC1670
DUNE_US_FNAL_DISK_STAGE1601
DUNE_UK_MANCHESTER_CEPH1370
NIKHEF820
DUNE_CERN_EOS630
QMUL600
PRAGUE260
DUNE_ES_PIC230
DUNE_IT_INFN_CNAF190
DUNE_CA_SFU160
RAL-PP140
DUNE_FR_CCIN2P3_DISK10

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
US_UChicago1670
US_FNAL-FermiGrid1460
UK_RAL-PPD1030
NL_NIKHEF600
UK_Manchester490
CERN480
CZ_FZU460
ES_PIC250
UK_QMUL240
NL_SURFsara220
UK_Brunel130
UK_Bristol120
IT_CNAF120
CA_Victoria110
UK_Sheffield100
CA_SFU40

Jobscript

#!/bin/bash
#

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

if [ -n "${DUNESW_DIR}" ]; then
  stat ${DUNESW_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat dunesw dir"
    exit 1
  fi

  export PRODUCTS=$DUNESW_DIR:$PRODUCTS
fi


export PRODUCTS=$INPUT_TAR_DIR_LOCAL:${PRODUCTS}

echo "PRODUCTS $PRODUCTS"

DUNE_TAG=v10_14_00d00
#Setup recent lar software suite
DUNE_VERSION=${DUNE_VERSION:-${DUNE_TAG}}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit 1
fi

export PROTODUNEANA_DIR=${INPUT_TAR_DIR_LOCAL}/protoduneana/${DUNE_VERSION}
export PROTODUNEANA_INC=${PROTODUNEANA_DIR}/include
export PROTODUNEANA_FQ_DIR=${PROTODUNEANA_DIR}/slf7.x86_64.e26.prof
export PROTODUNEANA_LIB=${PROTODUNEANA_FQ_DIR}/lib

export CET_PLUGIN_PATH=$PROTODUNEANA_LIB:$CET_PLUGIN_PATH

export FHICL_FILE_PATH=${PROTODUNEANA_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${INPUT_TAR_DIR_LOCAL}:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${FHICL_TAR}:${FHICL_FILE_PATH}

export FW_SEARCH_PATH=${INPUT_TAR_DIR_LOCAL}:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${FHICL_TAR}:$FW_SEARCH_PATH

echo "FHICL_FILE_PATH: ${FHICL_FILE_PATH}"

if [ -n "${USE_INPUT_FCL}" ]; then
  
  if [ -z ${INPUT_DIR} ]; then
    echo "Error, INPUT_DIR is undefined but user requested USE_INPUT_FCL"
    exit 1
  fi

  stat ${INPUT_DIR}
  if [ $? -ne 0 ]; then
    echo "Failed to stat input dir. Exiting safely"
    exit 0
  fi

  FHICL_FILE_PATH=${INPUT_DIR}:${FHICL_FILE_PATH}
  echo "FCL PATH: $FHICL_FILE_PATH"
fi

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

# Temporary fix to get the propoer PDS map
export FHICL_FILE_PATH=${METADATA_DIR}:${FHICL_FILE_PATH}
export FW_SEARCH_PATH=${METADATA_DIR}:${FW_SEARCH_PATH}

FCL1=${FCL1:-"pdvd_MichelAnalysis_perso.fcl"}
echo "FCL1 dump:" ${FCL1}
fhicl-dump ${FCL1}
if [ $? -ne 0 ]; then
  echo "fhicl-dump ${FCL1} failed"
  exit 1
fi

echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "printing env"
env

echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "pfn: ${pfn}"
echo "did: ${did}"
now=$(date -u +"%Y%m%dT%H%M%SZ")

nevents=${NEVENTS:--1}
nevents=1

extra_line=""
if [ -n "${SKIPFCL2}" ]; then
  jobsub_id=`echo ${JUSTIN_JOBSUB_ID:-1.1@1} | cut -f1 -d'@' | sed -e"s/\./_/"`
fi

echo "Running reco stage1"
touch reco.log
starttime=`date +"%s"`.0
lar -c ${FCL1} \
    -n ${nevents} \
    ${pfn} #>reco.log 2>&1
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco1"
  cat reco.log
  exit $larExit
fi

output_reco_file=`ls *.root`

echo "Output files:"
echo "\tReco: ${output_reco_file}"

echo "Forming reco metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "root-tuple-virtual"\
                     --campaign "usertests" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_reco_file" -j "${output_reco_file}.json"
if [ $? -ne 0 ]; then
  echo "Error in reco metadata"
  exit 1
fi
echo "Ran successfully"
## TODO -- CHECK
cat ${output_reco_file}.json

echo "$pfn" > justin-processed-pfns.txt
justIN time: 2025-12-19 14:31:49 UTC       justIN version: 01.05.03