justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 11836, Stage 1

Workflow11836
Campaign249
Priority50
Processors4
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes6815744000 (6500 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio usertests:usertests-fnal-w11836s1p1*.root7776000False

Environment variables

NameValue
FHICL_TAR/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/9bce52b1ac1261057869b0044843c4097848bc24
INPUT_TAR_DIR_LOCAL/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/c6a4318845c6abd24779182f1f3f5480d4285e0d
METADATA_DIR/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/273045f43d7c86de1848a0b2b33e0d614d73cf83

Condor Class Ads

NameValue
HAS_CVMFS_dune_osgstorage_orgtrue

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
241000024100

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
5510000551000000
Files processed002020404060608080100100120120140140160160180180200200220220240240Jan-13 15:00Jan-13 16:00Jan-13 17:00Files processedBin start timesNumber per binCA_SFUNL_NIKHEFUS_UChicagoUS_FNAL-FermiG…US_FNAL-FermiGridUS_UCSD
Replicas per RSE240382.8563214744223354.487917757518971375.71611949850853134.558429186913371378.5717189126476134.48427095121576Replicas per RSEDUNE_US_FNAL_DISK_STAGE (99%)NIKHEF (0%)SURFSARA (0%)

RSEs used

NameInputsOutputs
DUNE_US_FNAL_DISK_STAGE239241
NIKHEF10
SURFSARA10

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash
#

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

if [ -n "${DUNESW_DIR}" ]; then
  stat ${DUNESW_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat dunesw dir"
    exit 1
  fi

  export PRODUCTS=$DUNESW_DIR:$PRODUCTS
fi


export PRODUCTS=$INPUT_TAR_DIR_LOCAL:${PRODUCTS}

echo "PRODUCTS $PRODUCTS"

DUNE_TAG=v10_16_00d00
#Setup recent lar software suite
DUNE_VERSION=${DUNE_VERSION:-${DUNE_TAG}}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit 1
fi

export PROTODUNEANA_DIR=${INPUT_TAR_DIR_LOCAL}/protoduneana/${DUNE_VERSION}
export PROTODUNEANA_INC=${PROTODUNEANA_DIR}/include
export PROTODUNEANA_FQ_DIR=${PROTODUNEANA_DIR}/slf7.x86_64.e26.prof
export PROTODUNEANA_LIB=${PROTODUNEANA_FQ_DIR}/lib

export CET_PLUGIN_PATH=$PROTODUNEANA_LIB:$CET_PLUGIN_PATH

export FHICL_FILE_PATH=${PROTODUNEANA_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${INPUT_TAR_DIR_LOCAL}:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${FHICL_TAR}:${FHICL_FILE_PATH}

export FW_SEARCH_PATH=${INPUT_TAR_DIR_LOCAL}:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${FHICL_TAR}:$FW_SEARCH_PATH

echo "FHICL_FILE_PATH: ${FHICL_FILE_PATH}"

if [ -n "${USE_INPUT_FCL}" ]; then
  
  if [ -z ${INPUT_DIR} ]; then
    echo "Error, INPUT_DIR is undefined but user requested USE_INPUT_FCL"
    exit 1
  fi

  stat ${INPUT_DIR}
  if [ $? -ne 0 ]; then
    echo "Failed to stat input dir. Exiting safely"
    exit 0
  fi

  FHICL_FILE_PATH=${INPUT_DIR}:${FHICL_FILE_PATH}
  echo "FCL PATH: $FHICL_FILE_PATH"
fi

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

# Temporary fix to get the propoer PDS map
export FHICL_FILE_PATH=${METADATA_DIR}:${FHICL_FILE_PATH}
export FW_SEARCH_PATH=${METADATA_DIR}:${FW_SEARCH_PATH}

FCL1=${FCL1:-"pdvd_MichelAnalysis_perso.fcl"}
echo "FCL1 dump:" ${FCL1}
fhicl-dump ${FCL1}
if [ $? -ne 0 ]; then
  echo "fhicl-dump ${FCL1} failed"
  exit 1
fi

echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "printing env"
env

echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "pfn: ${pfn}"
echo "did: ${did}"
now=$(date -u +"%Y%m%dT%H%M%SZ")

nevents=${NEVENTS:--1}
nevents=-1

extra_line=""
if [ -n "${SKIPFCL2}" ]; then
  jobsub_id=`echo ${JUSTIN_JOBSUB_ID:-1.1@1} | cut -f1 -d'@' | sed -e"s/\./_/"`
fi

echo "Running reco stage1"
touch reco.log
starttime=`date +"%s"`.0
lar -c ${FCL1} \
    -n ${nevents} \
    ${pfn} #>reco.log 2>&1
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco1"
  cat reco.log
  exit $larExit
fi

output_reco_file=`ls *.root`

echo "Output files:"
echo "\tReco: ${output_reco_file}"

echo "Forming reco metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "root-tuple-virtual"\
                     --campaign "usertests" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_reco_file" -j "${output_reco_file}.json"
if [ $? -ne 0 ]; then
  echo "Error in reco metadata"
  exit 1
fi
echo "Ran successfully"
## TODO -- CHECK
cat ${output_reco_file}.json

echo "$pfn" > justin-processed-pfns.txt
justIN time: 2026-02-04 08:38:51 UTC       justIN version: 01.06.00