Workflow 10079, Stage 1

Workflow	10079
Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	2097152000 (2000 MiB)
Max distance for inputs	0.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio usertests:lardon-cb-fnal-w10079s1p1	cbbot*.hdf5	86400	False

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10	0	10	0	0	0	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
0	0	0	0	0	0	0	0	0	0	0	0

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash
#

export WORK=$HOME/workspace

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup python v3_9_15
setup cmake v3_27_4

#SETUP STUFF HERE
if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "Justin specific env vars"
env | grep JUSTIN
now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}



echo "Installing uv"
pip3 install --user uv
export PATH=$HOME/.local/bin/:$PATH
unsetup python
setup gcc v12_1_0
setup hdf5 v1_12_2b -q e26:prof
export HDF5_VERSION='1.12.2'
export CPATH=${CPATH}:${HDF5_DIR}/Linux64bit+3.10-2.17-e26-prof/include
export LIBRARY_PATH=${HDF5_LIB}:${LIBRARY_PATH}
export LD_LIBRARY_PATH=${HDF5_LIB}:${LD_LIBRARY_PATH}

export PYTHONPATH=${HOME}/.local/share/uv/python/cpython-3.12.12-linux-x86_64-gnu/lib/python3.12/
export PYTHON_ROOT=${HOME}/.local/share/uv/python/cpython-3.12.12-linux-x86_64-gnu/
export PYTHON_INCLUDE=${HOME}/.local/share/uv/python/cpython-3.12.12-linux-x86_64-gnu/include/python3.12/
export PYTHON_LIB=${HOME}/.local/share/uv/python/cpython-3.12.12-linux-x86_64-gnu/lib

echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`

input_filename=`echo $did | cut -f2 -d':'`
echo "input file: $input_filename"
echo $DID_PFN_RSE

#for testing
#pfn=root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/tape_backed/dunepro//vd-protodune/raw/2025/detector/cosmics/None/00/04/03/80/np02vd_raw_run040380_0019_df-s04-d3_dw_0_20251105T180444.hdf5

#Setup ifdhc. Copy the file, then unsetup ifdhc
cd $HOME
setup ifdhc 
ifdh cp -D $pfn ./
unsetup ifdhc 

echo "Done"

echo $PYTHONPATH

export PREFIX=$HOME/local
mkdir -p $PREFIX
cd $HOME

#mkdir -p $HOME/local/src && cd $HOME/local/src
wget https://github.com/libspatialindex/libspatialindex/releases/download/1.9.3/spatialindex-src-1.9.3.tar.gz
tar xzf spatialindex-src-1.9.3.tar.gz
cd spatialindex-src-1.9.3
#./configure --prefix=$HOME/local
mkdir build && cd build

cmake .. \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_INSTALL_PREFIX=$PREFIX

#cmake . -B $HOME/local
make -j$(nproc)
make install

ls $PREFIX/lib/libspatialindex*

# Make sure Python can find the library
export LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH

cd $HOME
echo $LD_LIBRARY_PATH
git clone https://github.com/calcuttj/lardon.git
export LARDON_PATH=$HOME/lardon/lardon
export LARDON_RECO=$HOME/workspace
export LARDON_PLOT=$HOME

cd lardon

uv sync --python 3.11

uv run lardon -det cbbot -run 37041 -sub 0 -file "$HOME/$input_filename" -trk -out crp8


exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error running. Exiting with ${exitcode}"
  exit $exitcode
fi

echo "lsing"
ls -lhS
echo "disk usage"
du -sh .

#THIS TELLS JUSTIN THIS FILE WAS SUCCESSFULLY PROCESSED
echo "$pfn" > $WORK/justin-processed-pfns.txt