Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)
File reset events, by site
Site
Allocated
Outputting
UK_Lancaster
163
20
CERN
133
86
UK_Sheffield
58
42
ES_PIC
54
10
UK_RAL-PPD
40
159
NL_SURFsara
40
10
UK_Manchester
40
12
UK_RAL-Tier1
20
81
CZ_FZU
0
59
UK_QMUL
0
45
NL_NIKHEF
0
25
UK_Bristol
0
14
UK_Oxford
0
6
FR_CCIN2P3
0
4
Jobscript
#!/bin/sh
#
# Example jobscript that runs lar for all the files
# referred to by the MQL expression given on the justin command line.
#
# Submit with something like this:
#
# ./justin simple-workflow \
# --max-distance 30 \
# --mql "rucio-dataset protodune-sp:np04_raw_run_number_5769" \
# --jobscript lar.jobscript
#
# Then monitor with dashboard or ./justin show-jobs --workflow-id ID
# where ID is the value printed by the first command
#
# the xroot lib for streaming non-root files is in testproducts,
# so add it to the start of the path
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
if [ -z ${JUSTIN_PROCESSORS} ]; then
JUSTIN_PROCESSORS=1
fi
echo "Justin processors: ${JUSTIN_PROCESSORS}"
export TF_NUM_THREADS=${JUSTIN_PROCESSORS}
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS}
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS}
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS}
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS}
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}
# From jobsub
export CLUSTER=${CLUSTER:-1}
export PROCESS=${PROCESS:-1}
setup dunesw ${DUNESW_VERSION:-v10_10_04d00} -q e26:prof
setup_exit=$?
if [ "$setup_exit" != 0 ]; then
echo "Error setting up dunesw ${DUNESW_VERSION}"
exit
fi
export RUCIO_ACCOUNT=justinreadonly
if [ -n "${extra_fcl_path}" ]; then
echo "ADDING ${extra_fcl_path} to fcl file path"
sleep 1
export FHICL_FILE_PATH=${extra_fcl_path}:${FHICL_FILE_PATH}
fi
# Get an unprocessed file from this stage
#did_pfn_rse=`$JUSTIN_PATH/justin-get-file`
#if [ "${did_pfn_rse}" == "" ] ; then
# echo "Could not get file"
# exit 0
#fi
#
#did=`echo $did_pfn_rse | cut -f1 -d' '`
#pfn=`echo $did_pfn_rse | cut -f2 -d' '`
#rse=`echo $did_pfn_rse | cut -f3 -d' '`
#name_only=$(echo $did | cut -f2 -d':')
#echo "name" $name_only
NFILES=${NFILES:-1}
files=()
nfiles=0
#if [ $NFILES -eq 1 ]; then
# echo "Will use justin-get-file"
# DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
# if [ "${DID_PFN_RSE}" == "" ] ; then
# echo "Could not get file"
# exit 0
# fi
# export pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
# export did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
#
# if [ -z ${TESTFILE} ]; then
# files+=($pfn)
# else
# files+=(${TESTFILE})
# fi
#
if [ -n "${TESTFILE}" ]; then
echo "Using testfile"
files+=(${TESTFILE})
nfiles=1
else
nfiles=0
for i in `seq 1 ${NFILES:-1}`; do
echo $i
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
echo "Could not get file -- exiting loop"
break
fi
echo "did_pfn_rse: ${DID_PFN_RSE}"
THISFILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
echo $THISFILE
files+=(${THISFILE})
nfiles=$(( nfiles + 1 ))
done
fi
echo "Got $nfiles Files"
echo ${files[@]}
if [[ $nfiles == 0 ]]; then
echo "No files exiting"
exit 0
fi
echo "Found input file URL $pfn at $rse"
#Get the stage and momentum
export stage=${stage:-"Gen"}
export momentum=${momentum:-1}
beam_fcl=${beam_fcl:-"run_pdhd_beamevent.fcl"}
now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}
output_preamble="pdhd_prod_beam_"
nevents=${nevents:--1}
nskip=${nskip:-0}
echo "XROOTD: $XROOTD_LIB"
#Generation
LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so lar \
-n ${nevents} \
--nskip ${nskip} \
-c ${beam_fcl} \
-o pdhd_${jobid}_${stageid}_${now}_beam.root \
-T pdhd_${jobid}_${stageid}_${now}_beam_hist.root \
-s "${files[@]}"
larReturnCode=$?
echo "gen lar returns $larReturnCode"
if [ "$larReturnCode" != 0 ] ; then
exit $larReturnCode
fi
#Generation Done
####EDIT METADATA FOR THE FOLLOWING FILE
#${name_only}_${jobid}_${stageid}_${now}_detsim_reco1.root
# Record that we processed the input file ok (did we???)
#echo "$pfn" > justin-processed-pfns.txt
for i in ${files[@]}; do
echo "${i}" >> justin-processed-pfns.txt
done
# For debugging
for i in *.json
do
echo "==== Start $i ===="
cat $i
echo "==== End $i ===="
done
ls -ltR
exit 0
justIN time: 2025-11-04 01:33:08 UTC justIN version: 01.05.01
