Workflow 12581, Stage 1
| Workflow | 12581 |
| Campaign | 899 |
| Priority | 50 |
| Processors | 1 |
| Wall seconds | 80000 |
| Image | /cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest |
| RSS bytes | 10485760000 (10000 MiB) |
| Max distance for inputs | 100.0 |
| Enabled input RSEs |
CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC |
| Enabled output RSEs |
CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC |
| Enabled sites |
BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin |
| Scope | usertests |
| Events for this stage |
Output patterns
| | Destination | Pattern | Lifetime | For next stage | RSE expression |
|---|
| 1 | Rucio usertests:pdvd-cosmics-pds-test-fnal-w12581s1p1 | prod_cosmics_protodunevd_*.* | 2592000 | False | |
Environment variables
| Name | Value |
|---|
| CAMPAIGN | pdvd-cosmics-pds-test |
| DUNEOPDET_TAR | /cvmfs/fifeuser4.opensciencegrid.org/sw/dune/8f5cc33c7a1d79f8848d3835488e07c9f68c80d1 |
| DUNESW_TAR | /cvmfs/fifeuser4.opensciencegrid.org/sw/dune/de73107d103be5d8b106e880e597d6bcdae43425 |
| FHICL_TAR | /cvmfs/fifeuser3.opensciencegrid.org/sw/dune/99eff67c1d77c2810e812e46ec111a70b466244f |
| JSONFILE | pdvd_cosmics_base_meta.json |
| MCJob | 1 |
| NEVENTS | 1 |
| pipyaml | 1 |
| UTIL_TAR | /cvmfs/fifeuser2.opensciencegrid.org/sw/dune/1579ab7ed3bdd86f0ff3f994bcc57056ffbd0cd3 |
| YAMLFILE | pdvd_cosmics_gen.yaml |
File states
| Total files | Finding | Unallocated | Allocated | Outputting | Processed | Not found | Failed |
|---|
|
| 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 |
Job states
| Total | Submitted | Started | Processing | Outputting | Finished | Notused | Aborted | Stalled | Jobscript error | Outputting failed | None processed |
|---|
| 6 | 0 | 0 | 0 | 0 | 2 | 2 | 0 | 0 | 2 | 0 | 0 |
RSEs used
| Name | Inputs | Outputs |
|---|
| MONTECARLO | 2 | 0 |
Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)
Jobscript
#!/bin/bash
#
#These must be defined
#if [ -z $FCL_FILE ]; then
# echo "Fatal Must provide FCL_FILE env var"
# exit 1
#fi
if [ -z $UTIL_TAR ]; then
echo "Fatal must provide UTIL_TAR env var"
exit 1
fi
export PYTHONPATH=$UTIL_TAR:$PYTHONPATH
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune
if [ -z ${JUSTIN_PROCESSORS} ]; then
JUSTIN_PROCESSORS=1
fi
echo "Justin processors: ${JUSTIN_PROCESSORS}"
export TF_NUM_THREADS=${JUSTIN_PROCESSORS}
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS}
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS}
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS}
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS}
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}
echo "Justin specific env vars"
env | grep JUSTIN
now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}
a=1
if [ -n "$DUNESW_TAR" ]; then
stat ${DUNESW_TAR}
if [ $? -ne 0 ]; then
echo "Failed to stat $DUNESW_TAR. Exiting safely"
exit 0
fi
export PRODUCTS=$DUNESW_TAR:$PRODUCTS
echo "Set dunesw to $DUNESW_TAR"
fi
if [ -n "$WIRECELL_TAR" ]; then
stat ${WIRECELL_TAR}
if [ $? -ne 0 ]; then
echo "Failed to stat $WIRECELL_TAR. Exiting safely"
exit 0
fi
export PRODUCTS=$WIRECELL_TAR:$PRODUCTS
echo "Set wirecell to $WIRECELL_TAR"
fi
#Setup recent lar software suite
DUNESW_VERSION=${DUNESW_VERSION:-v10_17_00d00}
setup dunesw \
"${DUNESW_VERSION}" \
-q "${DUNE_QUALIFIER:-e26:prof}"
setup_exit=$?
if [ $? -ne 0 ]; then
echo "Failed to setup dunesw $DUNESW_VERSION $DUNE_QUALIFIER"
exit $setup_exit
fi
export LARSIMDNN_DIR=${DUNESW_TAR}/larsimdnn/v10_01_07
export LARSIMDNN_INC=${LARSIMDNN_DIR}/include
export LARSIMDNN_FQ_DIR=${LARSIMDNN_DIR}/slf7.x86_64.e26.prof
export LARSIMDNN_LIB=${LARSIMDNN_FQ_DIR}/lib
export DUNEOPDET_DIR=${DUNEOPDET_TAR}/duneopdet/${DUNE_TAG}
export DUNEOPDET_INC=${DUNEOPDET_DIR}/include
export DUNEOPDET_FQ_DIR=${DUNEOPDET_DIR}/slf7.x86_64.e26.prof
export DUNEOPDET_LIB=${DUNEOPDET_FQ_DIR}/lib
export LD_LIBRARY_PATH=$DUNEOPDET_LIB:$LD_LIBRARY_PATH
export CET_PLUGIN_PATH=$DUNEOPDET_LIB:$CET_PLUGIN_PATH
export CMAKE_PREFIX_PATH=$DUNEOPDET_FQ_DIR:$CMAKE_PREFIX_PATH
export LD_LIBRARY_PATH=$LARSIMDNN_LIB:$LD_LIBRARY_PATH
export CET_PLUGIN_PATH=$LARSIMDNN_LIB:$CET_PLUGIN_PATH
export CMAKE_PREFIX_PATH=$LARSIMDNN_FQ_DIR:$CMAKE_PREFIX_PATH
export FW_SEARCH_PATH=${DUNESW_TAR}:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${DUNEOPDET_TAR}:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${FHICL_TAR}:$FW_SEARCH_PATH
echo "DUNESW loc:"
ups active | grep dunesw
echo "WIRECELL loc:"
ups active | grep wirecell
echo "LARSIMDNN loc:"
ups active | grep larsimdnn
export FHICL_FILE_PATH=${DUNEOPDET_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${LARSIMDNN_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${DUNESW_TAR}:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${FHICL_TAR}:${FHICL_FILE_PATH}
echo "printing env"
env
echo "DUNE specific env vars"
env | grep DUNE
# User can specify to pip install pyyaml
if [ -n "$pipyaml" ]; then
echo "Installing yaml"
pip install --user pyyaml
echo "Done"
# elif [ -n "${YAML_TAR}" ]; then
# echo "Using venv at ${YAML_TAR}"
# source ${YAML_TAR}/yaml_venv/bin/activate
fi
echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
echo "Could not get file"
exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
input_filename=`echo $did | cut -f2 -d':'`
echo "input file: $input_filename"
echo "jobsub_id: ${JUSTIN_JOBSUB_ID:-1}"
#Get run/subrun from parent if using input -- else from justin job ID/process"
if [ -z "$MCJob" ]; then
echo "Getting run subrun from $did"
run_subrun=`python -m beam_job_utils get_run_subrun -i $did`
get_run_ret=$?
if [ $get_run_ret -ne 0 ]; then
echo "error in get_run_subrun"
fi
echo $run_subrun
run=`echo $run_subrun | cut -f1 -d' '`
subrun=`echo $run_subrun | cut -f2 -d' '`
else
run=`echo $jobid | cut -f1 -d'_'` ##TODO -- check order here
subrun=`echo $jobid | cut -f2 -d'_'`
fi
nevents=${NEVENTS:--1} #Set from env override or -1
NSKIP=${NSKIP:-0}
echo "nevents: $nevents"
overrides="core.data_tier=full-reconstructed \
core.application.version=${DUNESW_VERSION} \
core.application.name=reco \
core.application.family=dunesw \
core.application=dunesw.reco \
"
#User specifies if they'd like to use an input file or not
if [ -n "$MCJob" ]; then
inputflag=""
else
inputflag="-i $pfn --parent $did"
fi
echo "input flag: $inputflag"
starttime=`date +"%s"`.0
# TODO -- metadata file configurable
python -m beam_job_utils run_job \
--json ${UTIL_TAR}/${JSONFILE} \
--overrides ${overrides} \
--run $run \
--subrun $subrun \
--yaml ${UTIL_TAR}/${YAMLFILE} \
--nevents $nevents \
$inputflag
endtime=`date +"%s"`.0
output_g4_files=`ls *_g4_stage2.root`
echo "Forming g4 metadata"
for FILE in ${output_g4_files[@]}
do
echo " G4 file: $FILE"
#Retrieve the key in output filename that corresponds to YAML config. stage
IFS=_ read -ra substrs <<< $(echo ${FILE} | cut -d '.' -f 1)
key_found='false'
KEY='g4_stage2'
iter=0
declare -i nsub=${#substrs[@]}-1
for substr in ${substrs[@]}; do
if [ $key_found == 'true' ]; then
KEY+=$substr
if [ $iter -lt ${nsub} ]; then
KEY+='_'
fi
fi
if [ $substr == 'stage2' ]; then
key_found='true'
fi
let iter=$iter+1
done
echo " KEY: ${KEY}"
#Retrieve the FHICL file that corresponds to the output file key
FCL=$(grep ${UTIL_TAR}/${YAMLFILE} -e "${KEY}:" -A 1 | grep -e "fcl" | cut -d ':' -f 2)
echo " FCL: ${FCL}"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "artroot" \
--app_family "dunesw" --app_name "reco" --app_version ${DUNESW_VERSION} \
--data_tier "detector-simulated" --get_events \
--campaign $CAMPAIGN \
--runs 1 --subruns ${subrun} --fcl $FCL \
-f "${JUSTIN_SCOPE}:${FILE}" -j "${FILE}.json"
if [ $? -ne 0 ]; then
echo " Error in g4 metadata"
exit 1
fi
echo " Ran successfully"
cat ${FILE}.json
done
RECO_STAGES="stage0 stage1"
subrun=0
echo "Forming reco metadata"
for STAGE in ${RECO_STAGES[@]}
do
FILE=`ls *_reco_${STAGE}.root`
echo " subrun: ${subrun} -> reco file: $FILE"
KEY="reco_${STAGE}"
echo " KEY: ${KEY}"
#Retrieve the FHICL file that corresponds to the output file key
FCL=$(grep ${UTIL_TAR}/${YAMLFILE} -e "${KEY}:" -A 1 | grep -e "fcl" | cut -d ':' -f 2)
echo " FCL: ${FCL}"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "artroot" \
--app_family "dunesw" --app_name "reco" --app_version ${DUNESW_VERSION} \
--data_tier "full-reconstructed" --get_events \
--campaign $CAMPAIGN --runs [1] --subruns [${subrun}] \
--fcl $FCL -p "${JUSTIN_SCOPE}:${output_g4_files}" \
-f "${JUSTIN_SCOPE}:${FILE}" -j "${FILE}.json"
if [ $? -ne 0 ]; then
echo " Error in reco metadata"
exit 1
fi
echo " Ran successfully"
cat ${FILE}.json
if [ "${STAGE}" == "stage0" ]; then
continue
fi
larcv_file=`ls larcv_*.root`
output_larcv_file=${FILE%.*}_larcv.root
mv ${larcv_file} ${output_larcv_file}
echo "Forming larcv metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
--app_family "dunesw" --app_name "reco" --app_version ${DUNESW_VERSION} \
--data_tier "root-tuple-virtual" -p "${JUSTIN_SCOPE}:${FILE}" \
--campaign $CAMPAIGN \
--fcl $FCL \
-f "${JUSTIN_SCOPE}:$output_larcv_file" -j "${output_larcv_file}.json"
# --parent_as_json \
if [ $? -ne 0 ]; then
echo "Error in larcv metadata"
exit 1
fi
echo "formed"
cat ${output_larcv_file}.json
let subrun=${subrun}+1
done
exitcode=$?
if [ $exitcode -ne 0 ]; then
echo "Error running. Exiting with ${exitcode}"
exit $exitcode
fi
echo "$pfn" > justin-processed-pfns.txt
