justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 2972, Stage 1

Priority50
Processors1
Wall seconds3600
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes2097152000 (2000 MiB)
Max distance for inputs100.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1https://fndcadoor.fnal.gov:2880/dune/scratch/users/twalton/ProductionDataFor2x2/v1.5.0/2025-09-18-13-47-54/data/fnal/02972/1*.root
2https://fndcadoor.fnal.gov:2880/dune/scratch/users/twalton/ProductionDataFor2x2/v1.5.0/2025-09-18-13-47-54/logs/fnal/02972/1*.log
3https://fndcadoor.fnal.gov:2880/dune/scratch/users/twalton/ProductionDataFor2x2/v1.5.0/2025-09-18-13-47-54/json/fnal/02972/1*.json

Environment variables

NameValue
CAMPAIGN_NAMENone
DATA_STREAMgenie
DATA_TIERgenie
DATA_TYPEmc
DEBUG_SUBMISSION_SCRIPT1
DETECTOR_CONFIGproto_nd
END_POSITIONNone
GENIE_ANTINU1
GENIE_BEAM_EXPOSURE1E5
GENIE_NU0
GENIE_ROCK0
GENIE_TOP_VOLUMEWorld
GENIE_USE_MAX_PATH1
GENIE_ZMIN8
JOBSCRIPT_TEST0
MX2_WORKFLOW_ID772
NEVENTS-1
RUN_CAF_MX20
RUN_CAF_PANDORA0
RUN_CAF_PANDORA_MX20
RUN_CAF_PANDORA_SPINE0
RUN_CAF_PANDORA_SPINE_MX20
RUN_CAF_SPINE0
RUN_CAF_SPINE_MX20
RUN_PERIODrun1
SAND_GEOMsand_straw
SPINE_WORKFLOW_ID1
START_POSITIONNone
TWOBYTWO_RELEASEv1.5.0
USERtwalton

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
1000100000

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
15001005000000

RSEs used

NameInputsOutputs
MONTECARLO100
None010

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#
# 	This script for running the genie workflow 
#            It is based on the 2x2 development (https://github.com/DUNE/2x2_sim/blob/feature_spine_on_data/run-genie/run_genie.sh)
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++


#++++++++++++++++++++++++++++++++++++++++
# the script with common functions
#++++++++++++++++++++++++++++++++++++++++
#source /exp/dune/app/users/twalton/2x2ProdWorkspace/v1.5.0/ND_Production/toolbox/scripts/NDUtilsForJustin.sh
source /cvmfs/dune.opensciencegrid.org/dunend/2x2/releases/${TWOBYTWO_RELEASE}/ndlar_prod_scripts/ND_Production/toolbox/scripts/NDUtilsForJustin.sh


#++++++++++++++++++++++++++++++++++++++++++
# sanity check
#++++++++++++++++++++++++++++++++++++++++++
if [[ "${DATA_TIER}" != "genie" ]]; then
   echo -e "This script [$(basename $BASH_SOURCE)] submits the GENIE event generator jobs. Please see the help menu. The data tier is not defined correctly." 
   exit 0
fi


#++++++++++++++++++++++++++++++++++++++++
# Begin JustIN
#++++++++++++++++++++++++++++++++++++++++
justin_begin_of_job_commands


#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# containers to store the generated files
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
CREATED_FILES=()
WORKFLOW=()


#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# get the genie primary input parameters
#
#  The seed is based on justIN physical file name
#  For a monte carlo event, justIN defines the physical file name as a integer with leading zeros
#  Therefore, the first file will be defined as 
#  DID_PFN_RSE=monte-carlo-000001-000001 000001 MONTECARLO
#  This will give a seed=1
#
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
IN=$((10#$INPUT_FILE)) 

export USE_MAXPATH=0
export SEED=$((IN + 0))
export RUN=$((SEED - 1))
export GENIE_PRODUCTION_TUNE=AR23_20i_00_000
export GENIE_XSEC_FILE=${CVMFS_STASHCACHE_DIR}/Flux/2x2EventGeneration/XSec/numi/genie/v3r4p00/AR2320i00000-k250-e1000/gxspl-NUsmall.xml
echo -e "\tseed is [$SEED], and run number is [$RUN]" 2>&1 | tee -a $envlog


if [[ "${DETECTOR_CONFIG}" == "proto_nd" ]]; then
   
   if [[ "${GENIE_ANTINU}" == "1" ]]; then
      export DK2NU_DIR=${CVMFS_STASHCACHE_DIR}/Flux/g4numi/20220409/QGSP_BERT/MediumEnergyNewTarget_OnAxis/antineutrino/flux
   elif [[ "${GENIE_NU}" == "1" ]]; then
      export DK2NU_DIR=${CVMFS_STASHCACHE_DIR}/Flux/g4numi/20210920/QGSP_BERT/MediumEnergyNewTarget_OnAxis/neutrino/flux
   fi

   if [[ "${GENIE_ROCK}" == "1" ]]; then
      export FLUX_DET_LOCATION="MiniRun5-Rock"
      export GENIE_DETECTOR_GEOM=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/geometry/Merged2x2MINERvA_v4/Merged2x2MINERvA_v4_justRock.gdml
      export MAX_PATH_XML_FILE=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/maxpath/Merged2x2MINERvA_v4_justRock.AR23_20i_00_000.maxpath.xml
   else 
      export FLUX_DET_LOCATION="MiniRun5-Nu"
      export GENIE_DETECTOR_GEOM=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/geometry/Merged2x2MINERvA_v4/Merged2x2MINERvA_v4_noRock.gdml
      export MAX_PATH_XML_FILE=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/maxpath/Merged2x2MINERvA_v4_noRock.AR23_20i_00_000.maxpath.xml
   fi

elif [[ "${DETECTOR_CONFIG}" == "ndlar" ]]; then

   if [[ "${GENIE_ANTINU}" == "1" ]]; then
      export DK2NU_DIR=${CVMFS_STASHCACHE_DIR}/Flux/g4lbne/v3r5p10/QGSP_BERT/OfficialEngDesignSept2021_OnAxis/antineutrino/flux 
   elif [[ "${GENIE_NU}" == "1" ]]; then
      export DK2NU_DIR=${CVMFS_STASHCACHE_DIR}/Flux/g4lbne/v3r5p10/QGSP_BERT/OfficialEngDesignSept2021_OnAxis/neutrino/flux
   fi

   if [[ "${GENIE_ROCK}" == "1" ]]; then
      export FLUX_DET_LOCATION="DUNENDROCK"
      if [[ "${SAND_GEOM}" == "sand_drift" ]]; then
         export GENIE_DETECTOR_GEOM=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/geometry/anti_fiducial_nd_hall_with_lar_tms_sand_drift1_v2025.08.11.gdml
	 export MAX_PATH_XML_FILE=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/maxpath/anti_fiducial_nd_hall_with_lar_tms_sand_drift1_v2025.08.11.AR23_20i_00_000.maxpath.xml
      elif [[ "${SAND_GEOM}" == "sand_straw" ]]; then
         export GENIE_DETECTOR_GEOM=${CVMFS_WORKKING_DIR}/ndlar_prod_scripts/ND_Production/geometry/anti_fiducial_nd_hall_with_lar_tms_sand_stt1_v2025.08.11.gdml
	 export MAX_PATH_XML_FILE=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/maxpath/anti_fiducial_nd_hall_with_lar_tms_sand_stt1_v2025.08.11.AR23_20i_00_000.maxpath.xml
      fi

   else 
      export FLUX_DET_LOCATION="DUNEND"
      if [[ "${SAND_GEOM}" == "sand_drift" ]]; then
     	 export GENIE_DETECTOR_GEOM=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/geometry/nd_hall_with_lar_tms_sand_drift1_v2025.08.11.gdml
	 export MAX_PATH_XML_FILE=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/maxpath/nd_hall_with_lar_tms_sand_drift1_v2025.08.11.AR23_20i_00_000.maxpath.xml
      elif [[ "${SAND_GEOM}" == "sand_straw" ]]; then
         export GENIE_DETECTOR_GEOM=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/geometry/nd_hall_with_lar_tms_sand_stt1_v2025.08.11.gdml
	 export MAX_PATH_XML_FILE=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/maxpath/nd_hall_with_lar_tms_sand_stt1_v2025.08.11.AR23_20i_00_000.maxpath.xml
      fi
   fi

else
   echo -e "Cannot determine the parameters for [${DETECTOR_CONFIG}], which is needed for the flux driver. Please see the help menu." 2>&1 | tee -a $envlog
fi


#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# get the flux file based on the seed (run number)
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
INDEX=${RUN}
DK2NU_FILES=("$DK2NU_DIR"/*.dk2nu*)
export DK2NU_FILE=${DK2NU_FILES[${INDEX}]}
echo -e "\tUsing the index [${INDEX}] to get the flux file [${DK2NU_FILE}]" 2>&1 | tee -a $envlog


#+++++++++++++++++++++++++++++++++++++++++++++++++++
# create an output file
#+++++++++++++++++++++++++++++++++++++++++++++++++++
BEAM="NuMI"
if [[ "${DETECTOR_CONFIG}" == "ndlar" ]]; then
   BEAM="LBNF"
fi

HORN="FHC"
if [[ "${GENIE_ANTINU}" == "1" ]]; then
    HORN="RHC"
fi

CHANNEL="_"
if [[ -n "${GENIE_EVENT_GENERATOR_LIST}" ]]; then
     IFS=',' read -a channels <<< "${GENIE_EVENT_GENERATOR_LIST}"
     CHANNEL=$( IFS=$'_'; echo "${channels[*]}" )
     CHANNEL="_${CHANNEL}_"
fi

TIMESTAMP=$(date +'%Y_%m_%d_%H_%M_%S')
if [[ "${GENIE_ROCK}" == "1" ]]; then
   TIMESTAMP="Rock_$(date +'%Y_%m_%d_%H_%M_%S')"
fi

export GENIE_OUTPUT_FILE_BASE="${BEAM}_${HORN}${CHANNEL}${TIMESTAMP}_${GENIE_VERSION}_${GENIE_BEAM_EXPOSURE}.${INPUT_FILE}"
export GENIE_OUTPUT_FILE="${GENIE_OUTPUT_FILE_BASE}.GHEP.root"
export GTRAC_OUTPUT_FILE="${GENIE_OUTPUT_FILE_BASE}.GTRAC.root"

echo -e "\tThe output GENIE file names are [${GENIE_OUTPUT_FILE}, ${GTRAC_OUTPUT_FILE}]" 2>&1 | tee -a $envlog


#++++++++++++++++++++++++++++++++++++++++++++++
# genie arguments
#++++++++++++++++++++++++++++++++++++++++++++++
ARGS_GEVGEN_FNAL=( \
    -e "$GENIE_BEAM_EXPOSURE" \
    -f "$DK2NU_FILE,$FLUX_DET_LOCATION" \
    -g "$GENIE_DETECTOR_GEOM" \
    -r "$RUN" \
    -L cm -D g_cm3 \
    --cross-sections "$GENIE_XSEC_FILE" \
    --tune "$GENIE_PRODUCTION_TUNE" \
    --seed "$SEED" \
    -o "$GENIE_OUTPUT_FILE_BASE" \
    )


#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# get the optional input parameters
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
[ "${GENIE_USE_MAX_PATH}" == "1" ] && ARGS_GEVGEN_FNAL+=( -m "$MAX_PATH_XML_FILE" )
[ -n "${GENIE_TOP_VOLUME}" ] && ARGS_GEVGEN_FNAL+=( -t "$GENIE_TOP_VOLUME" )
[ -n "${GENIE_ZMIN}" ] && ARGS_GEVGEN_FNAL+=( -z "$GENIE_ZMIN" )
[ -n "${GENIE_EVENT_GENERATOR_LIST}" ] && ARGS_GEVGEN_FNAL+=( --event-generator-list "$GENIE_EVENT_GENERATOR_LIST" )


#+++++++++++++++++++++++++++++++++++++++
# Execute the workflow
#++++++++++++++++++++++++++++++++++++++++
export PASSED_WORKFLOW=1
(
    echo -e "\nSetup the dune software (genie) software [setup dunesw ${DUNESW_VERSION} -q ${DUNESW_QUALIFIER}]." 2>&1 | tee -a $envlog

    source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
    setup dunesw ${DUNESW_VERSION} -q ${DUNESW_QUALIFIER}
    export GDK2NUFLUXXML=${CVMFS_WORKING_DIR}/ndlar_prod_scripts/ND_Production/run-genie/flux/GNuMIFlux.xml

    echo -e "\nExecuting the genie workflow for data stream [${DATA_STREAM}] and input file [${INPUT_FILE}]" 2>&1 | tee -a $envlog
    echo -e "\tRunning the genie command [gevgen_fnal ${ARGS_GEVGEN_FNAL[@]}]" 2>&1 | tee -a $envlog

    gevgen_fnal ${ARGS_GEVGEN_FNAL[@]}

    if [ ! -f ${GENIE_OUTPUT_FILE_BASE}.${RUN}.ghep.root ]; then
       export PASSED_WORKFLOW=0
       echo -e "FATAL::The file [${GENIE_OUTPUT_FILE_BASE}.${RUN}.ghep.root] does not exist! Will not continue." 2>&1 | tee -a $envlog
       exit 1
    else
       mv ${GENIE_OUTPUT_FILE_BASE}.${RUN}.ghep.root ${GENIE_OUTPUT_FILE}
    fi 

    echo -e "\nConverting the native GENIE files to a format easily recognize by ROOT" 2>&1 | tee -a $envlog
    echo -e "\tRunning the genie command [gntpc -i \"${GENIE_OUTPUT_FILE}\" -f rootracker  -o \"${GTRAC_OUTPUT_FILE}\"]\n\n" 2>&1 | tee -a $envlog
    gntpc -i "${GENIE_OUTPUT_FILE}" -f rootracker  -o "${GTRAC_OUTPUT_FILE}"

    if [ ! -f ${GTRAC_OUTPUT_FILE} ]; then
       export PASSED_WORKFLOW=0
       echo -e "FATAL::The file [${GTRAC_OUTPUT_FILE}] does not exist! Will not continue." 2>&1 | tee -a $envlog
       exit 1
    fi
)

if [[ "${PASSED_WORKFLOW}" == "0" ]]; then
   exit 1
fi


#++++++++++++++++++++++++++++++++++++++++++++++++
# set information needed for metadata
#++++++++++++++++++++++++++++++++++++++++++++++++
CREATED_FILES+=("${GENIE_OUTPUT_FILE}")
CREATED_FILES+=("${GTRAC_OUTPUT_FILE}")

WORKFLOW+=("genie")
WORKFLOW+=("genie_converter")

export APPLICATION_DATA_TIER="genie-event-generator"

export NAMESPACE="neardet-2x2-lar"
if [[ "${DETECTOR_CONFIG}" == "ndlar" ]]; then
   export NAMESPACE="neardet-lar"
fi


#++++++++++++++++++++++++++++++++++++++++
# create metadata json file
#++++++++++++++++++++++++++++++++++++++++
create_metadata_file mc genie


#+++++++++++++++++++++++++++++++++++++++
# End of justin job running
#+++++++++++++++++++++++++++++++++++++++
justin_end_of_job_commands



##############################################
#
# END OF RUNNING GENIE PRODUCTION JOBS
#
##############################################

exit 0
justIN time: 2025-09-18 20:14:08 UTC       justIN version: 01.05.00