Workflow 11997, Stage 1

Workflow	11997
Campaign	395
Priority	50
Processors	1
Wall seconds	3600
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	30.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio usertests:fnal-w11997s1p1	*.root	7776000	False

Environment variables

Name	Value
FHICL_FILE	standard_reco_stage0_protodunevd_beam.fcl
METADATA_DIR	/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/978a6b912ee9494400d998aa8a7cabe34b7d26cf
NEVENTS	-1
WIRECELL_CFG	/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/b82577ef23a78703ca6de4597541c2bbbf36f3c9
WIRECELL_TOOLKIT	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/08ba6deac42974c2d3188156ca9518d0f2d7ce1a

Condor Class Ads

Name	Value
HAS_CVMFS_dune_osgstorage_org	true

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
1	0	0	0	0	1	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
4	0	0	0	0	2	0	0	2	0	0	0

RSEs used

Name	Inputs	Outputs
DUNE_CERN_EOS	2	0
DUNE_US_FNAL_DISK_STAGE	0	2

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
ES_PIC	1	0

Jobscript

#!/bin/bash
#

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

DUNE_TAG=v10_16_00d00
#Setup recent lar software suite
DUNE_VERSION=${DUNE_VERSION:-${DUNE_TAG}}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit 1
fi

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

export WIRECELL_PATH=${WIRECELL_CFG}/wire-cell-cfg:$WIRECELL_PATH
export LD_LIBRARY_PATH=/cvmfs/larsoft.opensciencegrid.org/products/jsonnet/v0_17_0a/Linux64bit+3.10-2.17-e20-prof/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=${WIRECELL_TOOLKIT}/wire-cell-toolkit/install/lib:$LD_LIBRARY_PATH
export FHICL_FILE_PATH=${METADATA_DIR}:${FHICL_FILE_PATH}

#FCL1=${FCL1:-"standard_reco_stage1_protodunevd_offline_reproc.fcl"}
FCL1=${FCL1:-${FHICL_FILE}}

echo "FCL1 dump:" ${FCL1}
fhicl-dump ${FCL1}
if [ $? -ne 0 ]; then
  echo "fhicl-dump ${FCL1} failed"
  exit 1
fi

echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "printing env"
env

echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "pfn: ${pfn}"
echo "did: ${did}"
now=$(date -u +"%Y%m%dT%H%M%SZ")

#output_file=$(basename ${pfn%.*})_${now}_singlehit.root
#echo "output_file: $output_file"

nevents=${NEVENTS:--1}

extra_line=""
if [ -n "${SKIPFCL2}" ]; then
  jobsub_id=`echo ${JUSTIN_JOBSUB_ID:-1.1@1} | cut -f1 -d'@' | sed -e"s/\./_/"`
fi

echo "Running reco stage1"
touch reco.log
starttime=`date +"%s"`.0
if [[ "${FCL1}" == *"stage0"* ]]; then
    LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so lar -c ${FCL1} \
    -n ${nevents} \
    ${pfn} #>reco.log 2>&1
else
    lar -c ${FCL1} \
    -n ${nevents} \
    ${pfn} #>reco.log 2>&
fi
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco1"
  cat reco.log
  exit $larExit
fi

if [ -n "${SKIPFCL2}" ]; then
  echo "$pfn" > justin-processed-pfns.txt
  exit 0
fi

#output_stage1_file=`ls *stage1.root`

#starttime=`date +"%s"`.0
#lar -c ${FCL2} \
#    $output_stage1_file #>reco.log 2>&1
#larExit=$?
#endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco2"
  cat reco.log
  exit $larExit
fi

if [ $(ls *_reproc.root | wc -l) -eq 0 ];
then
  output_reco_file=`ls *_beam.root`
else
  output_reco_file=`ls *_reproc.root`
fi

echo "Output files:"
echo "\tReco: ${output_reco_file}"
#echo "\tHists: ${output_mr_file}"

echo "Forming reco metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "artroot" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "full-reconstructed" --get_events -p "$did" \
                     --campaign "vd-protodune-det-reco-test" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_reco_file" -j "${output_reco_file}.json"
if [ $? -ne 0 ]; then
  echo "Error in reco metadata"
  exit 1
fi
echo "Ran successfully"
cat ${output_reco_file}.json


#echo "Forming hist metadata"
#python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
#                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
#                     --data_tier "root-tuple-virtual" -p "$did" \
#                     --campaign "hd-protodune-sim-reco" \
#                     --fcl $FCL1 \
#                     -f "${JUSTIN_SCOPE}:$output_mr_file" -j "${output_mr_file}.json"
# --parent_as_json \
#if [ $? -ne 0 ]; then
#  echo "Error in hist metadata"
#  exit 1
#fi
#echo "formed"
#cat ${output_mr_file}.json

echo "$pfn" > justin-processed-pfns.txt