Workflow 222, Stage 1

Priority	50
Processors	1
Wall seconds	18000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4193255424 (3999 MiB)
Max distance for inputs	30.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	vd-protodune-det-reco
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio vd-protodune-det-reco:fnal-w222s1p1	*_keepup.root	7776000	False

Environment variables

Name	Value
FHICL_TAR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/f74d2cb0f3847d6487a45a87df055df678960ad1
INPUT_TAR_DIR_LOCAL	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/d672b14eb7839876bbd543a30533c5f2278d8a12
METADATA_DIR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/20943968d0063a8f11ac6a4062b5ad0c8392b138

Condor Class Ads

Name	Value
HAS_CVMFS_dune_osgstorage_org	true

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
815	315	344	0	0	0	0	156

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
1152	0	0	0	0	0	0	0	162	990	0	0

RSEs used

Name	Inputs	Outputs
DUNE_CERN_EOS	1152	0

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
NL_SURFsara	36	0
UK_RAL-PPD	32	0
UK_RAL-Tier1	22	0
UK_QMUL	18	0
NL_NIKHEF	16	0
CZ_FZU	11	0
UK_Manchester	8	0
UK_Oxford	4	0
UK_Lancaster	4	0
UK_Durham	4	0
FR_CCIN2P3	3	0
ES_PIC	3	0

Jobscript

#!/bin/bash
#

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -n "${DUNESW_DIR}" ]; then
  stat ${DUNESW_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat dunesw dir"
    exit 1
  fi

  export PRODUCTS=$DUNESW_DIR:$PRODUCTS
fi

if [ -n "${LARRECO_DIR}" ]; then
  stat ${LARRECO_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat larreco dir"
    exit 1
  fi

  export PRODUCTS=$LARRECO_DIR:$PRODUCTS
fi


if [ -n "${DUNEPROTOTYPES_DIR}" ]; then
  stat ${DUNEPROTOTYPES_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat dunedetdataformats dir"
    exit 1
  fi

  export PRODUCTS=$DUNEPROTOTYPES_DIR:$PRODUCTS
fi

export PRODUCTS=$INPUT_TAR_DIR_LOCAL:${PRODUCTS}

echo "PRODUCTS $PRODUCTS"

DUNE_TAG=v10_08_01d00
#Setup recent lar software suite
DUNE_VERSION=${DUNE_VERSION:-${DUNE_TAG}}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit 1
fi

export DUNESW_DIR=${INPUT_TAR_DIR_LOCAL}/dunesw/${DUNE_VERSION}
export DUNESW_FQ_DIR=${DUNESW_DIR}/slf7.x86_64.e26.prof
export DUNESW_LIB=${DUNESW_FQ_DIR}/lib

export DUNECORE_DIR=${INPUT_TAR_DIR_LOCAL}/dunecore/${DUNE_VERSION}
export DUNECORE_LIB=${DUNECORE_DIR}/include
export DUNECORE_FQ_DIR=${DUNECORE_DIR}/slf7.x86_64.e26.prof
export DUNECORE_LIB=${DUNECORE_FQ_DIR}/lib

export DUNEPROTOTYPES_DIR=${INPUT_TAR_DIR_LOCAL}/duneprototypes/${DUNE_VERSION}
export DUNEPROTOTYPES_INC=${DUNEPROTOTYPES_DIR}/include
export DUNEPROTOTYPES_FQ_DIR=${DUNEPROTOTYPES_DIR}/slf7.x86_64.e26.prof
export DUNEPROTOTYPES_LIB=${DUNEPROTOTYPES_FQ_DIR}/lib

export FHICL_FILE_PATH=${DUNESW_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${DUNEPROTOTYPES_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${INPUT_TAR_DIR_LOCAL}:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${FHICL_TAR}:${FHICL_FILE_PATH}

export FW_SEARCH_PATH=${INPUT_TAR_DIR_LOCAL}/config_data:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${INPUT_TAR_DIR_LOCAL}:$FW_SEARCH_PATH
export FW_SEARCH_PATH=${FHICL_TAR}:$FW_SEARCH_PATH

echo "FHICL_FILE_PATH: ${FHICL_FILE_PATH}"


if [ -n "${USE_INPUT_FCL}" ]; then
  
  if [ -z ${INPUT_DIR} ]; then
    echo "Error, INPUT_DIR is undefined but user requested USE_INPUT_FCL"
    exit 1
  fi

  stat ${INPUT_DIR}
  if [ $? -ne 0 ]; then
    echo "Failed to stat input dir. Exiting safely"
    exit 0
  fi

  FHICL_FILE_PATH=${INPUT_DIR}:${FHICL_FILE_PATH}
  echo "FCL PATH: $FHICL_FILE_PATH"
fi

if [ -n "${METADATA_DIR}" ]; then
  stat ${METADATA_DIR}
  if [ $? -ne 0 ]; then
    echo "failed to stat metadata dir"
  fi

  echo "metadata dir contents:"
  ls $METADATA_DIR
  PYTHONPATH=${METADATA_DIR}:$PYTHONPATH
fi

# Temporary fix to get the propoer PDS map
export FHICL_FILE_PATH=${METADATA_DIR}:${FHICL_FILE_PATH}
export FW_SEARCH_PATH=${METADATA_DIR}:${FW_SEARCH_PATH}

FCL1=${FCL1:-"standard_reco_stage1_protodunevd_keepup_all.fcl"}
#FCL1=${FCL1:-"standard_reco_stage1_protodunevd_keepup.fcl"}
#FCL1=${FCL1:-"standard_reco_protodunevd_keepup.fcl"}
echo "FCL1 dump:" ${FCL1}
fhicl-dump ${FCL1}
if [ $? -ne 0 ]; then
  echo "fhicl-dump ${FCL1} failed"
  exit 1
fi

# not yet setup
#FCL2=${FCL2:-"protodunevd_data_reco_stage2_calibration.fcl"}
#echo "FCL2 dump:" ${FCL2}
#fhicl-dump ${FCL2}
#if [ $? -ne 0 ]; then
#  echo "fhicl-dump ${FCL2} failed"
#  exit 1
#fi


echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "printing env"
env

echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "pfn: ${pfn}"
echo "did: ${did}"
now=$(date -u +"%Y%m%dT%H%M%SZ")

nevents=${NEVENTS:--1}
nevents=1

extra_line=""
if [ -n "${SKIPFCL2}" ]; then
  jobsub_id=`echo ${JUSTIN_JOBSUB_ID:-1.1@1} | cut -f1 -d'@' | sed -e"s/\./_/"`
  extra_line="-T pdvd_${jobsub_id}_${JUSTIN_WORKFLOW_ID}_${now}_decoder.root"
fi
echo "Running reco stage1"
touch reco.log
starttime=`date +"%s"`.0
LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so lar \
    -c ${FCL1} \
    -n ${nevents} \
    ${extra_line} ${pfn} #>reco.log 2>&1
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco1"
  cat reco.log
  exit $larExit
fi

if [ -n "${SKIPFCL2}" ]; then
  echo "$pfn" > justin-processed-pfns.txt
  exit 0
fi

#output_stage1_file=`ls *stage1.root`

starttime=`date +"%s"`.0
#lar -c ${FCL2} \
#    $output_stage1_file #>reco.log 2>&1
#larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in reco2"
  cat reco.log
  exit $larExit
fi



output_reco_file=`ls *keepup.root`
output_mr_file=`ls *keepup_hists.root`

#new_mr_file=`echo $output_reco_file | sed -e "s/reco/reco_hists/"`
#mv $output_mr_file $new_mr_file
#output_mr_file=$new_mr_file

echo "Output files:"
echo "\tReco: ${output_reco_file}"
echo "\tHists: ${output_mr_file}"

echo "Forming reco metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "artroot" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "full-reconstructed" --get_events -p "$did" \
                     --campaign "vd-protodune-reco-keepup-v0" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_reco_file" -j "${output_reco_file}.json"
if [ $? -ne 0 ]; then
  echo "Error in reco metadata"
  exit 1
fi
echo "Ran successfully"
## TODO -- CHECK
cat ${output_reco_file}.json

echo "Forming hist metadata"
python -m meta_maker --start_time $starttime --end_time $endtime --file_format "root" \
                     --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
                     --data_tier "root-tuple-virtual" -p "$did" \
                     --campaign "vd-protodune-reco-keepup-v0" \
                     --fcl $FCL1 \
                     -f "${JUSTIN_SCOPE}:$output_mr_file" -j "${output_mr_file}.json"
 #--parent_as_json \
if [ $? -ne 0 ]; then
  echo "Error in hist metadata"
  exit 1
fi
echo "formed"
cat ${output_mr_file}.json



echo "$pfn" > justin-processed-pfns.txt