Workflow 11840, Stage 1

Workflow	11840
Campaign	248
Priority	50
Processors	1
Wall seconds	3600
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	30.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/jlechevr/fnal/11840/1	*michel.root

Environment variables

Name	Value
CAMPAIGN_META	pdhd_mc25a_michel
FHICL	pdhd_MichelAnalysis.fcl
METADATA_TAR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/89e09a0066eb78f7e471f9d7cd82854f343abf80/metadata
MODULE_TAR	/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/3f91ed35ab2e73313eb6f97d4ebb3f3d76d87dcd/exp/dune/app/users/jlechevr/sw/larsoft_v10_11_00/localProducts_larsoft_v10_11_00d00_prof_e26/protoduneana
NEVENTS	-1

Condor Class Ads

Name	Value
HAS_CVMFS_dune_osgstorage_org	true

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
1	0	0	0	0	0	0	1

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
12	0	0	0	0	0	0	0	0	6	0	6

RSEs used

Name	Inputs	Outputs
DUNE_US_FNAL_DISK_STAGE	6	0
None	0	6

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash

# Required env:
# - METADATA_TAR
# - MODULE_TAR
# - FHICL
# - CAMPAIGN_META
# - NEVENTS

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

DUNE_VERSION=${DUNE_VERSION:-v10_11_00d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"
if [ $? -ne 0 ]; then
  echo "setup dunesw $DUNE_VERSION -q $DUNE_QUALIFIER failed"
  exit 1
fi

stat $METADATA_TAR >&/dev/null
if [ $? -ne 0 ]; then
  echo "metadata.tar not found"
  exit 1
fi
PYTHONPATH=$METADATA_TAR:$PYTHONPATH

JUSTIN_PROCESSORS=${JUSTIN_PROCESSORS:-1}
export TF_NUM_THREADS=${JUSTIN_PROCESSORS}
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS}
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS}
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS}
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS}
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}

export PROTODUNEANA_DIR=${MODULE_TAR}/${DUNE_VERSION}
export PROTODUNEANA_INC=${PROTODUNEANA_DIR}/include
export PROTODUNEANA_FQ_DIR=${PROTODUNEANA_DIR}/slf7.x86_64.e26.prof
export PROTODUNEANA_LIB=${PROTODUNEANA_FQ_DIR}/lib
export CET_PLUGIN_PATH=$PROTODUNEANA_LIB:$CET_PLUGIN_PATH
export FW_SEARCH_PATH=${MODULE_TAR}:$FW_SEARCH_PATH

export FHICL_FILE_PATH=${METADATA_TAR}:${FHICL_FILE_PATH}
echo $FHICL_FILE_PATH

input_file=$($JUSTIN_PATH/justin-get-file | cut -f2 -d ' ')
if [ -z "$input_file" ]; then
  echo "justin-get-file failed"
  exit 1
fi

now=$(date -u +"%Y%m%dT%H%M%SZ")
output_file=$(basename ${input_file%.*})_${now}_michel.root
nev=${NEVENTS:--1}

start_time=$(date +"%s").0
lar -c $FHICL $input_file -T $output_file -n $nev
if [ $? -ne 0 ]; then 
  echo "lar -c failed"
  exit $?
fi
end_time=$(date +"%s").0
output_file=$(ls *michel.root)

python -m meta_maker \
  --start_time $start_time --end_time $end_time --file_format "root" \
  --app_family "dunesw" --app_name "reco" --app_version ${DUNE_VERSION} \
  --data_tier "root-tuple-virtual"\
  --campaign "$CAMPAIGN_META" \
  --fcl $FHICL \
  -f "${JUSTIN_SCOPE}:$output_file" -j "${output_file}.json"
if [ $? -ne 0 ]; then
  echo "meta_maker.py failed"
  exit $?
fi

cat ${output_file}.json
echo "$input_file" > processed_files.list