Workflow 8686, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	25165824000 (24000 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Glasgow, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/gabrielb/fnal/08686/1	cosmics_detsim_*.root

Environment variables

Name	Value
FCL_FILE	standard_g4_protodunehd.fcl
INPUT_TAR_DIR_LOCAL	/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/d7fd4eedbc5e8da23446e3b70e9f7be7f3f3cc43
NUM_EVENTS	10

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
100	0	0	0	0	100	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
152	0	0	0	0	101	0	44	2	5	0	0

RSEs used

Name	Inputs	Outputs
MONTECARLO	102	0
None	0	101

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
NL_SURFsara	1	0

Jobscript

#!/bin/bash

: <<'EOF'

To submit to tarball

1) tar cvz coisa.tar coisa_original ou tar -cvf my_fcl.tar gen_cosmics.fcl gen_cosmics_g4.fcl
2) setup justin
3) justin time
4) justin get-token
5) INPUT_TAR_DIR_LOCAL=`justin-cvmfs-upload coisa.tar`---> wait a little
6) ls -l $INPUT_TAR_DIR_LOCAL

To run this script with justin:

USERF=$USER
FNALURL='https://fndcadoor.fnal.gov:2880/dune/scratch/users'

justin simple-workflow \
  --monte-carlo 100 \
  --jobscript-git GBotogoske/my_grid/my_submit.jobscript:main \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \
  --scope usertests --lifetime-days 2 \
  --rss-mib 24000 \
  --output-pattern "cosmics_detsim_*.root:${FNALURL}/${USERF}" 

or with gpu

justin simple-workflow \
  --monte-carlo 1 --gpu \
  --jobscript-git GBotogoske/my_grid/my_submit.jobscript:main \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \
  --scope usertests --lifetime-days 2 \
  --rss-mib 24000 \
  --output-pattern "cosmics_g4.root:${FNALURL}/${USERF}" 
  
To test:

justin-test-jobscript \
  --monte-carlo 1 \
  --jobscript jobscript_local.jobscript  \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \

To stop:
justin finish-workflow --workflow-id <ID_DO_WORKFLOW>


EOF

# Set defaults
FCL_FILE_GEN=$INPUT_TAR_DIR_LOCAL/gen_cosmics.fcl
FCL_FILE2=$INPUT_TAR_DIR_LOCAL/gen_cosmics_g4.fcl
DUNE_VERSION=${DUNE_VERSION:-v10_08_00d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
#INPUT_FILE=${INPUT_FILE:-$INPUT_TAR_DIR_LOCAL/cosmics_gen.root}

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
pfn=`echo $DID_PFN_RSE | cut -f2 -d' '`

first_file="cosmics_gen_${pfn}.root"
second_file="cosmics_g4_${pfn}.root"
third_file="cosmics_detsim_${pfn}.root"


# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

# Define number of events if set
if [ -n "$NUM_EVENTS" ]; then
  events_option="-n $NUM_EVENTS"
fi

# Run LArSoft
#lar -c prod_cosmics_protodunehd.fcl -o data_gen.root -n 10
#lar -c "$FCL_FILE" -s "$INPUT_FILE" -o data_g4.root $events_option 
(
export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
echo "$LD_PRELOAD"

lar -c $FCL_FILE_GEN -o "$first_file"
lar -c $FCL_FILE -s "$first_file" -o "$second_file"
lar -c standard_detsim_protodunehd.fcl -s "$second_file" -o "$third_file"
  
)

larExit=$?
echo "lar exit code $larExit"
echo "pfn $pfn"
echo "outfile $third_file"

if [ $larExit -eq 0 ]; then
  echo "$pfn" > justin-processed-pfns.txt
  jobscriptExit=0
else
  jobscriptExit=1
fi

#tar -zcf "$(echo "${JUSTIN_JOBSUB_ID}.logs.tgz" | sed 's/@/_/g')" *.log
exit $jobscriptExit