justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Jobsub ID 243668.15@dunegpschedd02.fnal.gov

Jobsub ID243668.15@dunegpschedd02.fnal.gov
Workflow ID10174
Stage ID1
User nameemuldoon@fnal.gov
RequestedProcessors1
GPUNo
RSS bytes2097152000 (2000 MiB)
Wall seconds limit80000 (22 hours)
Submitted time2025-11-13 23:24:53
SiteUS_FNAL-FermiGrid
EntryFNAL_GPGrid_ce04_mcore_op_duneonly
Last heartbeat2025-11-14 01:05:33
From worker nodeHostnamedunegli-7743887-0-fnpc23019.fnal.gov
cpuinfoAMD EPYC 7543 32-Core Processor
OS releaseScientific Linux release 7.9 (Nitrogen)
Processors1
RSS bytes2097152000 (2000 MiB)
Wall seconds limit172800 (48 hours)
GPU
Inner Apptainer?True
Job stateoutputting_failed
Started2025-11-14 00:53:31
Input filesmonte-carlo-010174-000011
JobscriptExit code0
Real time9m (583s)
CPU time0m (7s = 1%)
Max RSS bytes453914624 (432 MiB)
Outputting started2025-11-14 01:03:15
Output files
Finished2025-11-14 01:05:33
List job events     (HTCondor job logs unavailable)

Jobscript log (last 10,000 characters)

93331Z_gen_g4_detsim_hitreco__20240223T220518Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "1Q6jR5PTRe2tU6RF",
      "name": "nu_dune10kt_1x2x6_1421_992_20230827T094115Z_gen_g4_detsim_hitreco__20240223T214846Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "0AchqXKGSGmYELaD",
      "name": "nu_dune10kt_1x2x6_1425_145_20230827T183303Z_gen_g4_detsim_hitreco__20240223T214931Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "0kCyv9HDT1S4dys3",
      "name": "nu_dune10kt_1x2x6_1425_19_20230827T183332Z_gen_g4_detsim_hitreco__20240223T221539Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "0lHstrd5REqMdT4l",
      "name": "nu_dune10kt_1x2x6_1425_355_20230827T183309Z_gen_g4_detsim_hitreco__20240223T225246Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "1aAmMdvmSca7GD39",
      "name": "nu_dune10kt_1x2x6_1425_389_20230827T183324Z_gen_g4_detsim_hitreco__20240223T214923Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "2w60P6XxSNSorVGT",
      "name": "nu_dune10kt_1x2x6_1427_511_20230827T213353Z_gen_g4_detsim_hitreco__20240223T220439Z_reco2_ana.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "0mjkzLX7R0eNK5Dp",
      "name": "nu_dune10kt_1x2x6_1427_617_20230827T213358Z_gen_g4_detsim_hitreco__20240223T214941Z_reco2_ana.root",
      "namespace": "fardet-hd"
    }
  ],
  "inputs": [
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/86/11/nu_dune10kt_1x2x6_1421_92_20230827T093331Z_gen_g4_detsim_hitreco__20240223T220518Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/bd/1c/nu_dune10kt_1x2x6_1421_992_20230827T094115Z_gen_g4_detsim_hitreco__20240223T214846Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/16/a5/nu_dune10kt_1x2x6_1425_145_20230827T183303Z_gen_g4_detsim_hitreco__20240223T214931Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/0d/a9/nu_dune10kt_1x2x6_1425_19_20230827T183332Z_gen_g4_detsim_hitreco__20240223T221539Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/bc/9d/nu_dune10kt_1x2x6_1425_355_20230827T183309Z_gen_g4_detsim_hitreco__20240223T225246Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/c8/92/nu_dune10kt_1x2x6_1425_389_20230827T183324Z_gen_g4_detsim_hitreco__20240223T214923Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/3b/2a/nu_dune10kt_1x2x6_1427_511_20230827T213353Z_gen_g4_detsim_hitreco__20240223T220439Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/f7/0b/nu_dune10kt_1x2x6_1427_617_20230827T213358Z_gen_g4_detsim_hitreco__20240223T214941Z_reco2_ana.root"
  ],
  "outputs": [
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root"
    },
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_files_20251113T231223_f1.txt",
      "metadata": {
        "core.file_format": "text"
      },
      "rename": "files.txt"
    },
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f1.txt",
      "metadata": {
        "core.file_format": "text"
      }
    }
  ],
  "settings": {
    "streaming": false,
    "method": "multi-output",
    "cmd": "hadd -f {output} {inputs} && echo \"{inputs}\" | tr \" \" \"\\n\" > files.txt && wc -l files.txt | cut -d\" \" -f1 > {outputs[2]}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root
hadd compression setting for all output: 1
hadd Source file 1: tmp/nu_dune10kt_1x2x6_1421_92_20230827T093331Z_gen_g4_detsim_hitreco__20240223T220518Z_reco2_ana.root
hadd Source file 2: tmp/nu_dune10kt_1x2x6_1421_992_20230827T094115Z_gen_g4_detsim_hitreco__20240223T214846Z_reco2_ana.root
hadd Source file 3: tmp/nu_dune10kt_1x2x6_1425_145_20230827T183303Z_gen_g4_detsim_hitreco__20240223T214931Z_reco2_ana.root
hadd Source file 4: tmp/nu_dune10kt_1x2x6_1425_19_20230827T183332Z_gen_g4_detsim_hitreco__20240223T221539Z_reco2_ana.root
hadd Source file 5: tmp/nu_dune10kt_1x2x6_1425_355_20230827T183309Z_gen_g4_detsim_hitreco__20240223T225246Z_reco2_ana.root
hadd Source file 6: tmp/nu_dune10kt_1x2x6_1425_389_20230827T183324Z_gen_g4_detsim_hitreco__20240223T214923Z_reco2_ana.root
hadd Source file 7: tmp/nu_dune10kt_1x2x6_1427_511_20230827T213353Z_gen_g4_detsim_hitreco__20240223T220439Z_reco2_ana.root
hadd Source file 8: tmp/nu_dune10kt_1x2x6_1427_617_20230827T213358Z_gen_g4_detsim_hitreco__20240223T214941Z_reco2_ana.root
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/flashmatchAna
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/emshowershweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/pandoraeff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/pandorashweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/pmtrackeff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/pmtracktceff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/pmtrajfiteff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/pmtrajfittceff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/runcheckereco
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/trajclustershweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root:/analysistree
Making local copy of input files in tmp:
  Copying nu_dune10kt_1x2x6_1421_92_20230827T093331Z_gen_g4_detsim_hitreco__20240223T220518Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_992_20230827T094115Z_gen_g4_detsim_hitreco__20240223T214846Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1425_145_20230827T183303Z_gen_g4_detsim_hitreco__20240223T214931Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1425_19_20230827T183332Z_gen_g4_detsim_hitreco__20240223T221539Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1425_355_20230827T183309Z_gen_g4_detsim_hitreco__20240223T225246Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1425_389_20230827T183324Z_gen_g4_detsim_hitreco__20240223T214923Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1427_511_20230827T213353Z_gen_g4_detsim_hitreco__20240223T220439Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1427_617_20230827T213358Z_gen_g4_detsim_hitreco__20240223T214941Z_reco2_ana.root
Copied 8 files
Merging 8 files into fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root using method multi-output
hadd -f fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root tmp/nu_dune10kt_1x2x6_1421_92_20230827T093331Z_gen_g4_detsim_hitreco__20240223T220518Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_992_20230827T094115Z_gen_g4_detsim_hitreco__20240223T214846Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_145_20230827T183303Z_gen_g4_detsim_hitreco__20240223T214931Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_19_20230827T183332Z_gen_g4_detsim_hitreco__20240223T221539Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_355_20230827T183309Z_gen_g4_detsim_hitreco__20240223T225246Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_389_20230827T183324Z_gen_g4_detsim_hitreco__20240223T214923Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1427_511_20230827T213353Z_gen_g4_detsim_hitreco__20240223T220439Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1427_617_20230827T213358Z_gen_g4_detsim_hitreco__20240223T214941Z_reco2_ana.root && echo "tmp/nu_dune10kt_1x2x6_1421_92_20230827T093331Z_gen_g4_detsim_hitreco__20240223T220518Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_992_20230827T094115Z_gen_g4_detsim_hitreco__20240223T214846Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_145_20230827T183303Z_gen_g4_detsim_hitreco__20240223T214931Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_19_20230827T183332Z_gen_g4_detsim_hitreco__20240223T221539Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_355_20230827T183309Z_gen_g4_detsim_hitreco__20240223T225246Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1425_389_20230827T183324Z_gen_g4_detsim_hitreco__20240223T214923Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1427_511_20230827T213353Z_gen_g4_detsim_hitreco__20240223T220439Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1427_617_20230827T213358Z_gen_g4_detsim_hitreco__20240223T214941Z_reco2_ana.root" | tr " " "\n" > files.txt && wc -l files.txt | cut -d" " -f1 > fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f1.txt
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f1.root
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_files_20251113T231223_f1.txt
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f1.txt
Deleting local input file copies
merged pass1_US_FNAL-FermiGrid_000011.json
justIN time: 2026-02-09 17:15:08 UTC       justIN version: 01.06.00