justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Jobsub ID 243668.7@dunegpschedd02.fnal.gov

Jobsub ID243668.7@dunegpschedd02.fnal.gov
Workflow ID10174
Stage ID1
User nameemuldoon@fnal.gov
RequestedProcessors1
GPUNo
RSS bytes2097152000 (2000 MiB)
Wall seconds limit80000 (22 hours)
Submitted time2025-11-13 23:24:53
SiteUS_FNAL-FermiGrid
EntryFNAL_GPGrid_ce04_mcore_op_duneonly
Last heartbeat2025-11-14 01:14:03
From worker nodeHostnamedunegli-7743887-0-fnpc23019.fnal.gov
cpuinfoAMD EPYC 7543 32-Core Processor
OS releaseScientific Linux release 7.9 (Nitrogen)
Processors1
RSS bytes2097152000 (2000 MiB)
Wall seconds limit172800 (48 hours)
GPU
Inner Apptainer?True
Job stateoutputting_failed
Started2025-11-14 00:53:12
Input filesmonte-carlo-010174-000009
JobscriptExit code0
Real time11m (664s)
CPU time0m (7s = 1%)
Max RSS bytes485277696 (462 MiB)
Outputting started2025-11-14 01:04:18
Output files
Finished2025-11-14 01:14:03
List job events     (HTCondor job logs unavailable)

Jobscript log (last 10,000 characters)

_1413_945_20230826T184453Z_gen_g4_detsim_hitreco__20240222T214722Z_reco2_ana.root",
      "namespace": "fardet-hd"
    }
  ],
  "inputs": [
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/da/f6/nu_dune10kt_1x2x6_1410_694_20230826T123358Z_gen_g4_detsim_hitreco__20240224T092740Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/a9/73/nu_dune10kt_1x2x6_1410_712_20230826T123405Z_gen_g4_detsim_hitreco__20240222T234757Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/b7/a6/nu_dune10kt_1x2x6_1410_766_20230826T123357Z_gen_g4_detsim_hitreco__20240222T231844Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/ef/8a/nu_dune10kt_1x2x6_1410_779_20230826T123403Z_gen_g4_detsim_hitreco__20240221T160944Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/b9/5f/nu_dune10kt_1x2x6_1410_904_20230826T123558Z_gen_g4_detsim_hitreco__20240222T221244Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/c9/28/nu_dune10kt_1x2x6_1412_80_20230826T153257Z_gen_g4_detsim_hitreco__20240223T005437Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/12/35/nu_dune10kt_1x2x6_1412_90_20230826T153254Z_gen_g4_detsim_hitreco__20240223T005509Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/55/ae/nu_dune10kt_1x2x6_1413_328_20230826T183327Z_gen_g4_detsim_hitreco__20240223T003500Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/63/fe/nu_dune10kt_1x2x6_1413_40_20230826T183316Z_gen_g4_detsim_hitreco__20240223T005505Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/20/21/nu_dune10kt_1x2x6_1413_945_20230826T184453Z_gen_g4_detsim_hitreco__20240222T214722Z_reco2_ana.root"
  ],
  "outputs": [
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root"
    },
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_files_20251113T231223_f0_c8.txt",
      "metadata": {
        "core.file_format": "text"
      },
      "rename": "files.txt"
    },
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f0_c8.txt",
      "metadata": {
        "core.file_format": "text"
      }
    }
  ],
  "settings": {
    "streaming": false,
    "method": "multi-output",
    "cmd": "hadd -f {output} {inputs} && echo \"{inputs}\" | tr \" \" \"\\n\" > files.txt && wc -l files.txt | cut -d\" \" -f1 > {outputs[2]}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root
hadd compression setting for all output: 1
hadd Source file 1: tmp/nu_dune10kt_1x2x6_1410_694_20230826T123358Z_gen_g4_detsim_hitreco__20240224T092740Z_reco2_ana.root
hadd Source file 2: tmp/nu_dune10kt_1x2x6_1410_712_20230826T123405Z_gen_g4_detsim_hitreco__20240222T234757Z_reco2_ana.root
hadd Source file 3: tmp/nu_dune10kt_1x2x6_1410_766_20230826T123357Z_gen_g4_detsim_hitreco__20240222T231844Z_reco2_ana.root
hadd Source file 4: tmp/nu_dune10kt_1x2x6_1410_779_20230826T123403Z_gen_g4_detsim_hitreco__20240221T160944Z_reco2_ana.root
hadd Source file 5: tmp/nu_dune10kt_1x2x6_1410_904_20230826T123558Z_gen_g4_detsim_hitreco__20240222T221244Z_reco2_ana.root
hadd Source file 6: tmp/nu_dune10kt_1x2x6_1412_80_20230826T153257Z_gen_g4_detsim_hitreco__20240223T005437Z_reco2_ana.root
hadd Source file 7: tmp/nu_dune10kt_1x2x6_1412_90_20230826T153254Z_gen_g4_detsim_hitreco__20240223T005509Z_reco2_ana.root
hadd Source file 8: tmp/nu_dune10kt_1x2x6_1413_328_20230826T183327Z_gen_g4_detsim_hitreco__20240223T003500Z_reco2_ana.root
hadd Source file 9: tmp/nu_dune10kt_1x2x6_1413_40_20230826T183316Z_gen_g4_detsim_hitreco__20240223T005505Z_reco2_ana.root
hadd Source file 10: tmp/nu_dune10kt_1x2x6_1413_945_20230826T184453Z_gen_g4_detsim_hitreco__20240222T214722Z_reco2_ana.root
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/flashmatchAna
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/emshowershweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/pandoraeff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/pandorashweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/pmtrackeff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/pmtracktceff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/pmtrajfiteff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/pmtrajfittceff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/runcheckereco
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/trajclustershweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root:/analysistree
Making local copy of input files in tmp:
  Copying nu_dune10kt_1x2x6_1410_694_20230826T123358Z_gen_g4_detsim_hitreco__20240224T092740Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1410_712_20230826T123405Z_gen_g4_detsim_hitreco__20240222T234757Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1410_766_20230826T123357Z_gen_g4_detsim_hitreco__20240222T231844Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1410_779_20230826T123403Z_gen_g4_detsim_hitreco__20240221T160944Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1410_904_20230826T123558Z_gen_g4_detsim_hitreco__20240222T221244Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1412_80_20230826T153257Z_gen_g4_detsim_hitreco__20240223T005437Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1412_90_20230826T153254Z_gen_g4_detsim_hitreco__20240223T005509Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1413_328_20230826T183327Z_gen_g4_detsim_hitreco__20240223T003500Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1413_40_20230826T183316Z_gen_g4_detsim_hitreco__20240223T005505Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1413_945_20230826T184453Z_gen_g4_detsim_hitreco__20240222T214722Z_reco2_ana.root
Copied 10 files
Merging 10 files into fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root using method multi-output
hadd -f fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root tmp/nu_dune10kt_1x2x6_1410_694_20230826T123358Z_gen_g4_detsim_hitreco__20240224T092740Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_712_20230826T123405Z_gen_g4_detsim_hitreco__20240222T234757Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_766_20230826T123357Z_gen_g4_detsim_hitreco__20240222T231844Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_779_20230826T123403Z_gen_g4_detsim_hitreco__20240221T160944Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_904_20230826T123558Z_gen_g4_detsim_hitreco__20240222T221244Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1412_80_20230826T153257Z_gen_g4_detsim_hitreco__20240223T005437Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1412_90_20230826T153254Z_gen_g4_detsim_hitreco__20240223T005509Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1413_328_20230826T183327Z_gen_g4_detsim_hitreco__20240223T003500Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1413_40_20230826T183316Z_gen_g4_detsim_hitreco__20240223T005505Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1413_945_20230826T184453Z_gen_g4_detsim_hitreco__20240222T214722Z_reco2_ana.root && echo "tmp/nu_dune10kt_1x2x6_1410_694_20230826T123358Z_gen_g4_detsim_hitreco__20240224T092740Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_712_20230826T123405Z_gen_g4_detsim_hitreco__20240222T234757Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_766_20230826T123357Z_gen_g4_detsim_hitreco__20240222T231844Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_779_20230826T123403Z_gen_g4_detsim_hitreco__20240221T160944Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1410_904_20230826T123558Z_gen_g4_detsim_hitreco__20240222T221244Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1412_80_20230826T153257Z_gen_g4_detsim_hitreco__20240223T005437Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1412_90_20230826T153254Z_gen_g4_detsim_hitreco__20240223T005509Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1413_328_20230826T183327Z_gen_g4_detsim_hitreco__20240223T003500Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1413_40_20230826T183316Z_gen_g4_detsim_hitreco__20240223T005505Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1413_945_20230826T184453Z_gen_g4_detsim_hitreco__20240222T214722Z_reco2_ana.root" | tr " " "\n" > files.txt && wc -l files.txt | cut -d" " -f1 > fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f0_c8.txt
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c8.root
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_files_20251113T231223_f0_c8.txt
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f0_c8.txt
Deleting local input file copies
merged pass1_US_FNAL-FermiGrid_000009.json
justIN time: 2026-02-09 17:14:16 UTC       justIN version: 01.06.00