justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Jobsub ID 243668.13@dunegpschedd02.fnal.gov

Jobsub ID243668.13@dunegpschedd02.fnal.gov
Workflow ID10174
Stage ID1
User nameemuldoon@fnal.gov
RequestedProcessors1
GPUNo
RSS bytes2097152000 (2000 MiB)
Wall seconds limit80000 (22 hours)
Submitted time2025-11-13 23:24:53
SiteUS_FNAL-FermiGrid
EntryFNAL_GPGrid_ce04_mcore_op_duneonly
Last heartbeat2025-11-14 01:10:16
From worker nodeHostnamedunegli-7742722-0-fnpc23002.fnal.gov
cpuinfoAMD EPYC 7543 32-Core Processor
OS releaseScientific Linux release 7.9 (Nitrogen)
Processors1
RSS bytes2097152000 (2000 MiB)
Wall seconds limit172800 (48 hours)
GPU
Inner Apptainer?True
Job stateoutputting_failed
Started2025-11-14 00:53:29
Input filesmonte-carlo-010174-000010
JobscriptExit code0
Real time0m (13s)
CPU time0m (7s = 53%)
Max RSS bytes542007296 (516 MiB)
Outputting started2025-11-14 00:53:42
Output files
Finished2025-11-14 01:10:16
List job events     (HTCondor job logs unavailable)

Jobscript log (last 10,000 characters)

_634_20230827T093405Z_gen_g4_detsim_hitreco__20240223T221555Z_reco2_ana.root",
      "namespace": "fardet-hd"
    }
  ],
  "inputs": [
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/8b/9f/nu_dune10kt_1x2x6_1413_963_20230826T184559Z_gen_g4_detsim_hitreco__20240224T092712Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/b8/fe/nu_dune10kt_1x2x6_1421_106_20230827T093336Z_gen_g4_detsim_hitreco__20240223T224127Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/e0/65/nu_dune10kt_1x2x6_1421_124_20230827T093329Z_gen_g4_detsim_hitreco__20240223T224055Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/9b/6f/nu_dune10kt_1x2x6_1421_13_20230827T093326Z_gen_g4_detsim_hitreco__20240223T225058Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/2f/ff/nu_dune10kt_1x2x6_1421_184_20230827T093342Z_gen_g4_detsim_hitreco__20240223T225239Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/ce/80/nu_dune10kt_1x2x6_1421_251_20230827T093343Z_gen_g4_detsim_hitreco__20240223T221533Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/88/b7/nu_dune10kt_1x2x6_1421_41_20230827T093351Z_gen_g4_detsim_hitreco__20240223T221530Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/05/ad/nu_dune10kt_1x2x6_1421_482_20230827T093355Z_gen_g4_detsim_hitreco__20240223T222013Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/a6/63/nu_dune10kt_1x2x6_1421_533_20230827T093358Z_gen_g4_detsim_hitreco__20240223T214922Z_reco2_ana.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/cf/af/nu_dune10kt_1x2x6_1421_634_20230827T093405Z_gen_g4_detsim_hitreco__20240223T221555Z_reco2_ana.root"
  ],
  "outputs": [
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root"
    },
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_files_20251113T231223_f0_c9.txt",
      "metadata": {
        "core.file_format": "text"
      },
      "rename": "files.txt"
    },
    {
      "name": "fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f0_c9.txt",
      "metadata": {
        "core.file_format": "text"
      }
    }
  ],
  "settings": {
    "streaming": false,
    "method": "multi-output",
    "cmd": "hadd -f {output} {inputs} && echo \"{inputs}\" | tr \" \" \"\\n\" > files.txt && wc -l files.txt | cut -d\" \" -f1 > {outputs[2]}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root
hadd compression setting for all output: 1
hadd Source file 1: tmp/nu_dune10kt_1x2x6_1413_963_20230826T184559Z_gen_g4_detsim_hitreco__20240224T092712Z_reco2_ana.root
hadd Source file 2: tmp/nu_dune10kt_1x2x6_1421_106_20230827T093336Z_gen_g4_detsim_hitreco__20240223T224127Z_reco2_ana.root
hadd Source file 3: tmp/nu_dune10kt_1x2x6_1421_124_20230827T093329Z_gen_g4_detsim_hitreco__20240223T224055Z_reco2_ana.root
hadd Source file 4: tmp/nu_dune10kt_1x2x6_1421_13_20230827T093326Z_gen_g4_detsim_hitreco__20240223T225058Z_reco2_ana.root
hadd Source file 5: tmp/nu_dune10kt_1x2x6_1421_184_20230827T093342Z_gen_g4_detsim_hitreco__20240223T225239Z_reco2_ana.root
hadd Source file 6: tmp/nu_dune10kt_1x2x6_1421_251_20230827T093343Z_gen_g4_detsim_hitreco__20240223T221533Z_reco2_ana.root
hadd Source file 7: tmp/nu_dune10kt_1x2x6_1421_41_20230827T093351Z_gen_g4_detsim_hitreco__20240223T221530Z_reco2_ana.root
hadd Source file 8: tmp/nu_dune10kt_1x2x6_1421_482_20230827T093355Z_gen_g4_detsim_hitreco__20240223T222013Z_reco2_ana.root
hadd Source file 9: tmp/nu_dune10kt_1x2x6_1421_533_20230827T093358Z_gen_g4_detsim_hitreco__20240223T214922Z_reco2_ana.root
hadd Source file 10: tmp/nu_dune10kt_1x2x6_1421_634_20230827T093405Z_gen_g4_detsim_hitreco__20240223T221555Z_reco2_ana.root
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/flashmatchAna
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/emshowershweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/pandoraeff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/pandorashweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/pmtrackeff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/pmtracktceff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/pmtrajfiteff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/pmtrajfittceff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/runcheckereco
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/trajclustershweff
hadd Target path: fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root:/analysistree
Making local copy of input files in tmp:
  Copying nu_dune10kt_1x2x6_1413_963_20230826T184559Z_gen_g4_detsim_hitreco__20240224T092712Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_106_20230827T093336Z_gen_g4_detsim_hitreco__20240223T224127Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_124_20230827T093329Z_gen_g4_detsim_hitreco__20240223T224055Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_13_20230827T093326Z_gen_g4_detsim_hitreco__20240223T225058Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_184_20230827T093342Z_gen_g4_detsim_hitreco__20240223T225239Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_251_20230827T093343Z_gen_g4_detsim_hitreco__20240223T221533Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_41_20230827T093351Z_gen_g4_detsim_hitreco__20240223T221530Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_482_20230827T093355Z_gen_g4_detsim_hitreco__20240223T222013Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_533_20230827T093358Z_gen_g4_detsim_hitreco__20240223T214922Z_reco2_ana.root
  Copying nu_dune10kt_1x2x6_1421_634_20230827T093405Z_gen_g4_detsim_hitreco__20240223T221555Z_reco2_ana.root
Copied 10 files
Merging 10 files into fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root using method multi-output
hadd -f fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root tmp/nu_dune10kt_1x2x6_1413_963_20230826T184559Z_gen_g4_detsim_hitreco__20240224T092712Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_106_20230827T093336Z_gen_g4_detsim_hitreco__20240223T224127Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_124_20230827T093329Z_gen_g4_detsim_hitreco__20240223T224055Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_13_20230827T093326Z_gen_g4_detsim_hitreco__20240223T225058Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_184_20230827T093342Z_gen_g4_detsim_hitreco__20240223T225239Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_251_20230827T093343Z_gen_g4_detsim_hitreco__20240223T221533Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_41_20230827T093351Z_gen_g4_detsim_hitreco__20240223T221530Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_482_20230827T093355Z_gen_g4_detsim_hitreco__20240223T222013Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_533_20230827T093358Z_gen_g4_detsim_hitreco__20240223T214922Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_634_20230827T093405Z_gen_g4_detsim_hitreco__20240223T221555Z_reco2_ana.root && echo "tmp/nu_dune10kt_1x2x6_1413_963_20230826T184559Z_gen_g4_detsim_hitreco__20240224T092712Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_106_20230827T093336Z_gen_g4_detsim_hitreco__20240223T224127Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_124_20230827T093329Z_gen_g4_detsim_hitreco__20240223T224055Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_13_20230827T093326Z_gen_g4_detsim_hitreco__20240223T225058Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_184_20230827T093342Z_gen_g4_detsim_hitreco__20240223T225239Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_251_20230827T093343Z_gen_g4_detsim_hitreco__20240223T221533Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_41_20230827T093351Z_gen_g4_detsim_hitreco__20240223T221530Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_482_20230827T093355Z_gen_g4_detsim_hitreco__20240223T222013Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_533_20230827T093358Z_gen_g4_detsim_hitreco__20240223T214922Z_reco2_ana.root tmp/nu_dune10kt_1x2x6_1421_634_20230827T093405Z_gen_g4_detsim_hitreco__20240223T221555Z_reco2_ana.root" | tr " " "\n" > files.txt && wc -l files.txt | cut -d" " -f1 > fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f0_c9.txt
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_merged_20251113T231223_f0_c9.root
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_files_20251113T231223_f0_c9.txt
Processing output file fd-hd_fd_mc_2023a_reco2_standard_ana_dune10kt_1x2x6.fcl_anatree_count_20251113T231223_f0_c9.txt
Deleting local input file copies
merged pass1_US_FNAL-FermiGrid_000010.json
justIN time: 2026-02-09 17:14:44 UTC       justIN version: 01.06.00