justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Jobsub ID 269574.1@dunegpschedd01.fnal.gov

Jobsub ID269574.1@dunegpschedd01.fnal.gov
Workflow ID11301
Stage ID1
User nameschellma@fnal.gov
RequestedProcessors1
GPUNo
RSS bytes2097152000 (2000 MiB)
Wall seconds limit80000 (22 hours)
Submitted time2025-12-21 17:59:36
SiteUS_FNAL-FermiGrid
EntryFNAL_GPGrid_ce03_mcore_op_duneonly
Last heartbeat2025-12-21 18:09:32
From worker nodeHostnamedunegli-7972646-0-fnpc23033.fnal.gov
cpuinfoAMD EPYC 7543 32-Core Processor
OS releaseScientific Linux release 7.9 (Nitrogen)
Processors1
RSS bytes2097152000 (2000 MiB)
Wall seconds limit172800 (48 hours)
GPU
Inner Apptainer?True
Job statejobscript_error
Started2025-12-21 18:07:29
Input filesmonte-carlo-011301-000003
JobscriptExit code2
Real time0m (0s)
CPU time0m (0s = 0%)
Max RSS bytes0 (0 MiB)
Outputting started 
Output files
Finished2025-12-21 18:09:32
Saved logsjustin-logs:269574.1-dunegpschedd01.fnal.gov.logs.tgz
List job events     Cached HTCondor job logs

Jobscript log (last 10,000 characters)

,
    {
      "fid": "jNfdZeTOTwu9qP9c"
    },
    {
      "fid": "MHkNnEanSAeuAQqq"
    },
    {
      "fid": "NFuE911ZRoe86DRr"
    },
    {
      "fid": "XmjrQ1luSzOtbF0g"
    },
    {
      "fid": "l51cykq6SaK1VAZU"
    },
    {
      "fid": "yWgdGaV4QOC3o2F3"
    },
    {
      "fid": "ZXf1dl5cSwO1LN2p"
    },
    {
      "fid": "WTXSVxe9RDCF1rkF"
    },
    {
      "fid": "ymH3q8YtTmqs5hSL"
    },
    {
      "fid": "LT6OTt7RQmmlwghr"
    },
    {
      "fid": "zPfcIxrOR0mxWgoN"
    },
    {
      "fid": "N2tnslxoTeyz4A6s"
    },
    {
      "fid": "jfILGvB6RaGe5g89"
    },
    {
      "fid": "L9sPRxI9SI67n5B6"
    },
    {
      "fid": "KFeuI99bSwCq6kuY"
    },
    {
      "fid": "zM54hp02SJOgllge"
    },
    {
      "fid": "nCEpXPMQQQ2ktFON"
    },
    {
      "fid": "mhejQD76Rseece7R"
    },
    {
      "fid": "XhHYN91CT6a3DTAU"
    },
    {
      "fid": "mdOPOafQTC2WnRFP"
    },
    {
      "fid": "melgR4j9Q42s8QoU"
    },
    {
      "fid": "mtFbTWRRSASPE70j"
    },
    {
      "fid": "jb7qbFJFQrKlfVgM"
    },
    {
      "fid": "nB05YJdXTUWrdoOK"
    },
    {
      "fid": "yEs1v69OSCGd0zVI"
    },
    {
      "fid": "n0iKOapqRrOCK7s8"
    },
    {
      "fid": "KqWRzX5eSeWqfiCh"
    },
    {
      "fid": "KIB1Mcm4Qc2oOtzO"
    },
    {
      "fid": "l6fKm50WTkGw2w0C"
    },
    {
      "fid": "IGOELqG3RYe3MPG2"
    },
    {
      "fid": "hzbWj653TZWtHJev"
    },
    {
      "fid": "nSl3XHRJT42n4b6V"
    },
    {
      "fid": "mon8F2OCQUq6tuCZ"
    },
    {
      "fid": "v2G6WmEAQ7qVHfg1"
    },
    {
      "fid": "lwoQg5ykQfyqzC8h"
    },
    {
      "fid": "kdo4ix1yQBSwDECK"
    },
    {
      "fid": "iCYgshizSyGvb46c"
    },
    {
      "fid": "mMOet2O6Qky6cqRG"
    },
    {
      "fid": "lQxPlWIUSRiL4z8C"
    },
    {
      "fid": "jJONF8iAQWiWRKAz"
    },
    {
      "fid": "I6fCVVX9SIuZOWVt"
    },
    {
      "fid": "UKet31rNT4K5Q2Fy"
    },
    {
      "fid": "II7WFIsOTCyL43QE"
    },
    {
      "fid": "Mg71MTVwRqKuRyZT"
    },
    {
      "fid": "u97W1CJmQUiujQ8N"
    },
    {
      "fid": "WxJJVlBHSuC04hUm"
    },
    {
      "fid": "LBoIQazwQPGSiDVL"
    },
    {
      "fid": "xCJyTPgUQKS1NJf3"
    },
    {
      "fid": "lEbaPpHMR12FZqMn"
    },
    {
      "fid": "WaNGNmZySPStaFxa"
    },
    {
      "fid": "wlAqgsiSQeqJXggw"
    },
    {
      "fid": "mLwCOIdbQD6jx6ME"
    },
    {
      "fid": "X7qTocuHTVKeNkYa"
    },
    {
      "fid": "XsEMT1jTRB677a9V"
    },
    {
      "fid": "W6T3VepZSzCOdfbt"
    },
    {
      "fid": "VROLZrLVQwOgu9KB"
    },
    {
      "fid": "YbudncxQTuuXRl8T"
    },
    {
      "fid": "utcGZ2K0TzW8yce5"
    },
    {
      "fid": "xWlUMginRr2TPwYu"
    },
    {
      "fid": "kV9sY8qZRIi0rrNT"
    },
    {
      "fid": "jDVVMOQ7RRmH8jFY"
    },
    {
      "fid": "JD3QX1bwQ5SXEdnP"
    },
    {
      "fid": "ntE4igiGTceUtucW"
    },
    {
      "fid": "jFwuxHh5SwCfT4kP"
    },
    {
      "fid": "MqOiZIVkQYSf0ejL"
    },
    {
      "fid": "vT6qZgqDToujAOmA"
    },
    {
      "fid": "KIx7pndwQtK1nmEz"
    },
    {
      "fid": "nFds4RGzQ82GcA32"
    },
    {
      "fid": "Zi3F64JPTJ2DU3JB"
    },
    {
      "fid": "VVbKbo37SauxKcQT"
    },
    {
      "fid": "jQoBQMeTSNy4m6ov"
    },
    {
      "fid": "HXGDqJZGQO6l2vfA"
    },
    {
      "fid": "JqENg47xQKa1yrkg"
    },
    {
      "fid": "LQqL93k5RYqrZCTk"
    },
    {
      "fid": "LLE2vtDlQdC3vWj3"
    },
    {
      "fid": "lM7yEXNETxOdzQck"
    },
    {
      "fid": "hYzc69dGQmGhaYZd"
    },
    {
      "fid": "ZJ0QYEHPTRWkX9eX"
    },
    {
      "fid": "ln9rQ0MsTAKOu3Oa"
    },
    {
      "fid": "jrHbszszRqySmlnF"
    },
    {
      "fid": "MnQ7M7gMQgmRA6Ji"
    },
    {
      "fid": "M76IbnQlSQWVRnLt"
    },
    {
      "fid": "nhiNqDUVQN2ux3Ne"
    },
    {
      "fid": "laPoWqRsTImuRDpo"
    },
    {
      "fid": "ic24PiinSe6tEIxE"
    },
    {
      "fid": "Lsrvq91tR8CZ3pQG"
    },
    {
      "fid": "Vax1CyeSSA6M0E6e"
    },
    {
      "fid": "wEQDyc8gRCGRaIfT"
    },
    {
      "fid": "yDPfa97BTl6owLum"
    },
    {
      "fid": "KTfzscl7SIaLPBMZ"
    },
    {
      "fid": "jMtUxQ7mQICtdfIH"
    },
    {
      "fid": "yKPhiZOvR92pwnC6"
    },
    {
      "fid": "VjJ5Rs1VRs6yD7q1"
    },
    {
      "fid": "n0YYg7VSQXGNwFTn"
    },
    {
      "fid": "ioFUS3UlQJaLW7o4"
    },
    {
      "fid": "Xy3lZzHNSAytA0TC"
    },
    {
      "fid": "ZaNWGo2hTOmSRaPy"
    },
    {
      "fid": "noRTa7MCTM6rNqcR"
    },
    {
      "fid": "lwVSVYV6TRqJ02V1"
    },
    {
      "fid": "X9piRdGFTJKPfKQ4"
    },
    {
      "fid": "jRYQjp0JTTyCsshi"
    },
    {
      "fid": "z2Ury2ljSiSQ8V9x"
    },
    {
      "fid": "HTTGXu79TpitPyU1"
    },
    {
      "fid": "nERE4ezcQRmCmrJG"
    },
    {
      "fid": "nOdmA9mhQ1KOrLGj"
    },
    {
      "fid": "zyBydpriTTGCjEIA"
    },
    {
      "fid": "lIHrtq9bSJ6QtkL7"
    },
    {
      "fid": "jZmj5cmgTUKo2Gyl"
    },
    {
      "fid": "UMDwabdORzeODmO2"
    },
    {
      "fid": "jUJvY9dERbmaUJZJ"
    },
    {
      "fid": "Ligo9WNCQRym365j"
    },
    {
      "fid": "MOJgzqHsRCytPuUN"
    },
    {
      "fid": "mvsCu8N5StKYwcqY"
    },
    {
      "fid": "l9DFIPajSqiJVo3N"
    },
    {
      "fid": "JM9Juzz1TBWgLxPg"
    },
    {
      "fid": "v7UcgYUZRbaR2ASg"
    },
    {
      "fid": "vtSyfUplRE60CnIS"
    },
    {
      "fid": "mx30A8ocSROMM1S3"
    },
    {
      "fid": "MkzPc3WoSnWOmeSJ"
    },
    {
      "fid": "VKixtbapTwSbFQAr"
    },
    {
      "fid": "yWnJHzZnTxOrSrY5"
    },
    {
      "fid": "vWMUxBCNTWqZYuB9"
    },
    {
      "fid": "IZCGavAPRJOzRJRX"
    },
    {
      "fid": "n1rfb8svR4yY3rY6"
    },
    {
      "fid": "KTxq3HIkRvqmzX7N"
    },
    {
      "fid": "IYb8OH3DT6eLkJnB"
    },
    {
      "fid": "iNWrsMAGTh2TzTX3"
    },
    {
      "fid": "JPGaKRNJTwyc0CBi"
    },
    {
      "fid": "jW28TJ3qQzeTcAIO"
    }
  ],
  "inputs": [
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/a2/40/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f16.root",
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/98/f2/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f8.root"
  ],
  "outputs": [
    {
      "name": "prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root",
      "metadata": {
        "core.data_tier": "root-tuple",
        "core.application.name": "triggerAna",
        "core.file_format": "root"
      }
    }
  ],
  "settings": {
    "streaming": true,
    "method": "hadd",
    "cmd": "hadd -f {output} {inputs}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root
hadd compression setting for all output: 1
hadd Source file 1: root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/a2/40/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f16.root
hadd Source file 2: root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/98/f2/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f8.root
hadd Target path: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root:/
hadd Target path: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root:/triggerAna
hadd Target path: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root:/triggerAna/TriggerPrimitives
Merging 2 files into prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root using method hadd
hadd -f prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/a2/40/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f16.root root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/98/f2/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f8.root
Processing output file prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f003.root
check any root output
python3: can't open file '/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/1d5114b31de3affae98069da4f4a8d4912550812/rootchecker.py': [Errno 2] No such file or directory
bad root files pass1_US_FNAL-FermiGrid_000003.json 2
justIN time: 2026-02-07 07:45:53 UTC       justIN version: 01.06.00