Jobsub ID 263745.1@dunegpschedd02.fnal.gov

Jobsub ID	263745.1@dunegpschedd02.fnal.gov
Workflow ID	11301
Stage ID	1
User name	schellma@fnal.gov
Requested	Processors	1
	GPU	No
	RSS bytes	2097152000 (2000 MiB)
	Wall seconds limit	80000 (22 hours)
Submitted time	2025-12-21 18:19:37
Site	US_FNAL-FermiGrid
Entry	FNAL_GPGrid_ce04_mcore_op_duneonly
Last heartbeat	2025-12-21 18:22:03
From worker node	Hostname	dunegli-8129005-0-fnpc22030.fnal.gov
	cpuinfo	AMD EPYC 7543 32-Core Processor
	OS release	Scientific Linux release 7.9 (Nitrogen)
	Processors	1
	RSS bytes	2097152000 (2000 MiB)
	Wall seconds limit	172800 (48 hours)
	GPU
	Inner Apptainer?	True
Job state	jobscript_error
Started	2025-12-21 18:20:45
Input files	monte-carlo-011301-000002
Jobscript	Exit code	2
	Real time	0m (0s)
	CPU time	0m (0s = 0%)
	Max RSS bytes	0 (0 MiB)
Outputting started
Output files
Finished	2025-12-21 18:22:03
Saved logs	justin-logs:263745.1-dunegpschedd02.fnal.gov.logs.tgz
List job events Cached HTCondor job logs
Jobscript log (last 10,000 characters)

"fid": "Pl7pxfKmQyisDZxT"
    },
    {
      "fid": "YGn72LqhRcqafyPN"
    },
    {
      "fid": "v9JxG3WoTcuS666x"
    },
    {
      "fid": "Vlofc98eQTuVvjws"
    },
    {
      "fid": "V82iKZ63TTi8BzKp"
    },
    {
      "fid": "S1llU1mbROubjtYQ"
    },
    {
      "fid": "tR1eQduwS3O9W4Go"
    },
    {
      "fid": "ZOMEQHVoR6CwvVjT"
    },
    {
      "fid": "VlkXsACLSKiuEaI4"
    },
    {
      "fid": "y3PYGPd9TIutSDvi"
    },
    {
      "fid": "RdpoZjKcR4u8Dr9q"
    },
    {
      "fid": "PzHc9afmRz6J1ETm"
    },
    {
      "fid": "xA6Z9fvLQJCiTM7c"
    },
    {
      "fid": "O41hIMaZTICTPLuw"
    },
    {
      "fid": "uHM3D8gJTW6Onfxn"
    },
    {
      "fid": "Pb0yj675QbaDsXb3"
    },
    {
      "fid": "vNXNT3dqRBC0yEvB"
    },
    {
      "fid": "sJfScDHxTE2i4mpk"
    },
    {
      "fid": "t9PB29aWQUu91r1u"
    },
    {
      "fid": "ymNMtRhJSCaB8NbK"
    },
    {
      "fid": "Wly7TFKzTcWsGiWY"
    },
    {
      "fid": "ULd4vVDTTnm8Mv5f"
    },
    {
      "fid": "sF6qLNyQS0GIefBG"
    },
    {
      "fid": "wa55nJnoQcKB05Ix"
    },
    {
      "fid": "wJGDZIVyRwaoq80c"
    },
    {
      "fid": "Qri8W9vQRnyNlgIM"
    },
    {
      "fid": "RSMi6HRwRVO32wIk"
    },
    {
      "fid": "vCr8z6sRRV6hmPBv"
    },
    {
      "fid": "Z9KoYnuJQoC7aIN0"
    },
    {
      "fid": "RX15Qcj1RsibMUPr"
    },
    {
      "fid": "V1N6Pj7kSVGwPvHV"
    },
    {
      "fid": "qdQOFyoRQ7a0gPq3"
    },
    {
      "fid": "yc27fNQaSgep089s"
    },
    {
      "fid": "X5QMps3wQKaWQPaW"
    },
    {
      "fid": "vJk7AtoMTsmNN4bo"
    },
    {
      "fid": "uqtKKyRpTeekLKdy"
    },
    {
      "fid": "PDqnKA8NSn2NVPLb"
    },
    {
      "fid": "qH3ZnJdmRE6ZejUe"
    },
    {
      "fid": "x4IVRVw5S2ey6b8R"
    },
    {
      "fid": "ZWiWOmAyQ6uTUhCR"
    },
    {
      "fid": "ytYxEujhRoucLU0Y"
    },
    {
      "fid": "yjRDrdk5T1KxSG0u"
    },
    {
      "fid": "we4JF3urQoWuW9Ua"
    },
    {
      "fid": "pHmzKOdBQyCVjMVe"
    },
    {
      "fid": "ScUJ3zabSGiUTFGw"
    },
    {
      "fid": "rqAHurrzRYSUc9Ys"
    },
    {
      "fid": "ys3kcKKwR0qjV0cI"
    },
    {
      "fid": "wwOr6KQlQpmJzRIb"
    },
    {
      "fid": "ZFGt78wHTzmztl1s"
    },
    {
      "fid": "sZuC0RPkRu2T9Kdw"
    },
    {
      "fid": "vnU8RY9DShyLe5WQ"
    },
    {
      "fid": "Plj2H2pyQa65HbXp"
    },
    {
      "fid": "ShhFAxbfQCyWChRz"
    },
    {
      "fid": "zQNfp9MZSICfQgWP"
    },
    {
      "fid": "woEMKlKCTeKzYYBm"
    },
    {
      "fid": "rSdjjeMkTlGV5wP5"
    },
    {
      "fid": "wDj0fch3RZ2rxagn"
    },
    {
      "fid": "Qj1KgrVNRhKm98iD"
    },
    {
      "fid": "wixeRfSPQjqLutSe"
    },
    {
      "fid": "p61k4SiiQpS4YG6j"
    },
    {
      "fid": "QXeEZdErRuunhkcP"
    },
    {
      "fid": "xvxPpbl4S5mUhT01"
    },
    {
      "fid": "ZGZ21lPiQ9WJNihL"
    },
    {
      "fid": "QYef2yTXQgyFklgh"
    },
    {
      "fid": "sgDcajCPQheYPiaU"
    },
    {
      "fid": "rxtWPJRPRJGJrD8x"
    },
    {
      "fid": "ZqfeCeaZTOmhAaGH"
    },
    {
      "fid": "zyLibmjYTlWZT03l"
    },
    {
      "fid": "VOWOamDMRNSYG0o9"
    },
    {
      "fid": "WNpekydsT9md0HhM"
    },
    {
      "fid": "ZtlNyeCoQ7Gg7ZgJ"
    },
    {
      "fid": "oucatZapTSqiBfrm"
    },
    {
      "fid": "yRCeO8MnTJOgqaSW"
    },
    {
      "fid": "ZltArndWSuCHQ6nq"
    },
    {
      "fid": "yKUQMwaGQbquSd9o"
    },
    {
      "fid": "V7Ib8pAOQna8EJns"
    },
    {
      "fid": "w1Nl4maaR4OZ8IER"
    },
    {
      "fid": "Vm3eRVGKS1uKpt0D"
    },
    {
      "fid": "ornIpMu4TjaTDzJI"
    },
    {
      "fid": "ZRG0xUViQHGQObMO"
    },
    {
      "fid": "OPwnkZuWSKKKqpct"
    },
    {
      "fid": "xUZNcpZoTymCG6vx"
    },
    {
      "fid": "vPrq735DSW2qb0kS"
    },
    {
      "fid": "VjH1gZ6ATxaa98mS"
    },
    {
      "fid": "WqO8I8bWQNyahM76"
    },
    {
      "fid": "ZXR7XRJzSHC7zCNU"
    },
    {
      "fid": "Uckzm7b5Q0uGRgl6"
    },
    {
      "fid": "vAZOV8RYSVyV26pm"
    },
    {
      "fid": "zmPVdAdYTxCh6639"
    },
    {
      "fid": "SFL2k7DiS0OF8KbK"
    },
    {
      "fid": "wzLEkqwiQQGTIzYJ"
    },
    {
      "fid": "qA7KvTOEQG2y6QVc"
    },
    {
      "fid": "PYIQHZPQQJyaPk9o"
    },
    {
      "fid": "vDj8SsRORt2z42Do"
    },
    {
      "fid": "yKITalUkSpGrHJv6"
    },
    {
      "fid": "ZXn1L9GrSeSU5Zwa"
    },
    {
      "fid": "ZcOQ8gAtSj6NCyNo"
    },
    {
      "fid": "OwS6OjeBT9yAhOon"
    },
    {
      "fid": "RnmPdXcBTCWhcHRT"
    },
    {
      "fid": "yTcGyLrSQVu9APyK"
    },
    {
      "fid": "xlW0Xd0GTkyKhFDi"
    },
    {
      "fid": "rmQzjpooSIuSfe81"
    },
    {
      "fid": "W4RiqtuKQYG7KLgo"
    },
    {
      "fid": "yXAOvGopSkKHITWS"
    },
    {
      "fid": "v7xNP7ERQfuPklNq"
    },
    {
      "fid": "OMa7DbQNR3e46Mzm"
    },
    {
      "fid": "W2I3MPlgSCWEaeYm"
    },
    {
      "fid": "xFe4EIxtTuSJ5SNk"
    },
    {
      "fid": "UToN8kuaQdulkpNp"
    },
    {
      "fid": "wuVzWuOfQMWytsfH"
    },
    {
      "fid": "PPmpFH9nTQKUHGhu"
    },
    {
      "fid": "udUPOSGVRaKu9nQ0"
    },
    {
      "fid": "Onvjo2gYRqOeAGsg"
    },
    {
      "fid": "zPsWdIZARG6ofnP7"
    },
    {
      "fid": "pq6Z2jQBSi6mnkW5"
    },
    {
      "fid": "q7E1qCpTTwa40JMh"
    },
    {
      "fid": "Pn3zJCdgS06d2uFI"
    },
    {
      "fid": "uIsH2npATwCIoUUa"
    },
    {
      "fid": "ZdIHTdGmRyysNobs"
    },
    {
      "fid": "wz69sxPTTmGvfg6R"
    },
    {
      "fid": "P5VCaA8MQEqiaZwA"
    },
    {
      "fid": "YXXJvXiGRdmcwyC5"
    },
    {
      "fid": "rnJEIwG5TcGCfjTX"
    },
    {
      "fid": "Z4qcYyNlQbqycr4a"
    },
    {
      "fid": "ZIH9AFIKTX2fBwcF"
    },
    {
      "fid": "xX1ZBSKySOyOKqoE"
    },
    {
      "fid": "xWQyjCNOSZOffHpj"
    },
    {
      "fid": "uZs73iQ1QzqouHg1"
    },
    {
      "fid": "OcX3aU7WQYKXDyhT"
    },
    {
      "fid": "pftIKQPFRPCKFnLW"
    },
    {
      "fid": "YeayzhXlQXOm04B9"
    },
    {
      "fid": "zkToHldERjWVEyug"
    },
    {
      "fid": "YmES0wpnQpC4V29n"
    },
    {
      "fid": "r72gq68nQ9a9UWls"
    },
    {
      "fid": "WuhUF6IWSnKYMib0"
    },
    {
      "fid": "p3Vdo2PVSOmboxVD"
    },
    {
      "fid": "vcaBvKI7RS6RxJvg"
    },
    {
      "fid": "P9kJvLltRNe6MIw7"
    },
    {
      "fid": "vGXMx99LTPWyomwl"
    },
    {
      "fid": "YS5GTQHlQRaiD9FR"
    },
    {
      "fid": "QafUsSITT42eLu7s"
    },
    {
      "fid": "YMyBjNKhQGu4hlP4"
    },
    {
      "fid": "vuc5oljsSBOgaKUW"
    },
    {
      "fid": "xFGaNGLcQWKrbgs0"
    },
    {
      "fid": "UUZOXHVUSSmPYgJq"
    },
    {
      "fid": "WJNgZafsQzqvUR6d"
    },
    {
      "fid": "ws2Nv67NSrKRJC8T"
    },
    {
      "fid": "ya5wXWi3SRCipLh6"
    }
  ],
  "inputs": [
    "root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/8a/58/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f13.root"
  ],
  "outputs": [
    {
      "name": "prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root",
      "metadata": {
        "core.data_tier": "root-tuple",
        "core.application.name": "triggerAna",
        "core.file_format": "root"
      }
    }
  ],
  "settings": {
    "streaming": true,
    "method": "hadd",
    "cmd": "hadd -f {output} {inputs}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root
hadd compression setting for all output: 1
hadd Source file 1: root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/8a/58/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f13.root
hadd Target path: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root:/
hadd Target path: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root:/triggerAna
hadd Target path: prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root:/triggerAna/TriggerPrimitives
Merging 1 files into prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root using method hadd
hadd -f prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root root://fndcadoor.fnal.gov:1094/pnfs/fnal.gov/usr/dune/persistent/staging/fardet-hd/8a/58/prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__NU_prod-pass2_v2_20251126T000804_f13.root
Processing output file prodgenie_nu_dune10kt_1x2x6__trg_mc_2025a_tpg__triggerana_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__TEST-OUTPUTS_l000005_20251221T175637_f002.root
check any root output
python3: can't open file '/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/1d5114b31de3affae98069da4f4a8d4912550812/rootchecker.py': [Errno 2] No such file or directory
bad root files pass1_US_FNAL-FermiGrid_000002.json 2