Jobsub ID 257284.1@dunegpschedd02.fnal.gov

Jobsub ID	257284.1@dunegpschedd02.fnal.gov
Workflow ID	10980
Stage ID	1
User name	schellma@fnal.gov
HTCondor Group	group_dune
Requested	Processors	1
	GPU	No
	RSS bytes	2097152000 (2000 MiB)
	Wall seconds limit	80000 (22 hours)
Submitted time	2025-12-04 02:29:01
Site	CERN
Entry	CMSHTPC_T2_CH_CERN_ce504
Last heartbeat	2025-12-04 02:34:40
From worker node	Hostname	b9p08p0357.cern.ch
	cpuinfo	AMD EPYC 7543 32-Core Processor
	OS release	Scientific Linux release 7.9 (Nitrogen)
	Processors	1
	RSS bytes	2097152000 (2000 MiB)
	Wall seconds limit	343800 (95 hours)
	GPU
	Inner Apptainer?	True
Job state	stalled
Started	2025-12-04 02:30:31
Input files	monte-carlo-010980-000025
Outputting started	2025-12-04 02:34:40
Output files
Finished	2025-12-04 03:27:04
List job events (HTCondor job logs unavailable)
Jobscript log (last 10,000 characters)

Config file: pass1_CERN_000025.json
{
  "namespace": "usertests",
  "metadata": {
    "core.first_event_number": 41,
    "core.last_event_number": 92680,
    "core.event_count": 2100,
    "core.runs": [
      8146,
      8180,
      8188
    ],
    "core.runs_subruns": [
      814600000,
      818800000,
      818000000
    ],
    "dune.workflow": {
      "user": "schellma@fnal.gov",
      "cpuinfo": "AMD EPYC 7543 32-Core Processor",
      "stage_id": 1,
      "site_name": "CERN",
      "os_release": "Scientific Linux release 7.9 (Nitrogen)"
    },
    "retention.status": "active",
    "merge.version": "0.7.3",
    "merge.method": "hadd",
    "merge.timestamp": "20251204T014000",
    "merge.cmd": "hadd -f {output} {inputs}",
    "merge.tag": "testCENTRAL2",
    "merge.query": "files where merge.tag=CENTRAL_prod and core.run_type='fardet-hd' and dune.output_status=confirmed and core.data_tier='root-tuple' and dune.campaign=trg_mc_2025a_tpg and merge.dataset='fardet-hd:fardet-hd__trg_mc_2025a_tpg__trigger-primitives__v10_12_01d01__tpg_dune10kt_1x2x6__prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__out1__v1_official' and merge.cfg='triggerana_dune10kt_1x2x6.fcl'",
    "dune.user": "dunepro",
    "merge.cfg": "triggerana_dune10kt_1x2x6.fcl",
    "core.group": "dune",
    "core.run_type": "fardet-hd",
    "dune.campaign": "trg_mc_2025a_tpg",
    "merge.dataset": "fardet-hd:fardet-hd__trg_mc_2025a_tpg__trigger-primitives__v10_12_01d01__tpg_dune10kt_1x2x6__prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__out1__v1_official",
    "core.data_tier": "root-tuple",
    "core.file_type": "mc",
    "dune.requestid": "ritm2612698",
    "retention.class": "unknown",
    "art.process_name": "TPGen",
    "core.application": "art.reco",
    "core.data_stream": "out1",
    "core.file_format": "root",
    "dune.config_file": "tpg_dune10kt_1x2x6.fcl",
    "dune.output_status": "confirmed",
    "art.file_format_era": "ART_2011a",
    "core.application.name": "triggerAna",
    "dune_mc.detector_type": "fardet-hd",
    "art.file_format_version": 15,
    "core.application.family": "art",
    "core.application.version": "v10_12_01d01",
    "core.file_content_status": "good",
    "dune_mc.gen_fcl_filename": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA.fcl",
    "merge.hash": "9dd448044265ab72b0d7594bb481a3004af970dc1a79ddd6fe11d73f23961725",
    "merge.group": 24
  },
  "parents": [
    {
      "fid": "BLmyfplQRdi7uTPu",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251126T010938_c9.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "IlRGX6rMS5yiScHJ",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c0.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "IN8SJHxTRIGyJ3YI",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c1.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "L0pJ81gVR9qkWPBI",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s004000_l002000_20251201T162356.root",
      "namespace": "fardet-hd"
    }
  ],
  "inputs": [
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/9f/39/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251126T010938_c9.root",
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/2c/ed/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c0.root",
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/ae/4b/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c1.root",
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/e5/8d/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s004000_l002000_20251201T162356.root"
  ],
  "outputs": [
    {
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root",
      "metadata": {
        "core.data_tier": "root-tuple",
        "core.application.name": "triggerAna",
        "core.file_format": "root"
      }
    }
  ],
  "settings": {
    "streaming": true,
    "method": "hadd",
    "cmd": "hadd -f {output} {inputs}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root
hadd compression setting for all output: 1
hadd Source file 1: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/9f/39/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251126T010938_c9.root
hadd Source file 2: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/2c/ed/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c0.root
hadd Source file 3: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/ae/4b/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c1.root
hadd Source file 4: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/e5/8d/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s004000_l002000_20251201T162356.root
hadd Target path: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root:/
hadd Target path: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root:/triggerAna
hadd Target path: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root:/triggerAna/TriggerPrimitives
Merging 4 files into prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root using method hadd
hadd -f prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/9f/39/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251126T010938_c9.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/2c/ed/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c0.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/ae/4b/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_l005000_20251201T032849_c1.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/e5/8d/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s004000_l002000_20251201T162356.root
Processing output file prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f24.root
merged pass1_CERN_000025.json