Jobsub ID 257347.0@dunegpschedd02.fnal.gov

Jobsub ID	257347.0@dunegpschedd02.fnal.gov
Workflow ID	10980
Stage ID	1
User name	schellma@fnal.gov
Requested	Processors	1
	GPU	No
	RSS bytes	2097152000 (2000 MiB)
	Wall seconds limit	80000 (22 hours)
Submitted time	2025-12-04 03:29:04
Site	CERN
Entry	CMSHTPC_T2_CH_CERN_ce509
Last heartbeat	2025-12-04 03:36:30
From worker node	Hostname	b9p21p1067.cern.ch
	cpuinfo	AMD EPYC 7302 16-Core Processor
	OS release	Scientific Linux release 7.9 (Nitrogen)
	Processors	1
	RSS bytes	2097152000 (2000 MiB)
	Wall seconds limit	343800 (95 hours)
	GPU
	Inner Apptainer?	True
Job state	stalled
Started	2025-12-04 03:30:12
Input files	monte-carlo-010980-000250
Outputting started	2025-12-04 03:36:30
Output files
Finished	2025-12-04 04:27:05
List job events (HTCondor job logs unavailable)
Jobscript log (last 10,000 characters)

Config file: pass1_CERN_000250.json
{
  "namespace": "usertests",
  "metadata": {
    "core.first_event_number": 4091,
    "core.last_event_number": 92770,
    "core.event_count": 2370,
    "core.runs": [
      8169,
      8201,
      1854,
      8146,
      8180,
      8188,
      8190,
      8191
    ],
    "core.runs_subruns": [
      814600000,
      816900000,
      818800000,
      818000000,
      185400000,
      819000000,
      819100000,
      820100000
    ],
    "dune.workflow": {
      "user": "schellma@fnal.gov",
      "stage_id": 1,
      "site_name": "CERN",
      "os_release": "Scientific Linux release 7.9 (Nitrogen)"
    },
    "retention.status": "active",
    "merge.version": "0.7.3",
    "merge.method": "hadd",
    "merge.timestamp": "20251204T014000",
    "merge.cmd": "hadd -f {output} {inputs}",
    "merge.tag": "testCENTRAL2",
    "merge.query": "files where merge.tag=CENTRAL_prod and core.run_type='fardet-hd' and dune.output_status=confirmed and core.data_tier='root-tuple' and dune.campaign=trg_mc_2025a_tpg and merge.dataset='fardet-hd:fardet-hd__trg_mc_2025a_tpg__trigger-primitives__v10_12_01d01__tpg_dune10kt_1x2x6__prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__out1__v1_official' and merge.cfg='triggerana_dune10kt_1x2x6.fcl'",
    "dune.user": "dunepro",
    "merge.cfg": "triggerana_dune10kt_1x2x6.fcl",
    "core.group": "dune",
    "merge.limit": 5000,
    "core.run_type": "fardet-hd",
    "dune.campaign": "trg_mc_2025a_tpg",
    "merge.dataset": "fardet-hd:fardet-hd__trg_mc_2025a_tpg__trigger-primitives__v10_12_01d01__tpg_dune10kt_1x2x6__prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__out1__v1_official",
    "core.data_tier": "root-tuple",
    "core.file_type": "mc",
    "dune.requestid": "ritm2612698",
    "retention.class": "unknown",
    "art.process_name": "TPGen",
    "core.application": "art.reco",
    "core.data_stream": "out1",
    "core.file_format": "root",
    "dune.config_file": "tpg_dune10kt_1x2x6.fcl",
    "dune.output_status": "confirmed",
    "art.file_format_era": "ART_2011a",
    "core.application.name": "triggerAna",
    "dune_mc.detector_type": "fardet-hd",
    "art.file_format_version": 15,
    "core.application.family": "art",
    "core.application.version": "v10_12_01d01",
    "core.file_content_status": "good",
    "dune_mc.gen_fcl_filename": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA.fcl",
    "merge.hash": "9dd448044265ab72b0d7594bb481a3004af970dc1a79ddd6fe11d73f23961725",
    "merge.group": 249
  },
  "parents": [
    {
      "fid": "Sfuo81sZSu2hfYQO",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c8.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "t8uqYaZ7SVuDCusz",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c9.root",
      "namespace": "fardet-hd"
    },
    {
      "fid": "aR18iL5NQjuX8jjc",
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s050000_l005000_20251126T012840.root",
      "namespace": "fardet-hd"
    }
  ],
  "inputs": [
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/b4/21/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c8.root",
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/f6/04/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c9.root",
    "root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/6b/ca/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s050000_l005000_20251126T012840.root"
  ],
  "outputs": [
    {
      "name": "prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root",
      "metadata": {
        "core.data_tier": "root-tuple",
        "core.application.name": "triggerAna",
        "core.file_format": "root"
      }
    }
  ],
  "settings": {
    "streaming": true,
    "method": "hadd",
    "cmd": "hadd -f {output} {inputs}"
  }
}Setting up larsoft UPS area... /cvmfs/larsoft.opensciencegrid.org
Setting up DUNE UPS area... /cvmfs/dune.opensciencegrid.org/products/dune/
hadd Target file: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root
hadd compression setting for all output: 1
hadd Source file 1: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/b4/21/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c8.root
hadd Source file 2: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/f6/04/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c9.root
hadd Source file 3: root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/6b/ca/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s050000_l005000_20251126T012840.root
hadd Target path: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root:/
hadd Target path: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root:/triggerAna
hadd Target path: prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root:/triggerAna/TriggerPrimitives
Merging 3 files into prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root using method hadd
hadd -f prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/b4/21/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c8.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/f6/04/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s045000_l005000_20251126T012649_c9.root root://eospublic.cern.ch:1094//eos/experiment/neutplatform/protodune/dune/fardet-hd/6b/ca/prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__reco__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__CENTRAL_prod_s050000_l005000_20251126T012840.root
Processing output file prodbackground_radiological_decay0_dune10kt_1x2x6_centralAPA__trg_mc_2025a_tpg__tpg_dune10kt_1x2x6__triggerAna__v10_12_01d01__triggerana_dune10kt_1x2x6__v10_12_01d01__ntuple__testCENTRAL2_20251204T014000_f249.root
merged pass1_CERN_000250.json