From 59d98c4bb6d4ec46d2c0634b9ec0fef68b85a03c Mon Sep 17 00:00:00 2001 From: jsamudio Date: Fri, 22 Dec 2023 01:42:43 -0600 Subject: [PATCH] Introduce HCAL only workflows using Alpaka-based PF clustering First attempt to add full reco workflow Fix to new workflows Update readme with new Alpaka-based PF workflows Remove extraneous process modifier and add new sequences for Legacy comparisons Fix to Alpaka Full Reco Workflow Removes gpu modifier in reco step, and note that once PF is included in HLT we can move the digi step from gpu to alpaka modifier Adjust HCAL-only alpaka workflow numbers Adjust full-reco Alpaka workflow Adjust full reco workflow number Remove harvesting in .424 Fix harvesting to none --- Configuration/PyReleaseValidation/README.md | 4 + .../python/upgradeWorkflowComponents.py | 80 ++++++++++++++++++- .../python/Reconstruction_cff.py | 9 +++ .../python/hcalGlobalReco_cff.py | 3 + .../Configuration/python/hcalLocalReco_cff.py | 3 + .../python/particleFlowCluster_cff.py | 3 + .../python/pfClusterHBHEAlpaka_cff.py | 76 ++++++++++++++++-- .../python/PFClusterValidation_cff.py | 5 ++ .../python/pfCaloGPUComparisonTask_cfi.py | 14 ++++ 9 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 Validation/RecoParticleFlow/python/pfCaloGPUComparisonTask_cfi.py diff --git a/Configuration/PyReleaseValidation/README.md b/Configuration/PyReleaseValidation/README.md index 041fefe015376..298e983ea99ec 100644 --- a/Configuration/PyReleaseValidation/README.md +++ b/Configuration/PyReleaseValidation/README.md @@ -34,6 +34,10 @@ The offsets currently in use are: * 0.403: Alpaka, pixel only quadruplets, portable vs. CPU validation * 0.404: Alpaka, pixel only quadruplets, portable profiling * 0.412: Alpaka, ECAL only, portable +* 0.422: Alpaka, HCAL only, portable +* 0.423: Alpaka, HCAL only, portable vs CPU validation +* 0.424: Alpaka, HCAL only, portable profiling +* 0.492: Alpaka, full reco with pixel quadruplets * 0.5: Pixel tracking only + 0.1 * 0.501: Patatrack, pixel only quadruplets, on CPU * 0.502: Patatrack, pixel only quadruplets, with automatic offload to GPU if available diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 9a9084b0d4978..b48be1d603647 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -1063,7 +1063,7 @@ def setup_(self, step, stepName, stepDict, k, properties): upgradeWFs['PatatrackECALOnlyAlpaka'] = PatatrackWorkflow( digi = { # customize the ECAL Local Reco part of the HLT menu for Alpaka - '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka }, reco = { '-s': 'RAW2DIGI:RawToDigi_ecalOnly,RECO:reconstruction_ecalOnly,VALIDATION:@ecalOnlyValidation,DQM:@ecalOnly', @@ -1232,6 +1232,78 @@ def setup_(self, step, stepName, stepDict, k, properties): offset = 0.524, ) +# HCAL-PF Only workflow running HCAL local reco on GPU and PF with Alpaka with DQM and Validation +# - HLT-alpaka +# - HCAL-only reconstruction using Alpaka with DQM and Validation +upgradeWFs['PatatrackHCALOnlyAlpakaValidation'] = PatatrackWorkflow( + digi = { + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + }, + reco = { + '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation,DQM:@hcalOnly+@hcal2Only', + '--procModifiers': 'alpaka' + }, + harvest = { + '-s': 'HARVESTING:@hcalOnlyValidation' + }, + suffix = 'Patatrack_HCALOnlyAlpaka_Validation', + offset = 0.422, +) + +# HCAL-PF Only workflow running HCAL local reco and PF with Alpaka with cluster level-validation +# - HLT-alpaka +# - HCAL-only reconstruction using GPU and Alpaka with DQM and Validation for PF Alpaka vs CPU comparisons +upgradeWFs['PatatrackHCALOnlyGPUandAlpakaValidation'] = PatatrackWorkflow( + digi = { + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + }, + reco = { + '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnlyLegacy+reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation+pfClusterHBHEOnlyAlpakaComparisonSequence,DQM:@hcalOnly+@hcal2Only', + '--procModifiers': 'alpaka' + }, + harvest = { + '-s': 'HARVESTING:@hcalOnlyValidation' + }, + suffix = 'Patatrack_HCALOnlyGPUandAlpaka_Validation', + offset = 0.423, +) + +# HCAL-PF Only workflow running HCAL local reco on CPU and PF with Alpaka slimmed for benchmarking +# - HLT-alpaka +# - HCAL-only reconstruction using Alpaka +upgradeWFs['PatatrackHCALOnlyAlpakaProfiling'] = PatatrackWorkflow( + digi = { + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + }, + reco = { + '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly', + '--procModifiers': 'alpaka' + }, + harvest = None, + suffix = 'Patatrack_HCALOnlyAlpaka_Profiling', + offset = 0.424, +) + +# Workflow running the Pixel quadruplets, ECAL and HCAL reconstruction on GPU (optional), PF using Alpaka, together with the full offline reconstruction on CPU +# - HLT on GPU (optional) +# - reconstruction on Alpaka, with DQM and validation +# - harvesting +upgradeWFs['PatatrackFullRecoAlpaka'] = PatatrackWorkflow( + digi = { + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + }, + reco = { + # skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction + '-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM', + '--procModifiers': 'alpaka,pixelNtupletFit' + }, + harvest = { + # skip the @pixelTrackingOnlyDQM harvesting + }, + suffix = 'Patatrack_FullRecoAlpaka', + offset = 0.492, +) + # Workflow running the Pixel quadruplets, ECAL and HCAL reconstruction on CPU # - HLT on CPU # - reconstruction on CPU, with DQM and validation @@ -1541,7 +1613,7 @@ def setup_(self, step, stepName, stepDict, k, properties): upgradeWFs['PatatrackPixelOnlyAlpaka'] = PatatrackWorkflow( digi = { - '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka }, reco = { '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM', @@ -1556,7 +1628,7 @@ def setup_(self, step, stepName, stepDict, k, properties): upgradeWFs['PatatrackPixelOnlyAlpakaValidation'] = PatatrackWorkflow( digi = { - '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka }, reco = { '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM', @@ -1571,7 +1643,7 @@ def setup_(self, step, stepName, stepDict, k, properties): upgradeWFs['PatatrackPixelOnlyAlpakaProfiling'] = PatatrackWorkflow( digi = { - '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka + '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka }, reco = { '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly', diff --git a/Configuration/StandardSequences/python/Reconstruction_cff.py b/Configuration/StandardSequences/python/Reconstruction_cff.py index 8bd230ea3ce09..158d2261913d4 100644 --- a/Configuration/StandardSequences/python/Reconstruction_cff.py +++ b/Configuration/StandardSequences/python/Reconstruction_cff.py @@ -243,7 +243,16 @@ pfClusteringHBHEHFOnlyTask ) +# define secondary validation task running only Legacy +reconstruction_hcalOnlyLegacyTask = cms.Task( + bunchSpacingProducer, + offlineBeamSpot, + hcalOnlyLegacyLocalRecoTask, + hcalOnlyLegacyGlobalRecoTask, + pfClusteringHBHEHFOnlyLegacyTask) + reconstruction_hcalOnly = cms.Sequence(reconstruction_hcalOnlyTask) +reconstruction_hcalOnlyLegacy = cms.Sequence(reconstruction_hcalOnlyLegacyTask) #need a fully expanded sequence copy modulesToRemove = list() # copy does not work well diff --git a/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py b/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py index c876f79eb5422..a665bb5c1c296 100644 --- a/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py +++ b/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py @@ -19,6 +19,9 @@ run3_HB.toReplaceWith(hbhereco.cpu, _phase1_hbheprereco) run3_HB.toReplaceWith(hcalOnlyGlobalRecoTask, cms.Task(hbhereco)) +#-- Legacy HCAL Only Task +hcalOnlyLegacyGlobalRecoTask = hcalOnlyGlobalRecoTask.copy() + #--- for Run 3 on GPU from Configuration.ProcessModifiers.gpu_cff import gpu diff --git a/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py b/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py index 45b3e511c6874..87b829c77a12b 100644 --- a/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py +++ b/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py @@ -58,6 +58,9 @@ from Configuration.Eras.Modifier_run3_HB_cff import run3_HB run3_HB.toReplaceWith(hcalLocalRecoTask, _run3_hcalLocalRecoTask) +#--- Legacy HCAL Only Task +hcalOnlyLegacyLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco]) + #--- for Run 3 on GPU from Configuration.ProcessModifiers.gpu_cff import gpu diff --git a/RecoParticleFlow/PFClusterProducer/python/particleFlowCluster_cff.py b/RecoParticleFlow/PFClusterProducer/python/particleFlowCluster_cff.py index 3436581964004..0b2a52db016a7 100644 --- a/RecoParticleFlow/PFClusterProducer/python/particleFlowCluster_cff.py +++ b/RecoParticleFlow/PFClusterProducer/python/particleFlowCluster_cff.py @@ -44,6 +44,9 @@ particleFlowClusterHF, particleFlowClusterHCALOnly) +#--- Legacy HCAL Only Task +pfClusteringHBHEHFOnlyLegacyTask = pfClusteringHBHEHFOnlyTask.copy() + pfClusteringHOTask = cms.Task(particleFlowRecHitHO,particleFlowClusterHO) pfClusteringHO = cms.Sequence(pfClusteringHOTask) diff --git a/RecoParticleFlow/PFClusterProducer/python/pfClusterHBHEAlpaka_cff.py b/RecoParticleFlow/PFClusterProducer/python/pfClusterHBHEAlpaka_cff.py index 631eee2cec974..e1c000b5a3d4a 100644 --- a/RecoParticleFlow/PFClusterProducer/python/pfClusterHBHEAlpaka_cff.py +++ b/RecoParticleFlow/PFClusterProducer/python/pfClusterHBHEAlpaka_cff.py @@ -11,9 +11,12 @@ from RecoParticleFlow.PFClusterProducer.pfClusterSoAProducer_cfi import pfClusterSoAProducer as _pfClusterSoAProducer from RecoParticleFlow.PFClusterProducer.legacyPFClusterProducer_cfi import legacyPFClusterProducer as _legacyPFClusterProducer -from RecoParticleFlow.PFClusterProducer.particleFlowCluster_cff import pfClusteringHBHEHFTask, particleFlowClusterHBHE, particleFlowRecHitHBHE, particleFlowClusterHCAL - +from RecoParticleFlow.PFClusterProducer.particleFlowCluster_cff import pfClusteringHBHEHFTask, pfClusteringHBHEHFOnlyTask, particleFlowClusterHBHE, particleFlowRecHitHBHE, particleFlowClusterHCAL, particleFlowClusterHBHEOnly, particleFlowRecHitHBHEOnly, particleFlowClusterHCALOnly +#Full Reco _alpaka_pfClusteringHBHEHFTask = pfClusteringHBHEHFTask.copy() +#HCAL Only +_alpaka_pfClusteringHBHEHFOnlyTask = pfClusteringHBHEHFOnlyTask.copy() + pfRecHitHCALParamsRecordSource = cms.ESSource('EmptyESSource', recordName = cms.string('PFRecHitHCALParamsRecord'), @@ -75,23 +78,86 @@ PFRecHitsLabelIn = 'pfRecHitSoAProducerHCAL' ) - +#Full Reco _alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALParamsRecordSource) _alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALTopologyRecordSource) _alpaka_pfClusteringHBHEHFTask.add(pfClusterParamsRecordSource) _alpaka_pfClusteringHBHEHFTask.add(hbheRecHitToSoA) _alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALParamsESProducer) +_alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALTopologyESProducer) _alpaka_pfClusteringHBHEHFTask.add(pfRecHitSoAProducerHCAL) _alpaka_pfClusteringHBHEHFTask.add(legacyPFRecHitProducer) _alpaka_pfClusteringHBHEHFTask.add(pfClusterParamsESProducer) _alpaka_pfClusteringHBHEHFTask.add(pfClusterSoAProducer) +_alpaka_pfClusteringHBHEHFTask.add(legacyPFClusterProducer) _alpaka_pfClusteringHBHEHFTask.remove(particleFlowRecHitHBHE) _alpaka_pfClusteringHBHEHFTask.remove(particleFlowClusterHBHE) _alpaka_pfClusteringHBHEHFTask.remove(particleFlowClusterHCAL) -_alpaka_pfClusteringHBHEHFTask.add(particleFlowClusterHBHE) _alpaka_pfClusteringHBHEHFTask.add(particleFlowClusterHCAL) -alpaka.toReplaceWith(particleFlowClusterHBHE, legacyPFClusterProducer) +alpaka.toModify(particleFlowClusterHCAL, clustersSource = "legacyPFClusterProducer") alpaka.toReplaceWith(pfClusteringHBHEHFTask, _alpaka_pfClusteringHBHEHFTask) + +#HCAL Only + +from Configuration.Eras.Modifier_run3_HB_cff import run3_HB + +hbheOnlyRecHitToSoA = _hcalRecHitSoAProducer.clone( + src = "hbheprereco" + ) +run3_HB.toModify(hbheOnlyRecHitToSoA, + src = "hbhereco" + ) + +pfRecHitSoAProducerHBHEOnly = _pfRecHitSoAProducerHCAL.clone( + producers = cms.VPSet( + cms.PSet( + src = cms.InputTag("hbheOnlyRecHitToSoA"), + params = cms.ESInputTag("pfRecHitHCALParamsESProducer:"), + ) + ), + topology = "pfRecHitHCALTopologyESProducer:", + synchronise = cms.untracked.bool(False) + ) + +legacyPFRecHitProducerHBHEOnly = _legacyPFRecHitProducer.clone( + src = "pfRecHitSoAProducerHBHEOnly" + ) + +pfClusterSoAProducerHBHEOnly = _pfClusterSoAProducer.clone( + pfRecHits = 'pfRecHitSoAProducerHBHEOnly', + topology = "pfRecHitHCALTopologyESProducer:", + pfClusterParams = 'pfClusterParamsESProducer:', + synchronise = cms.bool(False) + ) + +legacyPFClusterProducerHBHEOnly = _legacyPFClusterProducer.clone( + src = 'pfClusterSoAProducerHBHEOnly', + pfClusterParams = 'pfClusterParamsESProducer:', + pfClusterBuilder = particleFlowClusterHBHE.pfClusterBuilder, + recHitsSource = 'legacyPFRecHitProducerHBHEOnly', + PFRecHitsLabelIn = 'pfRecHitSoAProducerHBHEOnly' + ) + +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALParamsRecordSource) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALTopologyRecordSource) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterParamsRecordSource) +_alpaka_pfClusteringHBHEHFOnlyTask.add(hbheOnlyRecHitToSoA) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALParamsESProducer) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALTopologyESProducer) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitSoAProducerHBHEOnly) +_alpaka_pfClusteringHBHEHFOnlyTask.add(legacyPFRecHitProducerHBHEOnly) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterParamsESProducer) +_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterSoAProducerHBHEOnly) +_alpaka_pfClusteringHBHEHFOnlyTask.add(legacyPFClusterProducerHBHEOnly) + +_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowRecHitHBHEOnly) +_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowClusterHBHEOnly) +_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowClusterHCALOnly) +_alpaka_pfClusteringHBHEHFOnlyTask.add(particleFlowClusterHCALOnly) + +alpaka.toModify(particleFlowClusterHCALOnly, clustersSource = "legacyPFClusterProducerHBHEOnly") + +alpaka.toReplaceWith(pfClusteringHBHEHFOnlyTask, _alpaka_pfClusteringHBHEHFOnlyTask) diff --git a/Validation/RecoParticleFlow/python/PFClusterValidation_cff.py b/Validation/RecoParticleFlow/python/PFClusterValidation_cff.py index a112724922aaa..bf0fca176c6c0 100644 --- a/Validation/RecoParticleFlow/python/PFClusterValidation_cff.py +++ b/Validation/RecoParticleFlow/python/PFClusterValidation_cff.py @@ -1,10 +1,15 @@ import FWCore.ParameterSet.Config as cms from Validation.RecoParticleFlow.pfClusterValidation_cfi import pfClusterValidation +from Validation.RecoParticleFlow.pfCaloGPUComparisonTask_cfi import pfClusterHBHEOnlyAlpakaComparison, pfClusterHBHEAlpakaComparison pfClusterValidationSequence = cms.Sequence( pfClusterValidation ) +pfClusterAlpakaComparisonSequence = cms.Sequence( pfClusterHBHEAlpakaComparison ) + pfClusterCaloOnlyValidation = pfClusterValidation.clone( pflowClusterHCAL = 'particleFlowClusterHCALOnly' ) pfClusterCaloOnlyValidationSequence = cms.Sequence( pfClusterCaloOnlyValidation ) + +pfClusterHBHEOnlyAlpakaComparisonSequence = cms.Sequence( pfClusterHBHEOnlyAlpakaComparison ) diff --git a/Validation/RecoParticleFlow/python/pfCaloGPUComparisonTask_cfi.py b/Validation/RecoParticleFlow/python/pfCaloGPUComparisonTask_cfi.py new file mode 100644 index 0000000000000..01c2cdaaa5d46 --- /dev/null +++ b/Validation/RecoParticleFlow/python/pfCaloGPUComparisonTask_cfi.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms + +from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer +pfClusterHBHEOnlyAlpakaComparison = DQMEDAnalyzer("PFCaloGPUComparisonTask", + pfClusterToken_ref = cms.untracked.InputTag('particleFlowClusterHBHEOnly'), + pfClusterToken_target = cms.untracked.InputTag('legacyPFClusterProducerHBHEOnly'), + pfCaloGPUCompDir = cms.untracked.string("pfClusterHBHEAlpakaV") +) + +pfClusterHBHEAlpakaComparison = DQMEDAnalyzer("PFCaloGPUComparisonTask", + pfClusterToken_ref = cms.untracked.InputTag('particleFlowClusterHBHE'), + pfClusterToken_target = cms.untracked.InputTag('legacyPFClusterProducer'), + pfCaloGPUCompDir = cms.untracked.string("pfClusterHBHEAlpakaV") +)