Skip to content

Commit

Permalink
Merge pull request #43701 from jsamudio/alpakaPFWorkflow
Browse files Browse the repository at this point in the history
Introduce matrix workflows for Alpaka-based Particle Flow
  • Loading branch information
cmsbuild authored Feb 14, 2024
2 parents 74821ae + 59d98c4 commit ca93802
Show file tree
Hide file tree
Showing 9 changed files with 188 additions and 9 deletions.
4 changes: 4 additions & 0 deletions Configuration/PyReleaseValidation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ The offsets currently in use are:
* 0.403: Alpaka, pixel only quadruplets, portable vs. CPU validation
* 0.404: Alpaka, pixel only quadruplets, portable profiling
* 0.412: Alpaka, ECAL only, portable
* 0.422: Alpaka, HCAL only, portable
* 0.423: Alpaka, HCAL only, portable vs CPU validation
* 0.424: Alpaka, HCAL only, portable profiling
* 0.492: Alpaka, full reco with pixel quadruplets
* 0.5: Pixel tracking only + 0.1
* 0.501: Patatrack, pixel only quadruplets, on CPU
* 0.502: Patatrack, pixel only quadruplets, with automatic offload to GPU if available
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
upgradeWFs['PatatrackECALOnlyAlpaka'] = PatatrackWorkflow(
digi = {
# customize the ECAL Local Reco part of the HLT menu for Alpaka
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_ecalOnly,RECO:reconstruction_ecalOnly,VALIDATION:@ecalOnlyValidation,DQM:@ecalOnly',
Expand Down Expand Up @@ -1238,6 +1238,78 @@ def setup_(self, step, stepName, stepDict, k, properties):
offset = 0.524,
)

# HCAL-PF Only workflow running HCAL local reco on GPU and PF with Alpaka with DQM and Validation
# - HLT-alpaka
# - HCAL-only reconstruction using Alpaka with DQM and Validation
upgradeWFs['PatatrackHCALOnlyAlpakaValidation'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation,DQM:@hcalOnly+@hcal2Only',
'--procModifiers': 'alpaka'
},
harvest = {
'-s': 'HARVESTING:@hcalOnlyValidation'
},
suffix = 'Patatrack_HCALOnlyAlpaka_Validation',
offset = 0.422,
)

# HCAL-PF Only workflow running HCAL local reco and PF with Alpaka with cluster level-validation
# - HLT-alpaka
# - HCAL-only reconstruction using GPU and Alpaka with DQM and Validation for PF Alpaka vs CPU comparisons
upgradeWFs['PatatrackHCALOnlyGPUandAlpakaValidation'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnlyLegacy+reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation+pfClusterHBHEOnlyAlpakaComparisonSequence,DQM:@hcalOnly+@hcal2Only',
'--procModifiers': 'alpaka'
},
harvest = {
'-s': 'HARVESTING:@hcalOnlyValidation'
},
suffix = 'Patatrack_HCALOnlyGPUandAlpaka_Validation',
offset = 0.423,
)

# HCAL-PF Only workflow running HCAL local reco on CPU and PF with Alpaka slimmed for benchmarking
# - HLT-alpaka
# - HCAL-only reconstruction using Alpaka
upgradeWFs['PatatrackHCALOnlyAlpakaProfiling'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly',
'--procModifiers': 'alpaka'
},
harvest = None,
suffix = 'Patatrack_HCALOnlyAlpaka_Profiling',
offset = 0.424,
)

# Workflow running the Pixel quadruplets, ECAL and HCAL reconstruction on GPU (optional), PF using Alpaka, together with the full offline reconstruction on CPU
# - HLT on GPU (optional)
# - reconstruction on Alpaka, with DQM and validation
# - harvesting
upgradeWFs['PatatrackFullRecoAlpaka'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
# skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
'-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
'--procModifiers': 'alpaka,pixelNtupletFit'
},
harvest = {
# skip the @pixelTrackingOnlyDQM harvesting
},
suffix = 'Patatrack_FullRecoAlpaka',
offset = 0.492,
)

# Workflow running the Pixel quadruplets, ECAL and HCAL reconstruction on CPU
# - HLT on CPU
# - reconstruction on CPU, with DQM and validation
Expand Down Expand Up @@ -1547,7 +1619,7 @@ def setup_(self, step, stepName, stepDict, k, properties):

upgradeWFs['PatatrackPixelOnlyAlpaka'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
Expand All @@ -1562,7 +1634,7 @@ def setup_(self, step, stepName, stepDict, k, properties):

upgradeWFs['PatatrackPixelOnlyAlpakaValidation'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
Expand All @@ -1577,7 +1649,7 @@ def setup_(self, step, stepName, stepDict, k, properties):

upgradeWFs['PatatrackPixelOnlyAlpakaProfiling'] = PatatrackWorkflow(
digi = {
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
'--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
},
reco = {
'-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly',
Expand Down
9 changes: 9 additions & 0 deletions Configuration/StandardSequences/python/Reconstruction_cff.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,16 @@
pfClusteringHBHEHFOnlyTask
)

# define secondary validation task running only Legacy
reconstruction_hcalOnlyLegacyTask = cms.Task(
bunchSpacingProducer,
offlineBeamSpot,
hcalOnlyLegacyLocalRecoTask,
hcalOnlyLegacyGlobalRecoTask,
pfClusteringHBHEHFOnlyLegacyTask)

reconstruction_hcalOnly = cms.Sequence(reconstruction_hcalOnlyTask)
reconstruction_hcalOnlyLegacy = cms.Sequence(reconstruction_hcalOnlyLegacyTask)

#need a fully expanded sequence copy
modulesToRemove = list() # copy does not work well
Expand Down
3 changes: 3 additions & 0 deletions RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
run3_HB.toReplaceWith(hbhereco.cpu, _phase1_hbheprereco)
run3_HB.toReplaceWith(hcalOnlyGlobalRecoTask, cms.Task(hbhereco))

#-- Legacy HCAL Only Task
hcalOnlyLegacyGlobalRecoTask = hcalOnlyGlobalRecoTask.copy()

#--- for Run 3 on GPU
from Configuration.ProcessModifiers.gpu_cff import gpu

Expand Down
3 changes: 3 additions & 0 deletions RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@
from Configuration.Eras.Modifier_run3_HB_cff import run3_HB
run3_HB.toReplaceWith(hcalLocalRecoTask, _run3_hcalLocalRecoTask)

#--- Legacy HCAL Only Task
hcalOnlyLegacyLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco])

#--- for Run 3 on GPU
from Configuration.ProcessModifiers.gpu_cff import gpu

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@
particleFlowClusterHF,
particleFlowClusterHCALOnly)

#--- Legacy HCAL Only Task
pfClusteringHBHEHFOnlyLegacyTask = pfClusteringHBHEHFOnlyTask.copy()

pfClusteringHOTask = cms.Task(particleFlowRecHitHO,particleFlowClusterHO)
pfClusteringHO = cms.Sequence(pfClusteringHOTask)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,12 @@
from RecoParticleFlow.PFClusterProducer.pfClusterSoAProducer_cfi import pfClusterSoAProducer as _pfClusterSoAProducer
from RecoParticleFlow.PFClusterProducer.legacyPFClusterProducer_cfi import legacyPFClusterProducer as _legacyPFClusterProducer

from RecoParticleFlow.PFClusterProducer.particleFlowCluster_cff import pfClusteringHBHEHFTask, particleFlowClusterHBHE, particleFlowRecHitHBHE, particleFlowClusterHCAL

from RecoParticleFlow.PFClusterProducer.particleFlowCluster_cff import pfClusteringHBHEHFTask, pfClusteringHBHEHFOnlyTask, particleFlowClusterHBHE, particleFlowRecHitHBHE, particleFlowClusterHCAL, particleFlowClusterHBHEOnly, particleFlowRecHitHBHEOnly, particleFlowClusterHCALOnly
#Full Reco
_alpaka_pfClusteringHBHEHFTask = pfClusteringHBHEHFTask.copy()
#HCAL Only
_alpaka_pfClusteringHBHEHFOnlyTask = pfClusteringHBHEHFOnlyTask.copy()


pfRecHitHCALParamsRecordSource = cms.ESSource('EmptyESSource',
recordName = cms.string('PFRecHitHCALParamsRecord'),
Expand Down Expand Up @@ -75,23 +78,86 @@
PFRecHitsLabelIn = 'pfRecHitSoAProducerHCAL'
)


#Full Reco
_alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALParamsRecordSource)
_alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALTopologyRecordSource)
_alpaka_pfClusteringHBHEHFTask.add(pfClusterParamsRecordSource)
_alpaka_pfClusteringHBHEHFTask.add(hbheRecHitToSoA)
_alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALParamsESProducer)
_alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALTopologyESProducer)
_alpaka_pfClusteringHBHEHFTask.add(pfRecHitSoAProducerHCAL)
_alpaka_pfClusteringHBHEHFTask.add(legacyPFRecHitProducer)
_alpaka_pfClusteringHBHEHFTask.add(pfClusterParamsESProducer)
_alpaka_pfClusteringHBHEHFTask.add(pfClusterSoAProducer)
_alpaka_pfClusteringHBHEHFTask.add(legacyPFClusterProducer)

_alpaka_pfClusteringHBHEHFTask.remove(particleFlowRecHitHBHE)
_alpaka_pfClusteringHBHEHFTask.remove(particleFlowClusterHBHE)
_alpaka_pfClusteringHBHEHFTask.remove(particleFlowClusterHCAL)
_alpaka_pfClusteringHBHEHFTask.add(particleFlowClusterHBHE)
_alpaka_pfClusteringHBHEHFTask.add(particleFlowClusterHCAL)

alpaka.toReplaceWith(particleFlowClusterHBHE, legacyPFClusterProducer)
alpaka.toModify(particleFlowClusterHCAL, clustersSource = "legacyPFClusterProducer")

alpaka.toReplaceWith(pfClusteringHBHEHFTask, _alpaka_pfClusteringHBHEHFTask)

#HCAL Only

from Configuration.Eras.Modifier_run3_HB_cff import run3_HB

hbheOnlyRecHitToSoA = _hcalRecHitSoAProducer.clone(
src = "hbheprereco"
)
run3_HB.toModify(hbheOnlyRecHitToSoA,
src = "hbhereco"
)

pfRecHitSoAProducerHBHEOnly = _pfRecHitSoAProducerHCAL.clone(
producers = cms.VPSet(
cms.PSet(
src = cms.InputTag("hbheOnlyRecHitToSoA"),
params = cms.ESInputTag("pfRecHitHCALParamsESProducer:"),
)
),
topology = "pfRecHitHCALTopologyESProducer:",
synchronise = cms.untracked.bool(False)
)

legacyPFRecHitProducerHBHEOnly = _legacyPFRecHitProducer.clone(
src = "pfRecHitSoAProducerHBHEOnly"
)

pfClusterSoAProducerHBHEOnly = _pfClusterSoAProducer.clone(
pfRecHits = 'pfRecHitSoAProducerHBHEOnly',
topology = "pfRecHitHCALTopologyESProducer:",
pfClusterParams = 'pfClusterParamsESProducer:',
synchronise = cms.bool(False)
)

legacyPFClusterProducerHBHEOnly = _legacyPFClusterProducer.clone(
src = 'pfClusterSoAProducerHBHEOnly',
pfClusterParams = 'pfClusterParamsESProducer:',
pfClusterBuilder = particleFlowClusterHBHE.pfClusterBuilder,
recHitsSource = 'legacyPFRecHitProducerHBHEOnly',
PFRecHitsLabelIn = 'pfRecHitSoAProducerHBHEOnly'
)

_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALParamsRecordSource)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALTopologyRecordSource)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterParamsRecordSource)
_alpaka_pfClusteringHBHEHFOnlyTask.add(hbheOnlyRecHitToSoA)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALParamsESProducer)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALTopologyESProducer)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitSoAProducerHBHEOnly)
_alpaka_pfClusteringHBHEHFOnlyTask.add(legacyPFRecHitProducerHBHEOnly)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterParamsESProducer)
_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterSoAProducerHBHEOnly)
_alpaka_pfClusteringHBHEHFOnlyTask.add(legacyPFClusterProducerHBHEOnly)

_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowRecHitHBHEOnly)
_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowClusterHBHEOnly)
_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowClusterHCALOnly)
_alpaka_pfClusteringHBHEHFOnlyTask.add(particleFlowClusterHCALOnly)

alpaka.toModify(particleFlowClusterHCALOnly, clustersSource = "legacyPFClusterProducerHBHEOnly")

alpaka.toReplaceWith(pfClusteringHBHEHFOnlyTask, _alpaka_pfClusteringHBHEHFOnlyTask)
5 changes: 5 additions & 0 deletions Validation/RecoParticleFlow/python/PFClusterValidation_cff.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import FWCore.ParameterSet.Config as cms
from Validation.RecoParticleFlow.pfClusterValidation_cfi import pfClusterValidation
from Validation.RecoParticleFlow.pfCaloGPUComparisonTask_cfi import pfClusterHBHEOnlyAlpakaComparison, pfClusterHBHEAlpakaComparison

pfClusterValidationSequence = cms.Sequence( pfClusterValidation )

pfClusterAlpakaComparisonSequence = cms.Sequence( pfClusterHBHEAlpakaComparison )

pfClusterCaloOnlyValidation = pfClusterValidation.clone(
pflowClusterHCAL = 'particleFlowClusterHCALOnly'
)

pfClusterCaloOnlyValidationSequence = cms.Sequence( pfClusterCaloOnlyValidation )

pfClusterHBHEOnlyAlpakaComparisonSequence = cms.Sequence( pfClusterHBHEOnlyAlpakaComparison )
14 changes: 14 additions & 0 deletions Validation/RecoParticleFlow/python/pfCaloGPUComparisonTask_cfi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import FWCore.ParameterSet.Config as cms

from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer
pfClusterHBHEOnlyAlpakaComparison = DQMEDAnalyzer("PFCaloGPUComparisonTask",
pfClusterToken_ref = cms.untracked.InputTag('particleFlowClusterHBHEOnly'),
pfClusterToken_target = cms.untracked.InputTag('legacyPFClusterProducerHBHEOnly'),
pfCaloGPUCompDir = cms.untracked.string("pfClusterHBHEAlpakaV")
)

pfClusterHBHEAlpakaComparison = DQMEDAnalyzer("PFCaloGPUComparisonTask",
pfClusterToken_ref = cms.untracked.InputTag('particleFlowClusterHBHE'),
pfClusterToken_target = cms.untracked.InputTag('legacyPFClusterProducer'),
pfCaloGPUCompDir = cms.untracked.string("pfClusterHBHEAlpakaV")
)

0 comments on commit ca93802

Please sign in to comment.