Alternative options for creating a serial copy of alpaka modules

Configuration/ProcessModifiers/python/alpakaValidationPixel_cff.py

@@ -0,0 +1,6 @@
+import FWCore.ParameterSet.Config as cms
+
+# This modifier chain is for turning on DQM modules used for alpaka device/host validation for pixels
+
+alpakaValidationPixel = cms.Modifier()
+

Configuration/ProcessModifiers/python/alpakaValidation_cff.py

@@ -0,0 +1,11 @@
+import FWCore.ParameterSet.Config as cms
+
+from Configuration.ProcessModifiers.alpaka_cff import *
+from Configuration.ProcessModifiers.alpakaValidationPixel_cff import *
+
+# This modifier chain is for turning on DQM modules used for alpaka device/host validation
+
+alpakaValidation =  cms.ModifierChain(
+    alpaka,
+    alpakaValidationPixel
+)

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -894,6 +894,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
 #  - HLT on CPU
 #  - Pixel-only reconstruction on CPU, with DQM and validation
 #  - harvesting
+
 upgradeWFs['PatatrackPixelOnlyCPU'] = PatatrackWorkflow(
     digi = {
         # the HLT menu is already set up for using GPUs if available and if the "gpu" modifier is enabled
@@ -1513,6 +1514,53 @@ def setup_(self, step, stepName, stepDict, k, properties):
     offset = 0.597,
 )
 
+
+# Alpaka workflows
+
+upgradeWFs['PatatrackPixelOnlyAlpaka'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka'
+    },
+    reco = {
+        '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
+        '--procModifiers': 'alpaka'
+    },
+    harvest = {
+        '-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'
+    },
+    suffix = 'Patatrack_PixelOnlyAlpaka',
+    offset = 0.402,
+)
+
+upgradeWFs['PatatrackPixelOnlyAlpakaValidation'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka'
+    },
+    reco = {
+        '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
+        '--procModifiers': 'alpakaValidation'
+    },
+    harvest = {
+        '-s': 'HARVESTING:@trackingOnlyValidation+@pixelTrackingOnlyDQM'
+    },
+    suffix = 'Patatrack_PixelOnlyAlpaka_Validation',
+    offset = 0.403,
+)
+
+upgradeWFs['PatatrackPixelOnlyAlpakaProfiling'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka'
+    },
+    reco = {
+        '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly',
+        '--procModifiers': 'alpaka',
+        '--customise' : 'RecoTracker/Configuration/customizePixelOnlyForProfiling.customizePixelOnlyForProfilingGPUOnly'
+    },
+    harvest = None,
+    suffix = 'Patatrack_PixelOnlyAlpaka_Profiling',
+    offset = 0.404,
+)
+
 # end of Patatrack workflows
 
 class UpgradeWorkflow_ProdLike(UpgradeWorkflow):
@@ -2718,31 +2766,31 @@ def condition(self, fragment, stepList, key, hasHarvest):
     },
     '2022HI' : {
         'Geom' : 'DB:Extended',
-        'GT':'auto:phase1_2022_realistic_hi', 
+        'GT':'auto:phase1_2022_realistic_hi',
         'HLTmenu': '@fake2',
         'Era':'Run3_pp_on_PbPb',
         'BeamSpot': 'DBrealistic',
         'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
     },
     '2022HIRP' : {
         'Geom' : 'DB:Extended',
-        'GT':'auto:phase1_2022_realistic_hi', 
+        'GT':'auto:phase1_2022_realistic_hi',
         'HLTmenu': '@fake2',
         'Era':'Run3_pp_on_PbPb_approxSiStripClusters',
         'BeamSpot': 'DBrealistic',
         'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
     },
     '2023HI' : {
         'Geom' : 'DB:Extended',
-        'GT':'auto:phase1_2023_realistic_hi', 
+        'GT':'auto:phase1_2023_realistic_hi',
         'HLTmenu': '@fake2',
         'Era':'Run3_pp_on_PbPb',
         'BeamSpot': 'DBrealistic',
         'ScenToRun' : ['GenSim','Digi','RecoNano','HARVESTNano','ALCA'],
     },
     '2023HIRP' : {
         'Geom' : 'DB:Extended',
-        'GT':'auto:phase1_2023_realistic_hi', 
+        'GT':'auto:phase1_2023_realistic_hi',
         'HLTmenu': '@fake2',
         'Era':'Run3_pp_on_PbPb_approxSiStripClusters',
         'BeamSpot': 'DBrealistic',

DQM/SiPixelHeterogeneous/plugins/BuildFile.xml

@@ -5,8 +5,11 @@
 <use name="DataFormats/Common"/>
 <use name="Geometry/Records"/>
 <use name="Geometry/TrackerGeometryBuilder"/>
+<use name="DataFormats/TrackingRecHitSoA"/>
+<use name="DataFormats/TrackSoA"/>
+<use name="DataFormats/VertexSoA"/>
+<use name="DataFormats/BeamSpot"/>
 <use name="CUDADataFormats/TrackingRecHit"/>
 <use name="CUDADataFormats/Track"/>
 <use name="CUDADataFormats/Vertex"/>
-<use name="DataFormats/BeamSpot"/>
 <flags EDM_PLUGIN="1"/>

DQM/SiPixelHeterogeneous/plugins/SiPixelCompareRecHitsSoAAlpaka.cc

@@ -0,0 +1,244 @@
+#include "DQMServices/Core/interface/MonitorElement.h"
+#include "DQMServices/Core/interface/DQMEDAnalyzer.h"
+#include "DQMServices/Core/interface/DQMStore.h"
+#include "DataFormats/Math/interface/approx_atan2.h"
+#include "DataFormats/SiPixelDetId/interface/PixelSubdetector.h"
+#include "DataFormats/TrackerCommon/interface/TrackerTopology.h"
+#include "DataFormats/TrackingRecHitSoA/interface/TrackingRecHitsHost.h"
+#include "DataFormats/TrackingRecHitSoA/interface/TrackingRecHitsSoA.h"
+#include "FWCore/Framework/interface/Event.h"
+#include "FWCore/Framework/interface/Frameworkfwd.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
+#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
+#include "Geometry/CommonDetUnit/interface/PixelGeomDetUnit.h"
+#include "Geometry/CommonTopologies/interface/PixelTopology.h"
+#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h"
+#include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h"
+
+template <typename T>
+class SiPixelCompareRecHitsSoAAlpaka : public DQMEDAnalyzer {
+public:
+  using HitsOnHost = TrackingRecHitHost<T>;
+
+  explicit SiPixelCompareRecHitsSoAAlpaka(const edm::ParameterSet&);
+  ~SiPixelCompareRecHitsSoAAlpaka() override = default;
+  void dqmBeginRun(const edm::Run&, const edm::EventSetup&) override;
+  void bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& iRun, edm::EventSetup const& iSetup) override;
+  void analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) override;
+  static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
+
+private:
+  const edm::ESGetToken<TrackerGeometry, TrackerDigiGeometryRecord> geomToken_;
+  const edm::ESGetToken<TrackerTopology, TrackerTopologyRcd> topoToken_;
+  const edm::EDGetTokenT<HitsOnHost> tokenSoAHitsHost_;    //these two are both on Host but originally they have been
+  const edm::EDGetTokenT<HitsOnHost> tokenSoAHitsDevice_;  //produced on Host or on Device
+  const std::string topFolderName_;
+  const float mind2cut_;
+  static constexpr uint32_t invalidHit_ = std::numeric_limits<uint32_t>::max();
+  static constexpr float micron_ = 10000.;
+  const TrackerGeometry* tkGeom_ = nullptr;
+  const TrackerTopology* tTopo_ = nullptr;
+  MonitorElement* hnHits_;
+  MonitorElement* hBchargeL_[4];  // max 4 barrel hits
+  MonitorElement* hBsizexL_[4];
+  MonitorElement* hBsizeyL_[4];
+  MonitorElement* hBposxL_[4];
+  MonitorElement* hBposyL_[4];
+  MonitorElement* hFchargeD_[2][12];  // max 12 endcap disks
+  MonitorElement* hFsizexD_[2][12];
+  MonitorElement* hFsizeyD_[2][12];
+  MonitorElement* hFposxD_[2][12];
+  MonitorElement* hFposyD_[2][12];
+  //differences
+  MonitorElement* hBchargeDiff_;
+  MonitorElement* hFchargeDiff_;
+  MonitorElement* hBsizeXDiff_;
+  MonitorElement* hFsizeXDiff_;
+  MonitorElement* hBsizeYDiff_;
+  MonitorElement* hFsizeYDiff_;
+  MonitorElement* hBposXDiff_;
+  MonitorElement* hFposXDiff_;
+  MonitorElement* hBposYDiff_;
+  MonitorElement* hFposYDiff_;
+};
+
+//
+// constructors
+//
+template <typename T>
+SiPixelCompareRecHitsSoAAlpaka<T>::SiPixelCompareRecHitsSoAAlpaka(const edm::ParameterSet& iConfig)
+    : geomToken_(esConsumes<TrackerGeometry, TrackerDigiGeometryRecord, edm::Transition::BeginRun>()),
+      topoToken_(esConsumes<TrackerTopology, TrackerTopologyRcd, edm::Transition::BeginRun>()),
+      tokenSoAHitsHost_(consumes(iConfig.getParameter<edm::InputTag>("pixelHitsSrcHost"))),
+      tokenSoAHitsDevice_(consumes(iConfig.getParameter<edm::InputTag>("pixelHitsSrcDevice"))),
+      topFolderName_(iConfig.getParameter<std::string>("topFolderName")),
+      mind2cut_(iConfig.getParameter<double>("minD2cut")) {}
+
+//
+// Begin Run
+//
+template <typename T>
+void SiPixelCompareRecHitsSoAAlpaka<T>::dqmBeginRun(const edm::Run& iRun, const edm::EventSetup& iSetup) {
+  tkGeom_ = &iSetup.getData(geomToken_);
+  tTopo_ = &iSetup.getData(topoToken_);
+}
+
+//
+// -- Analyze
+//
+template <typename T>
+void SiPixelCompareRecHitsSoAAlpaka<T>::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) {
+  const auto& rhsoaHandleHost = iEvent.getHandle(tokenSoAHitsHost_);
+  const auto& rhsoaHandleDevice = iEvent.getHandle(tokenSoAHitsDevice_);
+  if (not rhsoaHandleHost or not rhsoaHandleDevice) {
+    edm::LogWarning out("SiPixelCompareRecHitsSoAAlpaka");
+    if (not rhsoaHandleHost) {
+      out << "reference (Host) rechits not found; ";
+    }
+    if (not rhsoaHandleDevice) {
+      out << "target (Device) rechits not found; ";
+    }
+    out << "the comparison will not run.";
+    return;
+  }
+
+  auto const& rhsoaHost = *rhsoaHandleHost;
+  auto const& rhsoaDevice = *rhsoaHandleDevice;
+
+  auto const& soa2dHost = rhsoaHost.const_view();
+  auto const& soa2dDevice = rhsoaDevice.const_view();
+
+  uint32_t nHitsHost = soa2dHost.metadata().size();
+  uint32_t nHitsDevice = soa2dDevice.metadata().size();
+
+  hnHits_->Fill(nHitsHost, nHitsDevice);
+  auto detIds = tkGeom_->detUnitIds();
+  for (uint32_t i = 0; i < nHitsHost; i++) {
+    float minD = mind2cut_;
+    uint32_t matchedHit = invalidHit_;
+    uint16_t indHost = soa2dHost[i].detectorIndex();
+    float xLocalHost = soa2dHost[i].xLocal();
+    float yLocalHost = soa2dHost[i].yLocal();
+    for (uint32_t j = 0; j < nHitsDevice; j++) {
+      if (soa2dDevice.detectorIndex(j) == indHost) {
+        float dx = xLocalHost - soa2dDevice[j].xLocal();
+        float dy = yLocalHost - soa2dDevice[j].yLocal();
+        float distance = dx * dx + dy * dy;
+        if (distance < minD) {
+          minD = distance;
+          matchedHit = j;
+        }
+      }
+    }
+    DetId id = detIds[indHost];
+    uint32_t chargeHost = soa2dHost[i].chargeAndStatus().charge;
+    int16_t sizeXHost = std::ceil(float(std::abs(soa2dHost[i].clusterSizeX()) / 8.));
+    int16_t sizeYHost = std::ceil(float(std::abs(soa2dHost[i].clusterSizeY()) / 8.));
+    uint32_t chargeDevice = 0;
+    int16_t sizeXDevice = -99;
+    int16_t sizeYDevice = -99;
+    float xLocalDevice = -999.;
+    float yLocalDevice = -999.;
+    if (matchedHit != invalidHit_) {
+      chargeDevice = soa2dDevice[matchedHit].chargeAndStatus().charge;
+      sizeXDevice = std::ceil(float(std::abs(soa2dDevice[matchedHit].clusterSizeX()) / 8.));
+      sizeYDevice = std::ceil(float(std::abs(soa2dDevice[matchedHit].clusterSizeY()) / 8.));
+      xLocalDevice = soa2dDevice[matchedHit].xLocal();
+      yLocalDevice = soa2dDevice[matchedHit].yLocal();
+    }
+    switch (id.subdetId()) {
+      case PixelSubdetector::PixelBarrel:
+        hBchargeL_[tTopo_->pxbLayer(id) - 1]->Fill(chargeHost, chargeDevice);
+        hBsizexL_[tTopo_->pxbLayer(id) - 1]->Fill(sizeXHost, sizeXDevice);
+        hBsizeyL_[tTopo_->pxbLayer(id) - 1]->Fill(sizeYHost, sizeYDevice);
+        hBposxL_[tTopo_->pxbLayer(id) - 1]->Fill(xLocalHost, xLocalDevice);
+        hBposyL_[tTopo_->pxbLayer(id) - 1]->Fill(yLocalHost, yLocalDevice);
+        hBchargeDiff_->Fill(chargeHost - chargeDevice);
+        hBsizeXDiff_->Fill(sizeXHost - sizeXDevice);
+        hBsizeYDiff_->Fill(sizeYHost - sizeYDevice);
+        hBposXDiff_->Fill(micron_ * (xLocalHost - xLocalDevice));
+        hBposYDiff_->Fill(micron_ * (yLocalHost - yLocalDevice));
+        break;
+      case PixelSubdetector::PixelEndcap:
+        hFchargeD_[tTopo_->pxfSide(id) - 1][tTopo_->pxfDisk(id) - 1]->Fill(chargeHost, chargeDevice);
+        hFsizexD_[tTopo_->pxfSide(id) - 1][tTopo_->pxfDisk(id) - 1]->Fill(sizeXHost, sizeXDevice);
+        hFsizeyD_[tTopo_->pxfSide(id) - 1][tTopo_->pxfDisk(id) - 1]->Fill(sizeYHost, sizeYDevice);
+        hFposxD_[tTopo_->pxfSide(id) - 1][tTopo_->pxfDisk(id) - 1]->Fill(xLocalHost, xLocalDevice);
+        hFposyD_[tTopo_->pxfSide(id) - 1][tTopo_->pxfDisk(id) - 1]->Fill(yLocalHost, yLocalDevice);
+        hFchargeDiff_->Fill(chargeHost - chargeDevice);
+        hFsizeXDiff_->Fill(sizeXHost - sizeXDevice);
+        hFsizeYDiff_->Fill(sizeYHost - sizeYDevice);
+        hFposXDiff_->Fill(micron_ * (xLocalHost - xLocalDevice));
+        hFposYDiff_->Fill(micron_ * (yLocalHost - yLocalDevice));
+        break;
+    }
+  }
+}
+
+//
+// -- Book Histograms
+//
+template <typename T>
+void SiPixelCompareRecHitsSoAAlpaka<T>::bookHistograms(DQMStore::IBooker& iBook,
+                                                       edm::Run const& iRun,
+                                                       edm::EventSetup const& iSetup) {
+  iBook.cd();
+  iBook.setCurrentFolder(topFolderName_);
+
+  // clang-format off
+  //Global
+  hnHits_ = iBook.book2I("nHits", "HostvsDevice RecHits per event;#Host RecHits;#Device RecHits", 200, 0, 5000,200, 0, 5000);
+  //Barrel Layer
+  for(unsigned int il=0;il<tkGeom_->numberOfLayers(PixelSubdetector::PixelBarrel);il++){
+    hBchargeL_[il] = iBook.book2I(Form("recHitsBLay%dCharge",il+1), Form("HostvsDevice RecHits Charge Barrel Layer%d;Host Charge;Device Charge",il+1), 250, 0, 100000, 250, 0, 100000);
+    hBsizexL_[il] = iBook.book2I(Form("recHitsBLay%dSizex",il+1), Form("HostvsDevice RecHits SizeX Barrel Layer%d;Host SizeX;Device SizeX",il+1), 30, 0, 30, 30, 0, 30);
+    hBsizeyL_[il] = iBook.book2I(Form("recHitsBLay%dSizey",il+1), Form("HostvsDevice RecHits SizeY Barrel Layer%d;Host SizeY;Device SizeY",il+1), 30, 0, 30, 30, 0, 30);
+    hBposxL_[il] = iBook.book2D(Form("recHitsBLay%dPosx",il+1), Form("HostvsDevice RecHits x-pos in Barrel Layer%d;Host pos x;Device pos x",il+1), 200, -5, 5, 200,-5,5);
+    hBposyL_[il] = iBook.book2D(Form("recHitsBLay%dPosy",il+1), Form("HostvsDevice RecHits y-pos in Barrel Layer%d;Host pos y;Device pos y",il+1), 200, -5, 5, 200,-5,5);
+  }
+  //Endcaps
+  //Endcaps Disk
+  for(int is=0;is<2;is++){
+    int sign=is==0? -1:1;
+    for(unsigned int id=0;id<tkGeom_->numberOfLayers(PixelSubdetector::PixelEndcap);id++){
+      hFchargeD_[is][id] = iBook.book2I(Form("recHitsFDisk%+dCharge",id*sign+sign), Form("HostvsDevice RecHits Charge Endcaps Disk%+d;Host Charge;Device Charge",id*sign+sign), 250, 0, 100000, 250, 0, 100000);
+      hFsizexD_[is][id] = iBook.book2I(Form("recHitsFDisk%+dSizex",id*sign+sign), Form("HostvsDevice RecHits SizeX Endcaps Disk%+d;Host SizeX;Device SizeX",id*sign+sign), 30, 0, 30, 30, 0, 30);
+      hFsizeyD_[is][id] = iBook.book2I(Form("recHitsFDisk%+dSizey",id*sign+sign), Form("HostvsDevice RecHits SizeY Endcaps Disk%+d;Host SizeY;Device SizeY",id*sign+sign), 30, 0, 30, 30, 0, 30);
+      hFposxD_[is][id] = iBook.book2D(Form("recHitsFDisk%+dPosx",id*sign+sign), Form("HostvsDevice RecHits x-pos Endcaps Disk%+d;Host pos x;Device pos x",id*sign+sign), 200, -5, 5, 200, -5, 5);
+      hFposyD_[is][id] = iBook.book2D(Form("recHitsFDisk%+dPosy",id*sign+sign), Form("HostvsDevice RecHits y-pos Endcaps Disk%+d;Host pos y;Device pos y",id*sign+sign), 200, -5, 5, 200, -5, 5);
+    }
+  }
+  //1D differences
+  hBchargeDiff_ = iBook.book1D("rechitChargeDiffBpix","Charge differnce of rechits in BPix; rechit charge difference (Host - Device)", 101, -50.5, 50.5);
+  hFchargeDiff_ = iBook.book1D("rechitChargeDiffFpix","Charge differnce of rechits in FPix; rechit charge difference (Host - Device)", 101, -50.5, 50.5);
+  hBsizeXDiff_ = iBook.book1D("rechitsizeXDiffBpix","SizeX difference of rechits in BPix; rechit sizex difference (Host - Device)", 21, -10.5, 10.5);
+  hFsizeXDiff_ = iBook.book1D("rechitsizeXDiffFpix","SizeX difference of rechits in FPix; rechit sizex difference (Host - Device)", 21, -10.5, 10.5);
+  hBsizeYDiff_ = iBook.book1D("rechitsizeYDiffBpix","SizeY difference of rechits in BPix; rechit sizey difference (Host - Device)", 21, -10.5, 10.5);
+  hFsizeYDiff_ = iBook.book1D("rechitsizeYDiffFpix","SizeY difference of rechits in FPix; rechit sizey difference (Host - Device)", 21, -10.5, 10.5);
+  hBposXDiff_ = iBook.book1D("rechitsposXDiffBpix","x-position difference of rechits in BPix; rechit x-pos difference (Host - Device)", 1000, -10, 10);
+  hFposXDiff_ = iBook.book1D("rechitsposXDiffFpix","x-position difference of rechits in FPix; rechit x-pos difference (Host - Device)", 1000, -10, 10);
+  hBposYDiff_ = iBook.book1D("rechitsposYDiffBpix","y-position difference of rechits in BPix; rechit y-pos difference (Host - Device)", 1000, -10, 10);
+  hFposYDiff_ = iBook.book1D("rechitsposYDiffFpix","y-position difference of rechits in FPix; rechit y-pos difference (Host - Device)", 1000, -10, 10);
+}
+
+template<typename T>
+void SiPixelCompareRecHitsSoAAlpaka<T>::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
+  // monitorpixelRecHitsSoAAlpaka
+  edm::ParameterSetDescription desc;
+  desc.add<edm::InputTag>("pixelHitsSrcHost", edm::InputTag("siPixelRecHitsPreSplittingAlpakaSerial"));
+  desc.add<edm::InputTag>("pixelHitsSrcDevice", edm::InputTag("siPixelRecHitsPreSplittingAlpaka"));
+  desc.add<std::string>("topFolderName", "SiPixelHeterogeneous/PixelRecHitsCompareDeviceVSHost");
+  desc.add<double>("minD2cut", 0.0001);
+  descriptions.addWithDefaultLabel(desc);
+}
+
+using SiPixelPhase1CompareRecHitsSoAAlpaka = SiPixelCompareRecHitsSoAAlpaka<pixelTopology::Phase1>;
+using SiPixelPhase2CompareRecHitsSoAAlpaka = SiPixelCompareRecHitsSoAAlpaka<pixelTopology::Phase2>;
+using SiPixelHIonPhase1CompareRecHitsSoAAlpaka = SiPixelCompareRecHitsSoAAlpaka<pixelTopology::HIonPhase1>;
+
+#include "FWCore/Framework/interface/MakerMacros.h"
+DEFINE_FWK_MODULE(SiPixelPhase1CompareRecHitsSoAAlpaka);
+DEFINE_FWK_MODULE(SiPixelPhase2CompareRecHitsSoAAlpaka);
+DEFINE_FWK_MODULE(SiPixelHIonPhase1CompareRecHitsSoAAlpaka);