Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make repack process be able to generate new data tiers: L1SCOUT, HLTSCOUT #44381

Merged
merged 2 commits into from
Mar 15, 2024
Merged

Make repack process be able to generate new data tiers: L1SCOUT, HLTSCOUT #44381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b0e72f7
Make repack process be able to generate new data tiers: L1SCOUT, HLTS…
haozturk Mar 12, 2024
0255266
Don't inherit output commands from the event content in Repack process
haozturk Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions Configuration/DataProcessing/python/Repack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ def repackProcess(**args):

"""
from Configuration.EventContent.EventContent_cff import RAWEventContent
from Configuration.EventContent.EventContent_cff import HLTSCOUTEventContent
from Configuration.EventContent.EventContent_cff import L1SCOUTEventContent
process = cms.Process("REPACK")
process.load("FWCore.MessageLogger.MessageLogger_cfi")

Expand All @@ -42,6 +44,16 @@ def repackProcess(**args):
fileNames = cms.untracked.vstring()
)

defaultDataTier = "RAW"

# Should we default to something if dataTier arg isn't provided?
dataTier = args.get('dataTier', defaultDataTier)
eventContent = RAWEventContent
if dataTier == "HLTSCOUT":
eventContent = HLTSCOUTEventContent
elif dataTier == "L1SCOUT":
eventContent = L1SCOUTEventContent

outputs = args.get('outputs', [])

if len(outputs) > 0:
Expand All @@ -55,12 +67,13 @@ def repackProcess(**args):

outputModule = cms.OutputModule(
"PoolOutputModule",
compressionAlgorithm=copy.copy(RAWEventContent.compressionAlgorithm),
compressionLevel=copy.copy(RAWEventContent.compressionLevel),
compressionAlgorithm=copy.copy(eventContent.compressionAlgorithm),
compressionLevel=copy.copy(eventContent.compressionLevel),
fileName = cms.untracked.string("%s.root" % moduleLabel)
)

outputModule.dataset = cms.untracked.PSet(dataTier = cms.untracked.string("RAW"))

outputModule.dataset = cms.untracked.PSet(dataTier = cms.untracked.string(dataTier))

if maxSize != None:
outputModule.maxSize = cms.untracked.int32(maxSize)
Expand All @@ -75,3 +88,6 @@ def repackProcess(**args):
process.outputPath += outputModule

return process



22 changes: 17 additions & 5 deletions Configuration/DataProcessing/test/RunRepack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,28 @@ class RunRepack:
def __init__(self):
self.selectEvents = None
self.inputLFN = None
self.dataTier = None

def __call__(self):
if self.inputLFN == None:
msg = "No --lfn specified"
raise RuntimeError(msg)
allowedDataTiers = ["RAW", "HLTSCOUT", "L1SCOUT"]
if self.dataTier == None:
self.dataTier = "RAW"
elif self.dataTier not in allowedDataTiers:
msg = f"{self.dataTier} isn't an allowed datatier for repacking. Allowed data tiers: {allowedDataTiers}"
raise RuntimeError(msg)

outputs = []
outputs.append( { 'moduleLabel' : "write_PrimDS1_RAW" } )
outputs.append( { 'moduleLabel' : "write_PrimDS2_RAW" } )
outputs.append( { 'moduleLabel' : f"write_PrimDS1_{self.dataTier}" } )
outputs.append( { 'moduleLabel' : f"write_PrimDS2_{self.dataTier}" } )
if self.selectEvents != None:
outputs[0]['selectEvents'] = self.selectEvents.split(',')
outputs[1]['selectEvents'] = self.selectEvents.split(',')

try:
process = repackProcess(outputs = outputs)
process = repackProcess(outputs = outputs, dataTier = self.dataTier)
except Exception as ex:
msg = "Error creating process for Repack:\n"
msg += str(ex)
Expand All @@ -54,7 +61,7 @@ def __call__(self):


if __name__ == '__main__':
valid = ["select-events=", "lfn="]
valid = ["select-events=", "lfn=", "data-tier="]

usage = \
"""
Expand All @@ -63,9 +70,10 @@ def __call__(self):
Where options are:
--select-events (option, event selection based on trigger paths)
--lfn=/store/input/lfn
--data-tier=RAW|HLTSCOUT|L1SCOUT

Example:
python RunRepack.py --select-events HLT:path1,HLT:path2 --lfn /store/whatever
python RunRepack.py --select-events HLT:path1,HLT:path2 --lfn /store/whatever --data-tier RAW|HLTSCOUT|L1SCOUT

"""
try:
Expand All @@ -83,5 +91,9 @@ def __call__(self):
repackinator.selectEvents = arg
if opt == "--lfn" :
repackinator.inputLFN = arg
if opt == "--data-tier" :
repackinator.dataTier = arg

repackinator()


25 changes: 25 additions & 0 deletions Configuration/EventContent/python/EventContent_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,30 @@ def SwapKeepAndDrop(l):
'keep DetIds_hltSiStripRawToDigi_*_*'
])

#
# HLTSCOUT Data Tier definition
#
#
HLTSCOUTEventContent = cms.PSet(
outputCommands = cms.untracked.vstring('drop *'),
splitLevel = cms.untracked.int32(0),
compressionAlgorithm=cms.untracked.string("LZMA"),
compressionLevel=cms.untracked.int32(4)
)
HLTSCOUTEventContent.outputCommands.extend(HLTriggerRAW.outputCommands)
Copy link
Contributor

@mmusich mmusich Mar 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we've been discussing internally within TSG that this could be just HLTScouting instead of HLTriggerRAWor even better we could add a new HLTriggerHLTSCOUT to HLTrigger_EventContent_cff.py replacing HLTScouting (same keep instructions but adding a 'drop *_hlt*_*_*' at the beginning). This would need to be done internally by TSG and can go as a refinement after this PR is integrated.


#
# L1SCOUT Data Tier definition
#
#
L1SCOUTEventContent = cms.PSet(
outputCommands = cms.untracked.vstring('drop *'),
splitLevel = cms.untracked.int32(0),
compressionAlgorithm=cms.untracked.string("LZMA"),
compressionLevel=cms.untracked.int32(4)
)
L1SCOUTEventContent.outputCommands.extend(L1TriggerRAW.outputCommands)
Copy link
Contributor

@Mmiglio Mmiglio Mar 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall this be changed in a different PR? The L1Scout event content is different from the L1Trigger event content.
For example, we would need something on the line of these commands

"keep *_GmtUnpacker_*_*"
"keep *_CaloUnpacker_*_*",
"keep *_BmtfStubsUnpacker_*_*",

I can pick a different name for the modules if needed.


#
#
# HLTONLY Data Tier definition
Expand Down Expand Up @@ -945,3 +969,4 @@ def SwapKeepAndDrop(l):
fastSim.toModify(_entry, outputCommands = _entry.outputCommands + fastSimEC.dropSimDigis)
for _entry in [MINIAODEventContent, MINIAODSIMEventContent]:
fastSim.toModify(_entry, outputCommands = _entry.outputCommands + fastSimEC.dropPatTrigger)