diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index bf1d7f5089ed4..60e1de3876af3 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -71,8 +71,10 @@ class Options: defaultOptions.inputEventContent = '' defaultOptions.dropDescendant = False defaultOptions.relval = None +defaultOptions.prefix = None defaultOptions.profile = None defaultOptions.heap_profile = None +defaultOptions.maxmem_profile = None defaultOptions.isRepacked = False defaultOptions.restoreRNDSeeds = False defaultOptions.donotDropOnInput = '' @@ -324,37 +326,13 @@ def heapProfileOptions(self): Function to add the jemalloc heap profile service so that you can dump in the middle of the run. """ - profileOpts = self._options.profile.split(':') + profileOpts = [] profilerStart = 1 profilerInterval = 100 - profilerFormat = None + profilerFormat = "jeprof_%s.heap" profilerJobFormat = None - if len(profileOpts): - #type, given as first argument is unused here - profileOpts.pop(0) - if len(profileOpts): - startEvent = profileOpts.pop(0) - if not startEvent.isdigit(): - raise Exception("%s is not a number" % startEvent) - profilerStart = int(startEvent) - if len(profileOpts): - eventInterval = profileOpts.pop(0) - if not eventInterval.isdigit(): - raise Exception("%s is not a number" % eventInterval) - profilerInterval = int(eventInterval) - if len(profileOpts): - profilerFormat = profileOpts.pop(0) - - if not profilerFormat: - profilerFormat = "%s___%s___%%I.heap" % ( - self._options.evt_type.replace("_cfi", ""), - hashlib.md5( - (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) + - str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8') - ).hexdigest() - ) if not profilerJobFormat and profilerFormat.endswith(".heap"): profilerJobFormat = profilerFormat.replace(".heap", "_EndOfJob.heap") elif not profilerJobFormat: diff --git a/Configuration/Applications/python/Options.py b/Configuration/Applications/python/Options.py index 5abc482d9d7bb..2ba1f0240ced9 100644 --- a/Configuration/Applications/python/Options.py +++ b/Configuration/Applications/python/Options.py @@ -424,10 +424,16 @@ expertSettings.add_argument("--heap_profile", help="add the JeProfService with the parameter provided PROFILER:START:STEP:PEREVENOUTPUTFORMAT:ENDOFJOBOUTPUTFORMAT", - default=None, - type=str, + default=False, + action="store_true", dest="heap_profile") +expertSettings.add_argument("--maxmem_profile", + help="add the PerfTools/MaxMemoryPreload monitor", + default=False, + action="store_true", + dest="maxmem_profile") + expertSettings.add_argument("--io", help="Create a json file with io informations", default=None, diff --git a/Configuration/Applications/python/cmsDriverOptions.py b/Configuration/Applications/python/cmsDriverOptions.py index eaf702762de64..412ef9f036707 100755 --- a/Configuration/Applications/python/cmsDriverOptions.py +++ b/Configuration/Applications/python/cmsDriverOptions.py @@ -17,7 +17,7 @@ def checkModifier(era): def checkOptions(): return - + def adaptOptions(): return @@ -34,7 +34,7 @@ def OptionsFromCommand(command): def OptionsFromCommandLine(): import sys options=OptionsFromItems(sys.argv[1:]) - # memorize the command line arguments + # memorize the command line arguments options.arguments = reduce(lambda x, y: x+' '+y, sys.argv[1:]) return options @@ -60,8 +60,8 @@ def OptionsFromItems(items): if options.triggerResultsProcess == None and "ALCAOUTPUT" in options.step: print("ERROR: If ALCA splitting is requested, the name of the process in which the alca producers ran needs to be specified. E.g. via --triggerResultsProcess RECO") sys.exit(1) - - if not options.evt_type: + + if not options.evt_type: options.evt_type=sys.argv[1] #now adjust the given parameters before passing it to the ConfigBuilder @@ -123,7 +123,7 @@ def OptionsFromItems(items): # add on the end of job sequence... addEndJob = True - if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "": + if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "": addEndJob = False if ("SKIM" in options.step and not "RECO" in options.step): addEndJob = False @@ -131,7 +131,7 @@ def OptionsFromItems(items): addEndJob = False if ('DQMIO' in options.datatier): addEndJob = False - if addEndJob: + if addEndJob: options.step=options.step+',ENDJOB' @@ -180,7 +180,7 @@ def OptionsFromItems(items): options.name = 'RESIM' elif 'reDIGI' in options.trimmedStep: options.name = 'REDIGI' - elif 'HLT' in options.trimmedStep: + elif 'HLT' in options.trimmedStep: options.name = 'HLT' elif 'RECO' in options.trimmedStep: options.name = 'RECO' @@ -244,12 +244,16 @@ def OptionsFromItems(items): raise Exception("Not a valid profiler type %s. Alternatives are pp, mp, fp=."%(profilerType)) options.prefix = "igprof -t cmsRun -%s" % profilerType - + if options.heap_profile: - if options.profile and options.prefix: + if options.prefix: raise Exception("--heap_profile and --prefix are incompatible") - profilerType = 'pp' - options.prefix = "MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out cmsRunJE " + options.prefix = "env MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out LD_PRELOAD=libjemalloc-prof.so " + + if options.maxmem_profile: + if options.prefix: + raise Exception("--maxmem_profile and --prefix are incompatible") + options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so " # If an "era" argument was supplied make sure it is one of the valid possibilities if options.era : @@ -262,7 +266,7 @@ def OptionsFromItems(items): validOptions="" # Create a stringified list of valid options to print to the user for key in eras.__dict__ : if checkModifier(eras.__dict__[key]): - if validOptions!="" : validOptions+=", " + if validOptions!="" : validOptions+=", " validOptions+="'"+key+"'" raise Exception( "'%s' is not a valid option for '--era'. Valid options are %s." % (eraName, validOptions) ) # If the "--fast" option was supplied automatically enable the fastSim era