Merge pull request #44001 from gartung/gartung-maxmempreload-cmsdrive…

…ropion

Add cmsDriver option --maxmem_profile to prefix LD_PRELOAD="libPerfToolsAllocMonitorPreload.so libPerfToolsMaxMemoryPreload.so" to cmsRun

Configuration/Applications/python/ConfigBuilder.py

@@ -71,8 +71,10 @@ class Options:
 defaultOptions.inputEventContent = ''
 defaultOptions.dropDescendant = False
 defaultOptions.relval = None
+defaultOptions.prefix = None
 defaultOptions.profile = None
 defaultOptions.heap_profile = None
+defaultOptions.maxmem_profile = None
 defaultOptions.isRepacked = False
 defaultOptions.restoreRNDSeeds = False
 defaultOptions.donotDropOnInput = ''
@@ -324,37 +326,13 @@ def heapProfileOptions(self):
         Function to add the jemalloc heap  profile service so that you can dump in the middle
         of the run.
         """
-        profileOpts = self._options.profile.split(':')
+        profileOpts = []
         profilerStart = 1
         profilerInterval = 100
-        profilerFormat = None
+        profilerFormat = "jeprof_%s.heap"
         profilerJobFormat = None
 
-        if len(profileOpts):
-            #type, given as first argument is unused here
-            profileOpts.pop(0)
-        if len(profileOpts):
-            startEvent = profileOpts.pop(0)
-            if not startEvent.isdigit():
-                raise Exception("%s is not a number" % startEvent)
-            profilerStart = int(startEvent)
-        if len(profileOpts):
-            eventInterval = profileOpts.pop(0)
-            if not eventInterval.isdigit():
-                raise Exception("%s is not a number" % eventInterval)
-            profilerInterval = int(eventInterval)
-        if len(profileOpts):
-            profilerFormat = profileOpts.pop(0)
-
 
-        if not profilerFormat:
-            profilerFormat = "%s___%s___%%I.heap" % (
-                self._options.evt_type.replace("_cfi", ""),
-                hashlib.md5(
-                    (str(self._options.step) + str(self._options.pileup) + str(self._options.conditions) +
-                    str(self._options.datatier) + str(self._options.profileTypeLabel)).encode('utf-8')
-                ).hexdigest()
-            )
         if not profilerJobFormat and profilerFormat.endswith(".heap"):
             profilerJobFormat = profilerFormat.replace(".heap", "_EndOfJob.heap")
         elif not profilerJobFormat:

Configuration/Applications/python/Options.py

@@ -424,10 +424,16 @@
 
 expertSettings.add_argument("--heap_profile",
                             help="add the JeProfService with the parameter provided PROFILER:START:STEP:PEREVENOUTPUTFORMAT:ENDOFJOBOUTPUTFORMAT",
-                            default=None,
-                            type=str,
+                            default=False,
+                            action="store_true",
                             dest="heap_profile")
 
+expertSettings.add_argument("--maxmem_profile",
+                            help="add the PerfTools/MaxMemoryPreload monitor",
+                            default=False,
+                            action="store_true",
+                            dest="maxmem_profile")
+
 expertSettings.add_argument("--io",
                             help="Create a json file with io informations",
                             default=None,

Configuration/Applications/python/cmsDriverOptions.py

@@ -17,7 +17,7 @@ def checkModifier(era):
 
 def checkOptions():
     return
-    
+
 def adaptOptions():
     return
 
@@ -34,7 +34,7 @@ def OptionsFromCommand(command):
 def OptionsFromCommandLine():
     import sys
     options=OptionsFromItems(sys.argv[1:])
-    # memorize the command line arguments 
+    # memorize the command line arguments
     options.arguments = reduce(lambda x, y: x+' '+y, sys.argv[1:])
     return options
 
@@ -60,8 +60,8 @@ def OptionsFromItems(items):
     if options.triggerResultsProcess == None and "ALCAOUTPUT" in options.step:
         print("ERROR: If ALCA splitting is requested, the name of the process in which the alca producers ran needs to be specified. E.g. via --triggerResultsProcess RECO")
         sys.exit(1)
-            
-    if not options.evt_type:            
+
+    if not options.evt_type:
         options.evt_type=sys.argv[1]
 
     #now adjust the given parameters before passing it to the ConfigBuilder
@@ -123,15 +123,15 @@ def OptionsFromItems(items):
 
     # add on the end of job sequence...
     addEndJob = True
-    if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "": 
+    if ("FASTSIM" in options.step and not "VALIDATION" in options.step) or "HARVESTING" in options.step or "ALCAHARVEST" in options.step or "ALCAOUTPUT" in options.step or options.step == "":
         addEndJob = False
     if ("SKIM" in options.step and not "RECO" in options.step):
         addEndJob = False
     if ("ENDJOB" in options.step):
         addEndJob = False
     if ('DQMIO' in options.datatier):
         addEndJob = False
-    if addEndJob:    
+    if addEndJob:
         options.step=options.step+',ENDJOB'
 
 
@@ -180,7 +180,7 @@ def OptionsFromItems(items):
             options.name = 'RESIM'
         elif 'reDIGI' in options.trimmedStep:
             options.name = 'REDIGI'
-        elif 'HLT' in options.trimmedStep:    
+        elif 'HLT' in options.trimmedStep:
             options.name = 'HLT'
         elif 'RECO' in options.trimmedStep:
             options.name = 'RECO'
@@ -244,12 +244,16 @@ def OptionsFromItems(items):
             raise Exception("Not a valid profiler type %s. Alternatives are pp, mp, fp=<function>."%(profilerType))
 
         options.prefix = "igprof -t cmsRun -%s" % profilerType
-        
+
     if options.heap_profile:
-        if options.profile and options.prefix:
+        if options.prefix:
             raise Exception("--heap_profile and --prefix are incompatible")
-        profilerType = 'pp'
-        options.prefix = "MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out cmsRunJE "
+        options.prefix = "env MALLOC_CONF=prof:true,prof_accum:true,prof_prefix:jeprof.out LD_PRELOAD=libjemalloc-prof.so "
+
+    if options.maxmem_profile:
+        if options.prefix:
+            raise Exception("--maxmem_profile and --prefix are incompatible")
+        options.prefix = "env LD_PRELOAD=libPerfToolsAllocMonitorPreload.so:libPerfToolsMaxMemoryPreload.so "
 
     # If an "era" argument was supplied make sure it is one of the valid possibilities
     if options.era :
@@ -262,7 +266,7 @@ def OptionsFromItems(items):
                 validOptions="" # Create a stringified list of valid options to print to the user
                 for key in eras.__dict__ :
                     if checkModifier(eras.__dict__[key]):
-                        if validOptions!="" : validOptions+=", " 
+                        if validOptions!="" : validOptions+=", "
                         validOptions+="'"+key+"'"
                 raise Exception( "'%s' is not a valid option for '--era'. Valid options are %s." % (eraName, validOptions) )
     # If the "--fast" option was supplied automatically enable the fastSim era