From 79afc5c679ee7dc16494252a10544eff958775fe Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Jul 2021 11:33:08 +0200 Subject: [PATCH 1/6] python3 cleaning --- .../Configuration/python/Tools/confdb.py | 1 - .../Configuration/scripts/edmPluginCoverage | 5 ++-- .../scripts/hltCheckPrescaleModules | 19 +++++++------- .../Configuration/scripts/hltConfigFromDB | 2 +- HLTrigger/Configuration/scripts/hltDumpStream | 25 ++++++++++--------- .../Configuration/scripts/hltFindDuplicates | 15 +++++------ .../Configuration/scripts/hltGetConfiguration | 9 ++++--- HLTrigger/Configuration/scripts/hltListPaths | 5 ++-- 8 files changed, 43 insertions(+), 38 deletions(-) diff --git a/HLTrigger/Configuration/python/Tools/confdb.py b/HLTrigger/Configuration/python/Tools/confdb.py index 434a15b7da00a..8890386a9cd54 100644 --- a/HLTrigger/Configuration/python/Tools/confdb.py +++ b/HLTrigger/Configuration/python/Tools/confdb.py @@ -4,7 +4,6 @@ import sys import re import os -import urllib, urllib2 from .pipe import pipe as _pipe from .options import globalTag from itertools import islice diff --git a/HLTrigger/Configuration/scripts/edmPluginCoverage b/HLTrigger/Configuration/scripts/edmPluginCoverage index a50ad37243ebd..25bfd38712ae2 100755 --- a/HLTrigger/Configuration/scripts/edmPluginCoverage +++ b/HLTrigger/Configuration/scripts/edmPluginCoverage @@ -3,6 +3,7 @@ # Author: Andrea 'fwyzard' Bocci , Università di Pisa +from __future__ import print_function import os, sys import re @@ -210,7 +211,7 @@ index.sort() for lib in index: if lib in packagemap: - print "%-80s\t%s" % (libs[lib], packagemap[lib][0]) + print("%-80s\t%s" % (libs[lib], packagemap[lib][0])) else: - print "%-80s\t*** not found ***" % libs[lib] + print("%-80s\t*** not found ***" % libs[lib]) diff --git a/HLTrigger/Configuration/scripts/hltCheckPrescaleModules b/HLTrigger/Configuration/scripts/hltCheckPrescaleModules index ae2a278ee9f6f..7184caae82b5d 100755 --- a/HLTrigger/Configuration/scripts/hltCheckPrescaleModules +++ b/HLTrigger/Configuration/scripts/hltCheckPrescaleModules @@ -1,5 +1,6 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 +from __future__ import print_function import sys import imp import re @@ -12,7 +13,7 @@ if '-w' in sys.argv: enableWarnings = True if len(sys.argv) != 2: - print "usage: %s [-w] menu.py" % sys.argv[0] + print("usage: %s [-w] menu.py" % sys.argv[0]) sys.exit(1) # whitelist paths exempt from validation @@ -21,7 +22,7 @@ whitelist = ('HLTriggerFirstPath', 'HLTriggerFinalPath') # load the menu and get the "process" object menu = imp.new_module('menu') name = sys.argv[1] -execfile(name, globals(), menu.__dict__) +exec(open(name).read(), globals(), menu.__dict__) process = menu.process # get all paths @@ -46,23 +47,23 @@ for path in paths: if module in process.filters_(): # found a filter if process.filters_()[module].type_() == 'HLTPrescaler': # it's a prescaler if found: - print 'ERROR: path %s has more than one HLTPrescaler module' % path + print('ERROR: path %s has more than one HLTPrescaler module' % path) found = True label = process.filters_()[module].label() if enableWarnings and label != goodLabel: - print 'WARNING: path %s has an HLTPrescaler module labaled "%s", while the suggested label is "%s"' % (path, label, goodLabel) + print('WARNING: path %s has an HLTPrescaler module labaled "%s", while the suggested label is "%s"' % (path, label, goodLabel)) if label in prescalerNames: duplicateNames.add(label) else: prescalerNames.add(label) if not found: - print 'ERROR: path %s dos not have an HLTPrescaler module' % path + print('ERROR: path %s dos not have an HLTPrescaler module' % path) # print the duplicate prescales, and the affected paths for label in duplicateNames: - print 'ERROR: prescale "%s" is shared by the paths' % label + print('ERROR: prescale "%s" is shared by the paths' % label) for path in paths: if label in paths[path].moduleNames(): - print '\t%s' % path - print + print('\t%s' % path) + print() diff --git a/HLTrigger/Configuration/scripts/hltConfigFromDB b/HLTrigger/Configuration/scripts/hltConfigFromDB index a5e520c2646c7..4db98dcd3f487 100755 --- a/HLTrigger/Configuration/scripts/hltConfigFromDB +++ b/HLTrigger/Configuration/scripts/hltConfigFromDB @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 from HLTrigger.Configuration.Tools.confdbOfflineConverter import main diff --git a/HLTrigger/Configuration/scripts/hltDumpStream b/HLTrigger/Configuration/scripts/hltDumpStream index f69d2116dc72e..bf2b1da772c5c 100755 --- a/HLTrigger/Configuration/scripts/hltDumpStream +++ b/HLTrigger/Configuration/scripts/hltDumpStream @@ -1,6 +1,7 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # -*- coding: utf-8 -*- +from __future__ import print_function import sys, imp, re import operator import FWCore.ParameterSet.Config as cms @@ -21,7 +22,7 @@ except: config = sys.stdin else: config = open(configname) -exec config in globals(), hlt.__dict__ +exec(config, globals(), hlt.__dict__) config.close() if 'process' in hlt.__dict__: @@ -263,7 +264,7 @@ def getEndPath(output): def dumpHeader(): if mode == 'csv': - print 'stream, dataset, path, %s, L1 trigger' % getPrescaleNames() + print('stream, dataset, path, %s, L1 trigger' % getPrescaleNames()) def dumpStream(stream): @@ -271,7 +272,7 @@ def dumpStream(stream): allpaths = set() if mode == 'text': - print 'stream', stream + print('stream', stream) out = 'hltOutput%s' % stream end = getEndPath(out) if end: @@ -281,32 +282,32 @@ def dumpStream(stream): pds = sorted( process.streams.__dict__[stream] ) for pd in pds: if mode == 'text': - print ' dataset', pd + print(' dataset', pd) if pd in process.datasets.__dict__: paths = sorted( path for path in process.datasets.__dict__[pd] ) assigned.update( paths ) for path in paths: - print dumpPath(stream, pd, path, out, end) + print(dumpPath(stream, pd, path, out, end)) else: if mode == 'text': - print ' *** not found ***' + print(' *** not found ***') unassigned = allpaths - assigned if unassigned: if mode == 'text': - print ' *** unassigned paths ***' + print(' *** unassigned paths ***') for path in sorted(unassigned): - print dumpPath(stream, '(unassigned)', path, out, end) + print(dumpPath(stream, '(unassigned)', path, out, end)) if not end: - print ' *** corresponding EndPath not found ***' + print(' *** corresponding EndPath not found ***') else: missing = assigned - allpaths if missing: if mode == 'text': - print ' *** paths missing from the EndPath\'s output module ***' + print(' *** paths missing from the EndPath\'s output module ***') for path in sorted(missing): - print dumpPath(stream, '(missing)', path, out, end) + print(dumpPath(stream, '(missing)', path, out, end)) # read the list of streams diff --git a/HLTrigger/Configuration/scripts/hltFindDuplicates b/HLTrigger/Configuration/scripts/hltFindDuplicates index 9a932f51acaeb..0dd2f41c92eee 100755 --- a/HLTrigger/Configuration/scripts/hltFindDuplicates +++ b/HLTrigger/Configuration/scripts/hltFindDuplicates @@ -1,5 +1,6 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 +from __future__ import print_function import sys, imp, re, itertools from HLTrigger.Configuration.Tools.frozendict import frozendict import FWCore.ParameterSet.Config as cms @@ -145,11 +146,11 @@ class ModuleList(object): def dump(self): for m in self.modules: - print "%s = (%s) {" % (m.label, m.type) + print("%s = (%s) {" % (m.label, m.type)) for k, v in m.params.iteritems(): - print "\t%s = %s" % (k, v) - print '}' - print + print("\t%s = %s" % (k, v)) + print('}') + print() @@ -175,7 +176,7 @@ def findDuplicates(process): for target, (group, regexp) in groups.iteritems(): dump.write('#%s\n%s\n\n' % ( target, '\n'.join(group))) dump.close() - print "found %3d duplicates in %3d groups" % (dups, len(groups)) + print("found %3d duplicates in %3d groups" % (dups, len(groups))) oldups = dups modules.apply_rename(groups) groups = modules.group() @@ -203,7 +204,7 @@ def main(): config = sys.stdin else: config = open(configname) - exec config in globals(), hlt.__dict__ + exec(config, globals(), hlt.__dict__) config.close() findDuplicates(hlt.process) diff --git a/HLTrigger/Configuration/scripts/hltGetConfiguration b/HLTrigger/Configuration/scripts/hltGetConfiguration index 6f6f002266882..7d1edae01f615 100755 --- a/HLTrigger/Configuration/scripts/hltGetConfiguration +++ b/HLTrigger/Configuration/scripts/hltGetConfiguration @@ -1,5 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +from __future__ import print_function import sys, os import argparse from HLTrigger.Configuration.extend_argparse import * @@ -227,6 +228,6 @@ config = parser.parse_args(namespace = options.HLTProcessOptions()) cmdArgs = sys.argv cmdArgs[0] = os.path.basename(sys.argv[0]) cmdLine = ' '.join(cmdArgs) -print '# ' + cmdLine -print -print confdb.HLTProcess(config).dump() +print('# ' + cmdLine) +print() +print(confdb.HLTProcess(config).dump()) diff --git a/HLTrigger/Configuration/scripts/hltListPaths b/HLTrigger/Configuration/scripts/hltListPaths index 45ebe6fd191f8..4d319f8b9cbc7 100755 --- a/HLTrigger/Configuration/scripts/hltListPaths +++ b/HLTrigger/Configuration/scripts/hltListPaths @@ -1,5 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +from __future__ import print_function import os import re import argparse @@ -83,5 +84,5 @@ parser.add_argument('-h', '--help', config = parser.parse_args() paths = getPathList(config.menu, config.selection) for path in paths: - print path + print(path) From 3470b005bf0942eab4a5a8c9e7739f015a95d011 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Jul 2021 14:11:08 +0200 Subject: [PATCH 2/6] python3 compatibility --- HLTrigger/Configuration/scripts/hltCheckPrescaleModules | 4 ++-- HLTrigger/Configuration/scripts/hltListPaths | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HLTrigger/Configuration/scripts/hltCheckPrescaleModules b/HLTrigger/Configuration/scripts/hltCheckPrescaleModules index 7184caae82b5d..210c53021aca3 100755 --- a/HLTrigger/Configuration/scripts/hltCheckPrescaleModules +++ b/HLTrigger/Configuration/scripts/hltCheckPrescaleModules @@ -2,7 +2,7 @@ from __future__ import print_function import sys -import imp +import types import re # global options @@ -20,7 +20,7 @@ if len(sys.argv) != 2: whitelist = ('HLTriggerFirstPath', 'HLTriggerFinalPath') # load the menu and get the "process" object -menu = imp.new_module('menu') +menu = types.ModuleType('menu') name = sys.argv[1] exec(open(name).read(), globals(), menu.__dict__) process = menu.process diff --git a/HLTrigger/Configuration/scripts/hltListPaths b/HLTrigger/Configuration/scripts/hltListPaths index 4d319f8b9cbc7..b594ad9f12a06 100755 --- a/HLTrigger/Configuration/scripts/hltListPaths +++ b/HLTrigger/Configuration/scripts/hltListPaths @@ -20,7 +20,7 @@ def _build_query(menu): def getPathList(menu, selection): cmdline = 'hltConfigFromDB --cff %s --noedsources --noes --noservices --nosequences --nomodules' % _build_query(menu) - data = pipe.pipe(cmdline) + data = pipe.pipe(cmdline).decode() if 'Exhausted Resultset' in data or 'CONFIG_NOT_FOUND' in data: raise ImportError('%s is not a valid HLT menu' % menu.value) From fc92a71b78c9c570bae18655b5ae6d40f79c1c3b Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Jul 2021 14:11:34 +0200 Subject: [PATCH 3/6] python3 compatibility --- .../Configuration/python/Tools/confdbOfflineConverter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py index 2575dff3c1147..27ad0746df992 100644 --- a/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py +++ b/HLTrigger/Configuration/python/Tools/confdbOfflineConverter.py @@ -2,7 +2,7 @@ import sys, os import os.path import tempfile -import urllib +import urllib.request import shutil import subprocess import atexit @@ -97,7 +97,7 @@ def __init__(self, version = 'v2', database = 'offline', url = None, verbose = F # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.') os.close(handle) - urllib.urlretrieve(self.baseUrl + '/' + jar, temp) + urllib.request.urlretrieve(self.baseUrl + '/' + jar, temp) if not os.path.exists(self.workDir + '/' + jar): os.rename(temp, self.workDir + '/' + jar) else: From 591cfe3e0db321c6d974e7dd73612ec1cf236588 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Jul 2021 14:11:57 +0200 Subject: [PATCH 4/6] python3 compatibility --- FWCore/ParameterSet/scripts/edmConfigDump | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/FWCore/ParameterSet/scripts/edmConfigDump b/FWCore/ParameterSet/scripts/edmConfigDump index e3d2124c0151e..fa19ab1f3242b 100755 --- a/FWCore/ParameterSet/scripts/edmConfigDump +++ b/FWCore/ParameterSet/scripts/edmConfigDump @@ -3,7 +3,8 @@ from __future__ import print_function import sys import os -import imp +from importlib.machinery import SourceFileLoader +import types import argparse # make the behaviour of 'cmsRun file.py' and 'edmConfigDump file.py' more consistent @@ -19,7 +20,10 @@ parser.add_argument("--pruneVerbose", action="store_true", options = parser.parse_args() handle = open(options.filename, 'r') -cfo = imp.load_source("pycfg", options.filename, handle) +loader=SourceFileLoader("pycfg", options.filename) +loaded = types.ModuleType("pycfg",loader.name) +loader.exec_module(loaded) +#cfo = imp.load_source("pycfg", options.filename, handle) cmsProcess = cfo.process handle.close() From e947bc601fe78b6b81af119ca36ef5910b3a8292 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Jul 2021 18:52:41 +0200 Subject: [PATCH 5/6] repair changes to edmConfigDump --- FWCore/ParameterSet/scripts/edmConfigDump | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/FWCore/ParameterSet/scripts/edmConfigDump b/FWCore/ParameterSet/scripts/edmConfigDump index fa19ab1f3242b..917184d374953 100755 --- a/FWCore/ParameterSet/scripts/edmConfigDump +++ b/FWCore/ParameterSet/scripts/edmConfigDump @@ -21,13 +21,12 @@ parser.add_argument("--pruneVerbose", action="store_true", options = parser.parse_args() handle = open(options.filename, 'r') loader=SourceFileLoader("pycfg", options.filename) -loaded = types.ModuleType("pycfg",loader.name) -loader.exec_module(loaded) -#cfo = imp.load_source("pycfg", options.filename, handle) -cmsProcess = cfo.process -handle.close() +mod=types.ModuleType(loader.name) +loader.exec_module(mod) +cmsProcess= mod.process if options.prune: cmsProcess.prune(options.pruneVerbose) print(cmsProcess.dumpPython()) + From 315309450f6ee02ff206cf0095fcc3b90d4971b4 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Jul 2021 18:57:20 +0200 Subject: [PATCH 6/6] move away from imp usage --- HLTrigger/Configuration/python/Utilities.py | 4 ++-- HLTrigger/Configuration/test/getDatasets.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/HLTrigger/Configuration/python/Utilities.py b/HLTrigger/Configuration/python/Utilities.py index 54af1f7e7d725..bfa3020938c55 100644 --- a/HLTrigger/Configuration/python/Utilities.py +++ b/HLTrigger/Configuration/python/Utilities.py @@ -1,4 +1,4 @@ -import imp as _imp +import types import HLTrigger.Configuration.Tools.options as _options import HLTrigger.Configuration.Tools.confdb as _confdb @@ -26,7 +26,7 @@ def loadHltConfiguration(process, menu, **args): args['fragment'] = True options = _build_options(**args) - hlt = _imp.new_module('hlt') + hlt = types.ModuleType('hlt') exec(_confdb.HLTProcess(options).dump(), globals(), hlt.__dict__) process.extend( hlt ) diff --git a/HLTrigger/Configuration/test/getDatasets.py b/HLTrigger/Configuration/test/getDatasets.py index dca3efdb297d1..2d25f01e7e2ec 100755 --- a/HLTrigger/Configuration/test/getDatasets.py +++ b/HLTrigger/Configuration/test/getDatasets.py @@ -2,7 +2,7 @@ import sys import subprocess -import imp +import types import re import FWCore.ParameterSet.Config as cms @@ -18,7 +18,7 @@ def extractDatasets(version, database, config): (out, err) = proc.communicate() # load the streams and Datasets - hlt = imp.new_module('hlt') + hlt = types.ModuleType('hlt') exec(out, globals(), hlt.__dict__) return hlt.process