From e33cbdcbc27bb2dc8282d908ae530954946e70b8 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Thu, 21 Dec 2023 11:26:37 +0100 Subject: [PATCH 1/2] Added --package-build-options which can be used to control package build --- cmsBuild | 59 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/cmsBuild b/cmsBuild index 385fbde..527fbf3 100755 --- a/cmsBuild +++ b/cmsBuild @@ -42,6 +42,7 @@ CMSPKG_CMD = "cmspkg" REF_CMSPKG_CMD = None SERVER_TMP_UPLOAD_DIRECTORY = None rpmEnvironmentScript = "true" +packageBuildOptions = "" PKGFactory = None SystemCompilerVersion = {} BlackListReferencePkgs = [] @@ -171,6 +172,23 @@ def cmspkg_showpkgs(packages): rpm_upload_cache[l.split('-1-')[0]].append(l) return (err, "") + +def getPackageBuildOptions(opts): + buildOpts = [] + if opts.packageBuildOptions: + for macro in opts.packageBuildOptions: + item = macro.split("=",1) + if len(item)==1: item.append("1") + buildOpts.append(item) + return buildOpts + + +def getRpmDefines(buildOptions=True): + rpmMacros = opts.rpmQueryDefines + if buildOptions: + rpmMacros = "%s %s" % (rpmMacros, packageBuildOptions) + return rpmMacros + ################################################################# # Utilities functions for installing packages from ref repository# ################################################################# @@ -1601,6 +1619,7 @@ class PackageFactory(object): pkg.generateInitEnv() chksum = pkg.checksum cachedVersion = pkg.version + pkg.buildOptions = pkg.getBuildOptions() pkg.calculateChecksum(force=True) regen_msg = "" if (pkg.checksum != chksum): @@ -2159,12 +2178,7 @@ class Package(object): pkgdir = property(lambda self: self.__getPkgdir()) def clean(self): - del self.sections - del self.specPreHeader - del self.pkgreqs - del self.directpkgreqs del self.spec - del self.requiresStatement del self.origSpec del self.installPostambles self.options = {} @@ -2231,6 +2245,7 @@ class Package(object): self.versionSuffix = True self.requiredBy = 1 self.buildWithRunPath = options.buildWithRunPath + self.buildOptions = "" def rpmLocation(self): return join(self.rpmdir, self.rpmfilename) @@ -2323,6 +2338,7 @@ class Package(object): self.parseRequires() self.dumpSpec() createDirs(architecture=self.cmsplatf) + self.buildOptions = self.getBuildOptions() self.calculateChecksum() self.rewriteRequires() self.dumpSpec() @@ -2445,6 +2461,20 @@ class Package(object): old = self.sections[section][subsection] self.sections[section][subsection] = "%s || exit 0\n" % self.buildCondition + old + def getBuildOptions(self): + buildOpts = "" + if self.options.packageBuildOptions: + allOpts = getPackageBuildOptions(self.options) + optsStrings = [ "prefix" ] + ["%s=%%{%s}" % (x[0],x[0]) for x in allOpts] + result = self.rpmEvalStrings(*optsStrings, buildOptions=False) + for res in result[1:]: + opt = allOpts.pop() + if res == ("%s=%%{%s}" % (opt[0],opt[0])): + continue + buildOpts += "%s=%s:" % (opt[0],opt[1]) + return buildOpts + + def calculateChecksum(self, force=False): """ """ @@ -2462,6 +2492,9 @@ class Package(object): self.version = calculateHumanReadableVersion(self) return + if self.buildOptions: + checksumCalculator.addStrings(self.buildOptions) + checksumCalculator.addStrings(self.origSpec) checksumCalculator.addStrings(DEFAULT_SECTIONS.values()) checksumCalculator.addStrings(PKGFactory.sectionOptions.values()) @@ -2694,7 +2727,7 @@ class Package(object): # FIXME: make this static? Should not change over the whole period. return join(abspath(self.tempDirPrefix), "tmpspec-%s" % self.name) - def rpmEvalStrings(self, *strings): + def rpmEvalStrings(self, *strings, buildOptions=True): spec = "\n".join([self.specPreHeader, SPEC_HEADER % self.__dict__, PKGFactory.preamble, @@ -2704,8 +2737,9 @@ class Package(object): f.write(PKGFactory.postProcessSpec(spec).replace('%%', '%')) f.close() commandPrefix = getCommandPrefix(self.options) + rpmMacros = getRpmDefines(buildOptions) evalCommand = "%s ; %s rpm -q --specfile %s --info %s --define 'buildroot /foo'" % (rpmEnvironmentScript, commandPrefix, - self.tmpspec, self.options.rpmQueryDefines) + self.tmpspec, rpmMacros) log(evalCommand, DEBUG) error, output = getstatusoutput(evalCommand) if error: @@ -2851,7 +2885,7 @@ class Package(object): buildDeps = [] queryCommand = "%s ; %s rpm -q --info --specfile %s %s --define 'buildroot /foo' 2>/dev/null" % ( rpmEnvironmentScript, getCommandPrefix(self.options), - self.tmpspec, self.options.rpmQueryDefines) + self.tmpspec, getRpmDefines()) regexps = PKGFactory.headerMatchingRegexp matchers = [(regexps.REQUIRES_REGEXP, deps), (regexps.REMOTE_SOURCE_REGEXP, sources), @@ -2988,6 +3022,11 @@ def parseOptions(): action="append", help="Define extra macros to pass to build process.", default=None) + advancedBuildOptions.add_option("--package-build-options", + dest="packageBuildOptions", + action="append", + help="Extra build options passed set as rpmbuild macros", + default=None) advancedBuildOptions.add_option("--sources", dest="sources", action="store_true", @@ -3577,7 +3616,7 @@ def buildPackage(pkg, scheduler): optionsDict["rpmenv"] = rpmEnvironmentScript optionsDict.update({"prefix": getCommandPrefix(pkg.options), "buildRoot": join(optionsDict["tmpdir"], "BUILDROOT", pkg.checksum), - "extraRpmDefines": pkg.options.rpmQueryDefines}) + "extraRpmDefines": getRpmDefines()}) if not pkg.options.bootstrap: optionsDict["nodeps"] = "--nodeps" @@ -4637,6 +4676,8 @@ if __name__ == "__main__": cms_debug_packages[pkg].add(dbg) else: opts.rpmQueryDefines += ' --define "%s"' % macro + if opts.packageBuildOptions: + packageBuildOptions = " ".join(['--define "%s %s"' % (x[0], x[1]) for x in getPackageBuildOptions(opts)]) #concatenate all cms_debug_packages in one place dbg_pkgs = [] for pkg in cms_debug_packages: From 86d11aa1c37304974824425fbf1671b3a828bda9 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Thu, 21 Dec 2023 15:26:00 +0100 Subject: [PATCH 2/2] read build options from cmsdist --- cmsBuild | 57 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/cmsBuild b/cmsBuild index 527fbf3..3c7f88b 100755 --- a/cmsBuild +++ b/cmsBuild @@ -43,6 +43,7 @@ REF_CMSPKG_CMD = None SERVER_TMP_UPLOAD_DIRECTORY = None rpmEnvironmentScript = "true" packageBuildOptions = "" +packageBuildOptionsChanged = None PKGFactory = None SystemCompilerVersion = {} BlackListReferencePkgs = [] @@ -175,8 +176,8 @@ def cmspkg_showpkgs(packages): def getPackageBuildOptions(opts): buildOpts = [] - if opts.packageBuildOptions: - for macro in opts.packageBuildOptions: + if opts.buildOptions: + for macro in opts.buildOptions: item = macro.split("=",1) if len(item)==1: item.append("1") buildOpts.append(item) @@ -1584,12 +1585,14 @@ class PackageFactory(object): raise IncludedFileNotFound(pkgName, incFile) def createWithSpec(self, pkgName): + global packageBuildOptionsChanged if self.has_key(pkgName): return self[pkgName] specLines = self.readSpec(pkgName) pkg = None specCache = "%s/specCache/%s.%s" % ( self.options.tempdir, pkgName, sha256((self.__basePackageHash + str(specLines)).encode()).hexdigest()) forceRecreate = False + if (not packageBuildOptionsChanged is None): forceRecreate = packageBuildOptionsChanged while True: if forceRecreate or (not exists(specCache)): getstatusoutput("rm -f %s/specCache/%s.*" % (self.options.tempdir, pkgName)) @@ -1604,6 +1607,9 @@ class PackageFactory(object): forceRecreate = True pkg = pickle.load(open(specCache,'rb')) if not exists(pkg.specFilename()): continue + if pkg.globalBuildOptions != self.options.buildOptions: + packageBuildOptionsChanged = True + continue pkg.setOptions(self.options) for sp in pkg.subpackages: sp.checksum = '%{nil}' if pkg.override_version: specLines = fixVersionLine (specLines,pkg.override_version) @@ -1619,7 +1625,6 @@ class PackageFactory(object): pkg.generateInitEnv() chksum = pkg.checksum cachedVersion = pkg.version - pkg.buildOptions = pkg.getBuildOptions() pkg.calculateChecksum(force=True) regen_msg = "" if (pkg.checksum != chksum): @@ -2178,7 +2183,12 @@ class Package(object): pkgdir = property(lambda self: self.__getPkgdir()) def clean(self): + del self.sections + del self.specPreHeader + del self.pkgreqs + del self.directpkgreqs del self.spec + del self.requiresStatement del self.origSpec del self.installPostambles self.options = {} @@ -2245,6 +2255,7 @@ class Package(object): self.versionSuffix = True self.requiredBy = 1 self.buildWithRunPath = options.buildWithRunPath + self.globalBuildOptions = "" self.buildOptions = "" def rpmLocation(self): @@ -2338,6 +2349,7 @@ class Package(object): self.parseRequires() self.dumpSpec() createDirs(architecture=self.cmsplatf) + self.globalBuildOptions = self.options.buildOptions self.buildOptions = self.getBuildOptions() self.calculateChecksum() self.rewriteRequires() @@ -2463,16 +2475,17 @@ class Package(object): def getBuildOptions(self): buildOpts = "" - if self.options.packageBuildOptions: + if self.options.buildOptions: allOpts = getPackageBuildOptions(self.options) optsStrings = [ "prefix" ] + ["%s=%%{%s}" % (x[0],x[0]) for x in allOpts] result = self.rpmEvalStrings(*optsStrings, buildOptions=False) - for res in result[1:]: + while len(result)>1: + res = result.pop() opt = allOpts.pop() if res == ("%s=%%{%s}" % (opt[0],opt[0])): continue - buildOpts += "%s=%s:" % (opt[0],opt[1]) - return buildOpts + buildOpts += "%s=%s," % (opt[0],opt[1]) + return buildOpts.strip(",") def calculateChecksum(self, force=False): @@ -3022,8 +3035,8 @@ def parseOptions(): action="append", help="Define extra macros to pass to build process.", default=None) - advancedBuildOptions.add_option("--package-build-options", - dest="packageBuildOptions", + advancedBuildOptions.add_option("--build-options", + dest="buildOptions", action="append", help="Extra build options passed set as rpmbuild macros", default=None) @@ -4534,6 +4547,20 @@ def checkOptionsLocalSource(opts): sourceOptionDict[pkg]["Source"] = source_path opts.localSources = sourceOptionDict + +def get_build_options(opts): + sys.path.insert(0,join(opts.cmsdist, "etc")) + pkg = __import__('build_options') + build_opts = {} + try: + build_opts = pkg.get_build_options(opts) + except: + pass + sys.modules.pop('build_options') + sys.path.pop(0) + return build_opts + + if __name__ == "__main__": # Stop processing if SCRAM runtime env is set to avoid picking up # python from cms external area @@ -4676,8 +4703,18 @@ if __name__ == "__main__": cms_debug_packages[pkg].add(dbg) else: opts.rpmQueryDefines += ' --define "%s"' % macro - if opts.packageBuildOptions: + if opts.buildOptions: + pkgOptions= get_build_options(opts) + newOpts = [] + for bopts in opts.buildOptions: + for opt in [ o for o in bopts.split(",") if o]: + if not opt in pkgOptions: + fatal("Unknown package build option '%s'. Please add it to your cmsdist/etc/build_options.py file." % opt) + newOpts += pkgOptions[opt] + opts.buildOptions = sorted(newOpts) packageBuildOptions = " ".join(['--define "%s %s"' % (x[0], x[1]) for x in getPackageBuildOptions(opts)]) + print("Using Build Options:", packageBuildOptions) + #concatenate all cms_debug_packages in one place dbg_pkgs = [] for pkg in cms_debug_packages: