-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1873 from PathogenDavid/release-automation
Revamp CI infrastructure and implement release automation
- Loading branch information
Showing
78 changed files
with
1,675 additions
and
543 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
#!/usr/bin/env python3 | ||
import os | ||
|
||
import gha | ||
import nuget | ||
|
||
#================================================================================================== | ||
# Get inputs | ||
#================================================================================================== | ||
def get_environment_variable(name): | ||
ret = os.getenv(name) | ||
|
||
if ret is None or ret == '': | ||
gha.print_error(f"Missing required parameter '{name}'") | ||
return '' | ||
|
||
return ret | ||
|
||
version_file_path = get_environment_variable('version_file_path') | ||
just_released_version = get_environment_variable('just_released_version').strip('v') | ||
|
||
if not nuget.is_valid_version(just_released_version): | ||
gha.print_error('The specified just-released version is not a valid semver version.') | ||
|
||
gha.fail_if_errors() | ||
|
||
#================================================================================================== | ||
# Bump version number | ||
#================================================================================================== | ||
|
||
version = nuget.get_version_parts(just_released_version) | ||
version.patch += 1 | ||
version.prerelease = None | ||
version.build_metadata = None | ||
|
||
print(f"Bumping to version {version}") | ||
|
||
with open(version_file_path, 'w') as f: | ||
f.write("<!-- [auto-generated] This file is automatically re-created when Bonsai releases and generally should not be modified by hand [/auto-generated] -->\n") | ||
f.write("<Project>\n") | ||
f.write(" <PropertyGroup>\n") | ||
f.write(f" <BonsaiVersion>{version}</BonsaiVersion>\n") | ||
f.write(" </PropertyGroup>\n") | ||
f.write("</Project>") | ||
|
||
gha.set_environment_variable('NEXT_VERSION', str(version)) | ||
|
||
gha.fail_if_errors() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,223 @@ | ||
#!/usr/bin/env python3 | ||
import hashlib | ||
import os | ||
import sys | ||
|
||
from pathlib import Path | ||
from zipfile import ZipFile, ZipInfo | ||
|
||
import gha | ||
import nuget | ||
|
||
# Symbol packages will change even for changes that we don't care about because the deterministic hash embedded in the PDB | ||
# is affected by the MVID of a package's dependencies. We don't want to release a new package when the only things that | ||
# changed were external to the package, so we don't check them. | ||
CHECK_SYMBOL_PACKAGES = False | ||
|
||
# The following packages will always release no matter what | ||
always_release_packages = set([ | ||
'Bonsai', | ||
'Bonsai.Core', | ||
'Bonsai.Design', | ||
'Bonsai.Editor', | ||
'Bonsai.Player', | ||
]) | ||
|
||
if len(sys.argv) != 5: | ||
gha.print_error('Usage: compare-nuget-packages.py <previous-dummy-packages-path> <next-dummy-packages-path> <release-packages-path> <release-manifest-path>') | ||
sys.exit(1) | ||
else: | ||
previous_packages_path = Path(sys.argv[1]) | ||
next_packages_path = Path(sys.argv[2]) | ||
release_packages_path = Path(sys.argv[3]) | ||
release_manifest_path = Path(sys.argv[4]) | ||
|
||
if not previous_packages_path.exists(): | ||
gha.print_error(f"Previous packages path '{previous_packages_path}' does not exist.") | ||
if not next_packages_path.exists(): | ||
gha.print_error(f"Next packages path '{next_packages_path}' does not exist.") | ||
if not release_packages_path.exists(): | ||
gha.print_error(f"Release packages path '{previous_packages_path}' does not exist.") | ||
if release_manifest_path.exists(): | ||
gha.print_error(f"Release manifest '{release_manifest_path}' already exists.") | ||
gha.fail_if_errors() | ||
|
||
def verbose_log(message: str): | ||
gha.print_debug(message) | ||
|
||
def should_ignore(file: ZipInfo) -> bool: | ||
# Ignore metadata files which change on every pack | ||
if file.filename == '_rels/.rels': | ||
return True | ||
if file.filename.startswith('package/services/metadata/core-properties/') and file.filename.endswith('.psmdcp'): | ||
return True | ||
|
||
# Don't care about explicit directories | ||
if file.is_dir(): | ||
return True | ||
|
||
return False | ||
|
||
def nuget_packages_are_equivalent(a_path: Path, b_path: Path, is_snupkg: bool = False) -> bool: | ||
verbose_log(f"Comparing '{a_path}' and '{b_path}'") | ||
|
||
# One package exists and the other does not | ||
if a_path.exists() != b_path.exists(): | ||
verbose_log(f"Not equivalent: Only one package actually exists") | ||
return False | ||
|
||
# The package doesn't exist at all, assume mistake unless we're checking the optional symbol packages | ||
if not a_path.exists(): | ||
if is_snupkg: | ||
verbose_log("Equivalent: Neither package exists") | ||
return True | ||
raise FileNotFoundError(f"Neither package exists: '{a_path}' or '{b_path}'") | ||
|
||
# From this point on: Check everything and emit messages for debugging purposes | ||
is_equivalent = True | ||
|
||
# Check if corresponding symbol packages are equivalent | ||
if CHECK_SYMBOL_PACKAGES and not is_snupkg: | ||
if not nuget_packages_are_equivalent(a_path.with_suffix(".snupkg"), b_path.with_suffix(".snupkg"), True): | ||
verbose_log("Not equivalent: Symbol packages are not equivalent") | ||
is_equivalent = False | ||
else: | ||
verbose_log("Symbol packages are equivalent") | ||
|
||
# Compare the contents of the packages | ||
# NuGet package packing is unfortunately not fully deterministic so we cannot compare the packages directly | ||
# https://github.com/NuGet/Home/issues/8601 | ||
with ZipFile(a_path, 'r') as a_zip, ZipFile(b_path, 'r') as b_zip: | ||
b_infos = { } | ||
for b_info in b_zip.infolist(): | ||
if should_ignore(b_info): | ||
continue | ||
assert b_info.filename not in b_infos | ||
b_infos[b_info.filename] = b_info | ||
|
||
for a_info in a_zip.infolist(): | ||
if should_ignore(a_info): | ||
continue | ||
|
||
b_info = b_infos.pop(a_info.filename, None) | ||
if b_info is None: | ||
verbose_log(f"Not equivalent: '{a_info.filename}' exists in '{a_path}' but not in '{b_path}'") | ||
is_equivalent = False | ||
continue | ||
|
||
if a_info.CRC != b_info.CRC: | ||
verbose_log(f"Not equivalent: CRCs of '{a_info.filename}' do not match between '{a_path}' and '{b_path}'") | ||
is_equivalent = False | ||
continue | ||
|
||
if a_info.file_size != b_info.file_size: | ||
verbose_log(f"Not equivalent: File sizes of '{a_info.filename}' do not match between '{a_path}' and '{b_path}'") | ||
is_equivalent = False | ||
continue | ||
|
||
a_hash = hashlib.file_digest(a_zip.open(a_info), 'sha256').hexdigest() # type: ignore | ||
b_hash = hashlib.file_digest(b_zip.open(b_info), 'sha256').hexdigest() # type: ignore | ||
if a_hash != b_hash: | ||
verbose_log(f"Not equivalent: SHA256 hashes of '{a_info.filename}' do not match between '{a_path}' and '{b_path}'") | ||
is_equivalent = False | ||
continue | ||
|
||
# Ensure every file in B was processed | ||
if len(b_infos) > 0: | ||
is_equivalent = False | ||
verbose_log(f"Not equivalent: The following file(s) exist in '{a_path}' but not in '{b_path}'") | ||
for filename in b_infos: | ||
verbose_log(f" '{filename}'") | ||
|
||
return is_equivalent | ||
|
||
different_packages = [] | ||
force_released_packages = [] | ||
next_packages = set() | ||
for file in os.listdir(next_packages_path): | ||
if not file.endswith(".nupkg"): | ||
continue | ||
|
||
# We don't tolerate build metadata here because the nuget_packages_are_equivalent call doesn't either | ||
if not file.endswith(".99.99.99.nupkg"): | ||
gha.print_error(f"Package '{file}' does not have a dummy version.") | ||
|
||
package_name = nuget.get_package_name(file) | ||
next_packages.add(package_name) | ||
|
||
if not nuget_packages_are_equivalent(next_packages_path / file, previous_packages_path / file): | ||
verbose_log(f"'{file}' differs") | ||
different_packages.append(package_name) | ||
elif package_name in always_release_packages: | ||
force_released_packages.append(package_name) | ||
|
||
previous_packages = set() | ||
for file in os.listdir(previous_packages_path): | ||
if file.endswith(".nupkg"): | ||
previous_packages.add(nuget.get_package_name(file)) | ||
|
||
release_packages = set() | ||
for file in os.listdir(release_packages_path): | ||
if file.endswith(".nupkg"): | ||
release_packages.add(nuget.get_package_name(file)) | ||
|
||
with gha.JobSummary() as md: | ||
def write_both(line: str = ''): | ||
print(line) | ||
md.write_line(line) | ||
|
||
print() | ||
different_packages.sort() | ||
md.write_line("# Packages with changes\n") | ||
if len(different_packages) == 0: | ||
print("There are no packages with any changes.") | ||
md.write_line("*There are no packages with any changes.*") | ||
else: | ||
print("The following packages have changes:") | ||
for package in different_packages: | ||
print(f" {package}") | ||
md.write_line(f"* {package}") | ||
|
||
if len(force_released_packages) > 0: | ||
write_both() | ||
write_both("The following packages are configured to release anyway despite not being changed:") | ||
md.write_line() | ||
force_released_packages.sort() | ||
for package in force_released_packages: | ||
print(f" {package}") | ||
md.write_line(f"* {package}") | ||
|
||
different_packages += force_released_packages | ||
different_packages.sort() | ||
|
||
# Ensure the next dummy reference and release package sets contain the same packages | ||
def list_missing_peers(heading: str, md_heading: str, packages: set[str]) -> bool: | ||
if len(packages) == 0: | ||
return False | ||
|
||
print() | ||
print(heading) | ||
md.write_line(f"# {md_heading}") | ||
md.write_line() | ||
md.write_line(heading) | ||
md.write_line() | ||
for package in packages: | ||
print(f" {package}") | ||
md.write_line(f"* {package}") | ||
return True | ||
|
||
list_missing_peers("The following packages are new for this release:", "New packages", next_packages - previous_packages) | ||
list_missing_peers("The following packages were removed during this release:", "Removed packages", previous_packages - next_packages) | ||
|
||
if list_missing_peers("The following packages exist in the release package artifact, but not in the next dummy reference artifact:", "⚠ Missing reference packages", release_packages - next_packages): | ||
gha.print_error("Some packages exist in the release package artifact, but not in the next dummy reference artifact.") | ||
if list_missing_peers("The following packages exist in the next dummy reference artifact, but not in the release package artifact:", "⚠ Missing release packages", next_packages - release_packages): | ||
gha.print_error("Some packages exist in the next dummy reference artifact, but not in the release package artifact.") | ||
if list_missing_peers("The following packages are marked to always release but do not exist:", "⚠ Missing always-release packages", always_release_packages - release_packages): | ||
gha.print_error("Some packages exist in the always-release list, but not in the release package artifact.") | ||
|
||
with open(release_manifest_path, 'x') as manifest: | ||
for package in different_packages: | ||
manifest.write(f"{package}\n") | ||
|
||
gha.fail_if_errors() |
Oops, something went wrong.