From e71d4ae4fddf6a581de63049d4136e14a61b05a9 Mon Sep 17 00:00:00 2001 From: Alexey Rivkin Date: Sun, 19 Feb 2023 20:32:15 +0200 Subject: [PATCH] AZP/RELEASE: Add ARM release - Part #2 --- buildlib/az-GitHubDraft.yml | 47 ++++++++++++ buildlib/az-distro-release.yml | 83 ++++++++++++++------- buildlib/azure-pipelines-release.yml | 105 ++++++++++++++------------- 3 files changed, 160 insertions(+), 75 deletions(-) create mode 100644 buildlib/az-GitHubDraft.yml diff --git a/buildlib/az-GitHubDraft.yml b/buildlib/az-GitHubDraft.yml new file mode 100644 index 000000000000..298c626e493b --- /dev/null +++ b/buildlib/az-GitHubDraft.yml @@ -0,0 +1,47 @@ +# Create an empty draft to avoid race condition in distro release + +parameters: + name: + container: + demands: [] + +jobs: + - job: DraftRelease_${{ parameters.name }} + displayName: Draft release ${{ parameters.name }} + container: ${{ parameters.container }} + pool: + name: MLNX + demands: ${{ parameters.demands }} + + steps: + - checkout: self + clean: true + fetchDepth: 100 + path: "we/need/to/go/deeper" + + - bash: ./autogen.sh + displayName: Setup autotools + + - bash: | + set -eEx + # gcc --version + ./contrib/configure-release --with-java=no + ./contrib/buildrpm.sh -s -t -b + displayName: Build tarball + + - task: GithubRelease@0 + condition: eq(variables['Build.Reason'], 'IndividualCI') + displayName: Create/edit GitHub Draft Release + inputs: + githubConnection: release + repositoryName: openucx/ucx + action: edit + tag: $(Build.SourceBranchName) + isDraft: true + addChangeLog: false + releaseNotesSource: file + releaseNotesFile: NEWS + assetUploadMode: replace + assets: | + ./ucx-*.tar.gz + ./rpm-dist/ucx-*.src.rpm diff --git a/buildlib/az-distro-release.yml b/buildlib/az-distro-release.yml index 1be9b1ba949e..9fe8f3b68178 100644 --- a/buildlib/az-distro-release.yml +++ b/buildlib/az-distro-release.yml @@ -1,6 +1,10 @@ +parameters: + name: + demands: [] + jobs: - - job: distro_release - displayName: distro + - job: distro_release_${{ parameters.name }} + displayName: Distro ${{ parameters.name }} variables: ${{ if eq(variables['Build.Reason'], 'ResourceTrigger') }}: POSTFIX: ucx-${{ replace(variables['Build.SourceBranch'], 'refs/heads/', '') }} @@ -9,30 +13,51 @@ jobs: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: POSTFIX: ucx-pr$(System.PullRequest.PullRequestNumber) - pool: - name: MLNX - demands: - - harbor_registry -equals yes - - strategy: - matrix: - centos7_cuda11: - build_container: centos7_cuda11 - artifact_name: $(POSTFIX)-centos7-mofed5-cuda11.tar.bz2 - centos8_cuda11: - build_container: centos8_cuda11 - artifact_name: $(POSTFIX)-centos8-mofed5-cuda11.tar.bz2 - ubuntu18_cuda11: - build_container: ubuntu18_cuda11 - artifact_name: $(POSTFIX)-ubuntu18.04-mofed5-cuda11.tar.bz2 - ubuntu20_cuda11: - build_container: ubuntu20_cuda11 - artifact_name: $(POSTFIX)-ubuntu20.04-mofed5-cuda11.tar.bz2 - ubuntu22_cuda11: - build_container: ubuntu22_cuda11 - artifact_name: $(POSTFIX)-ubuntu22.04-mofed5-cuda11.tar.bz2 + ${{ if eq(parameters.name, 'x86_64') }}: + strategy: + matrix: + centos7_cuda11_x86_64: + build_container: centos7_cuda11_x86_64 + artifact_name: $(POSTFIX)-centos7-mofed5-cuda11-x86_64.tar.bz2 + centos8_cuda11_x86_64: + build_container: centos8_cuda11_x86_64 + artifact_name: $(POSTFIX)-centos8-mofed5-cuda11-x86_64.tar.bz2 + ubuntu16_cuda11_x86_64: + build_container: ubuntu16_cuda11_x86_64 + artifact_name: $(POSTFIX)-ubuntu16.04-mofed5-cuda11-x86_64.tar.bz2 + ubuntu18_cuda11_x86_64: + build_container: ubuntu18_cuda11_x86_64 + artifact_name: $(POSTFIX)-ubuntu18.04-mofed5-cuda11-x86_64.tar.bz2 + ubuntu20_cuda11_x86_64: + build_container: ubuntu20_cuda11_x86_64 + artifact_name: $(POSTFIX)-ubuntu20.04-mofed5-cuda11-x86_64.tar.bz2 + ubuntu22_cuda11_x86_64: + build_container: ubuntu22_cuda11_x86_64 + artifact_name: $(POSTFIX)-ubuntu22.04-mofed5-cuda11-x86_64.tar.bz2 + container: $[ variables['build_container'] ] + pool: + name: MLNX + demands: ${{ parameters.demands }} - container: $[ variables['build_container'] ] + ${{ if eq(parameters.name, 'aarch64') }}: + strategy: + matrix: + centos8_cuda11_aarch64: + build_container: centos8_cuda11_aarch64 + artifact_name: $(POSTFIX)-centos8-mofed5-cuda11-aarch64.tar.bz2 + ubuntu18_cuda11_aarch64: + build_container: ubuntu18_cuda11_aarch64 + artifact_name: $(POSTFIX)-ubuntu18.04-mofed5-cuda11-aarch64.tar.bz2 + ubuntu20_cuda11_aarch64: + build_container: ubuntu20_cuda11_aarch64 + artifact_name: $(POSTFIX)-ubuntu20.04-mofed5-cuda11-aarch64.tar.bz2 + ubuntu22_cuda11_aarch64: + build_container: ubuntu22_cuda11_aarch64 + artifact_name: $(POSTFIX)-ubuntu22.04-mofed5-cuda11-aarch64.tar.bz2 + container: $[ variables['build_container'] ] + pool: + name: MLNX + demands: ${{ parameters.demands }} steps: - checkout: self @@ -48,7 +73,7 @@ jobs: cd pkg-build ../contrib/configure-release --with-cuda --with-java=no ../contrib/buildrpm.sh -s -t -b --noclean - cd rpm-dist/`uname -m` + cd rpm-dist/$(uname -m) tar -cjf "../../../${AZ_ARTIFACT_NAME}" *.rpm cd ../../.. tar -tjf "${AZ_ARTIFACT_NAME}" @@ -70,9 +95,15 @@ jobs: echo 10 > debian/compat # https://www.debian.org/doc/manuals/maint-guide/dother.en.htmdpl#compat dpkg-buildpackage -us -uc -Pcuda,xpmem cd .. # Move back to the working directory +<<<<<<< HEAD # Rename DEB files VER="${POSTFIX#ucx-}" # Remove 'ucx-' prefix from the POSTFIX string +======= + find . -name '*.deb' + VER="${POSTFIX#ucx-}" # Remove 'ucx-' prefix from the POSTFIX string + # Rename DEB files +>>>>>>> AZP/RELEASE: Add ARM release - Part #2 find . -name "ucx*.deb" -exec bash -c 'mv "$1" "${1%%_*}-'"${VER}"'.deb"' _ {} \; find . -name '*.deb' # Show new names diff --git a/buildlib/azure-pipelines-release.yml b/buildlib/azure-pipelines-release.yml index c13bf6e8a2c4..729409f926d8 100644 --- a/buildlib/azure-pipelines-release.yml +++ b/buildlib/azure-pipelines-release.yml @@ -14,17 +14,30 @@ variables: resources: containers: - - container: centos7_cuda11 - image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7-mofed5-cuda11:2 + # x86_64 + - container: centos7_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos7-mofed5-cuda11:5 options: $(DOCKER_OPT_VOLUMES) - - container: centos8_cuda11 - image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos8-mofed5-cuda11:2 - - container: ubuntu18_cuda11 - image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu18.04-mofed5-cuda11:3 - - container: ubuntu20_cuda11 - image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu20.04-mofed5-cuda11:3 - - container: ubuntu22_cuda11 - image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu22.04-mofed5-cuda11:3 + - container: centos8_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:5 + - container: ubuntu16_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu16.04-mofed5-cuda11:5 + - container: ubuntu18_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu18.04-mofed5-cuda11:5 + - container: ubuntu20_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu20.04-mofed5-cuda11:5 + - container: ubuntu22_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu22.04-mofed5-cuda11:5 + # aarch64 + - container: centos8_cuda11_aarch64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:5 + - container: ubuntu18_cuda11_aarch64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu18.04-mofed5-cuda11:5 + - container: ubuntu20_cuda11_aarch64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu20.04-mofed5-cuda11:5 + - container: ubuntu22_cuda11_aarch64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu22.04-mofed5-cuda11:5 + stages: - stage: Prepare @@ -33,7 +46,7 @@ stages: pool: name: MLNX demands: - - ucx_docker -equals yes + - ucx_docker steps: - checkout: self fetchDepth: 100 @@ -47,50 +60,23 @@ stages: name: Result displayName: Check build condition - # Create an empty draft to avoid race condition in distro releases + - stage: GitHubDraft condition: eq(dependencies.Prepare.outputs['CheckRelease.Result.Launch'], 'True') dependsOn: Prepare jobs: - - job: DraftRelease - displayName: Create draft release - container: centos7_cuda11 - pool: - name: MLNX - demands: - - ucx_docker -equals yes - steps: - - checkout: self - clean: true - fetchDepth: 100 - path: "we/need/to/go/deeper" + - template: az-GitHubDraft.yml + parameters: + name: x86_64 + container: centos7_cuda11_x86_64 + demands: ucx_docker - - bash: ./autogen.sh - displayName: Setup autotools + - template: az-GitHubDraft.yml + parameters: + name: aarch64 + container: centos8_cuda11_aarch64 + demands: ucx-arm64 - - bash: | - set -eE - gcc --version - ./contrib/configure-release --with-java=no - ./contrib/buildrpm.sh -s -t -b - displayName: Build tarball - - - task: GithubRelease@0 - condition: eq(variables['Build.Reason'], 'IndividualCI') - displayName: Create/edit GitHub Draft Release - inputs: - githubConnection: release - repositoryName: openucx/ucx - action: edit - tag: $(Build.SourceBranchName) - isDraft: true - addChangeLog: false - releaseNotesSource: file - releaseNotesFile: NEWS - assetUploadMode: replace - assets: | - ./ucx-*.tar.gz - ./rpm-dist/ucx-*.src.rpm - stage: Build displayName: Build binary packages @@ -100,6 +86,27 @@ stages: condition: eq(dependencies.Prepare.outputs['CheckRelease.Result.Launch'], 'True') jobs: - template: az-distro-release.yml + parameters: + name: x86_64 + demands: ucx_docker + + - template: az-distro-release.yml + parameters: + name: aarch64 + demands: ucx-arm64 + + - template: jucx/jucx-build.yml + parameters: + name: amd64 # x86_64 + container: centos7_cuda11_x86_64 + demands: ucx_docker + + - template: jucx/jucx-build.yml + parameters: + name: aarch64 # ARM + container: centos8_cuda11_aarch64 + demands: ucx-arm64 + - template: jucx/jucx-publish.yml parameters: ${{ if eq(variables['Build.Reason'], 'IndividualCI') }}: