AZP/RELEASE: Add ARM release - Part openucx#2

Alexey-Rivkin · Apr 3, 2023 · e71d4ae · e71d4ae
1 parent 9402567
commit e71d4ae
Show file tree

Hide file tree

Showing 3 changed files with 160 additions and 75 deletions.
diff --git a/buildlib/az-GitHubDraft.yml b/buildlib/az-GitHubDraft.yml
@@ -0,0 +1,47 @@
+# Create an empty draft to avoid race condition in distro release
+
+parameters:
+  name:
+  container:  
+  demands: []
+
+jobs:
+  - job: DraftRelease_${{ parameters.name }}
+    displayName: Draft release ${{ parameters.name }}
+    container: ${{ parameters.container }}    
+    pool:
+      name: MLNX
+      demands: ${{ parameters.demands }}
+
+    steps:
+    - checkout: self
+      clean: true
+      fetchDepth: 100
+      path: "we/need/to/go/deeper"
+
+    - bash: ./autogen.sh
+      displayName: Setup autotools
+
+    - bash: |
+        set -eEx
+        # gcc --version
+        ./contrib/configure-release --with-java=no
+        ./contrib/buildrpm.sh -s -t -b
+      displayName: Build tarball
+
+    - task: GithubRelease@0
+      condition: eq(variables['Build.Reason'], 'IndividualCI')
+      displayName: Create/edit GitHub Draft Release
+      inputs:
+        githubConnection: release
+        repositoryName: openucx/ucx
+        action: edit
+        tag: $(Build.SourceBranchName)
+        isDraft: true
+        addChangeLog: false
+        releaseNotesSource: file
+        releaseNotesFile: NEWS
+        assetUploadMode: replace
+        assets: |
+          ./ucx-*.tar.gz
+          ./rpm-dist/ucx-*.src.rpm
diff --git a/buildlib/az-distro-release.yml b/buildlib/az-distro-release.yml
@@ -1,6 +1,10 @@
+parameters:
+  name:
+  demands: []
+
 jobs:
-  - job: distro_release
-    displayName: distro
+  - job: distro_release_${{ parameters.name }}
+    displayName: Distro ${{ parameters.name }}
     variables:
       ${{ if eq(variables['Build.Reason'], 'ResourceTrigger') }}:
         POSTFIX: ucx-${{ replace(variables['Build.SourceBranch'], 'refs/heads/', '') }}
@@ -9,30 +13,51 @@ jobs:
       ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
         POSTFIX: ucx-pr$(System.PullRequest.PullRequestNumber)
 
-    pool:
-      name: MLNX
-      demands:
-        - harbor_registry -equals yes
-
-    strategy:
-      matrix:
-        centos7_cuda11:
-          build_container: centos7_cuda11
-          artifact_name: $(POSTFIX)-centos7-mofed5-cuda11.tar.bz2
-        centos8_cuda11:
-          build_container: centos8_cuda11
-          artifact_name: $(POSTFIX)-centos8-mofed5-cuda11.tar.bz2
-        ubuntu18_cuda11:
-          build_container: ubuntu18_cuda11
-          artifact_name: $(POSTFIX)-ubuntu18.04-mofed5-cuda11.tar.bz2
-        ubuntu20_cuda11:
-          build_container: ubuntu20_cuda11
-          artifact_name: $(POSTFIX)-ubuntu20.04-mofed5-cuda11.tar.bz2
-        ubuntu22_cuda11:
-          build_container: ubuntu22_cuda11
-          artifact_name: $(POSTFIX)-ubuntu22.04-mofed5-cuda11.tar.bz2
+    ${{ if eq(parameters.name, 'x86_64') }}:
+      strategy:
+        matrix:
+          centos7_cuda11_x86_64:
+            build_container: centos7_cuda11_x86_64
+            artifact_name: $(POSTFIX)-centos7-mofed5-cuda11-x86_64.tar.bz2
+          centos8_cuda11_x86_64:
+            build_container: centos8_cuda11_x86_64
+            artifact_name: $(POSTFIX)-centos8-mofed5-cuda11-x86_64.tar.bz2
+          ubuntu16_cuda11_x86_64:
+            build_container: ubuntu16_cuda11_x86_64
+            artifact_name: $(POSTFIX)-ubuntu16.04-mofed5-cuda11-x86_64.tar.bz2
+          ubuntu18_cuda11_x86_64:
+            build_container: ubuntu18_cuda11_x86_64
+            artifact_name: $(POSTFIX)-ubuntu18.04-mofed5-cuda11-x86_64.tar.bz2
+          ubuntu20_cuda11_x86_64:
+            build_container: ubuntu20_cuda11_x86_64
+            artifact_name: $(POSTFIX)-ubuntu20.04-mofed5-cuda11-x86_64.tar.bz2
+          ubuntu22_cuda11_x86_64:
+            build_container: ubuntu22_cuda11_x86_64
+            artifact_name: $(POSTFIX)-ubuntu22.04-mofed5-cuda11-x86_64.tar.bz2
+      container: $[ variables['build_container'] ]
+      pool:
+        name: MLNX
+        demands: ${{ parameters.demands }}
 
-    container: $[ variables['build_container'] ]
+    ${{ if eq(parameters.name, 'aarch64') }}:
+      strategy:
+        matrix:
+          centos8_cuda11_aarch64:
+            build_container: centos8_cuda11_aarch64
+            artifact_name: $(POSTFIX)-centos8-mofed5-cuda11-aarch64.tar.bz2
+          ubuntu18_cuda11_aarch64:
+            build_container: ubuntu18_cuda11_aarch64
+            artifact_name: $(POSTFIX)-ubuntu18.04-mofed5-cuda11-aarch64.tar.bz2
+          ubuntu20_cuda11_aarch64:
+            build_container: ubuntu20_cuda11_aarch64
+            artifact_name: $(POSTFIX)-ubuntu20.04-mofed5-cuda11-aarch64.tar.bz2
+          ubuntu22_cuda11_aarch64:
+            build_container: ubuntu22_cuda11_aarch64
+            artifact_name: $(POSTFIX)-ubuntu22.04-mofed5-cuda11-aarch64.tar.bz2
+      container: $[ variables['build_container'] ]
+      pool:
+        name: MLNX
+        demands: ${{ parameters.demands }}
 
     steps:
       - checkout: self
@@ -48,7 +73,7 @@ jobs:
           cd pkg-build
           ../contrib/configure-release --with-cuda --with-java=no
           ../contrib/buildrpm.sh -s -t -b --noclean
-          cd rpm-dist/`uname -m`
+          cd rpm-dist/$(uname -m)
           tar -cjf "../../../${AZ_ARTIFACT_NAME}" *.rpm
           cd ../../..
           tar -tjf "${AZ_ARTIFACT_NAME}"
@@ -70,9 +95,15 @@ jobs:
           echo 10 > debian/compat   # https://www.debian.org/doc/manuals/maint-guide/dother.en.htmdpl#compat
           dpkg-buildpackage -us -uc -Pcuda,xpmem
           cd ..                             # Move back to the working directory
+<<<<<<< HEAD
 
           # Rename DEB files          
           VER="${POSTFIX#ucx-}"     # Remove 'ucx-' prefix from the POSTFIX string
+=======
+          find . -name '*.deb'
+          VER="${POSTFIX#ucx-}"   # Remove 'ucx-' prefix from the POSTFIX string
+          # Rename DEB files
+>>>>>>> AZP/RELEASE: Add ARM release - Part #2
           find . -name "ucx*.deb" -exec bash -c 'mv "$1" "${1%%_*}-'"${VER}"'.deb"' _ {} \;
           find . -name '*.deb'      # Show new names
 

diff --git a/buildlib/azure-pipelines-release.yml b/buildlib/azure-pipelines-release.yml
@@ -14,17 +14,30 @@ variables:
 
 resources:
   containers:
-    - container: centos7_cuda11
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7-mofed5-cuda11:2
+    # x86_64
+    - container: centos7_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos7-mofed5-cuda11:5
       options: $(DOCKER_OPT_VOLUMES)
-    - container: centos8_cuda11
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos8-mofed5-cuda11:2
-    - container: ubuntu18_cuda11
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu18.04-mofed5-cuda11:3
-    - container: ubuntu20_cuda11
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu20.04-mofed5-cuda11:3
-    - container: ubuntu22_cuda11
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/ubuntu22.04-mofed5-cuda11:3
+    - container: centos8_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:5
+    - container: ubuntu16_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu16.04-mofed5-cuda11:5
+    - container: ubuntu18_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu18.04-mofed5-cuda11:5
+    - container: ubuntu20_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu20.04-mofed5-cuda11:5
+    - container: ubuntu22_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu22.04-mofed5-cuda11:5
+    # aarch64
+    - container: centos8_cuda11_aarch64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:5
+    - container: ubuntu18_cuda11_aarch64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu18.04-mofed5-cuda11:5
+    - container: ubuntu20_cuda11_aarch64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu20.04-mofed5-cuda11:5
+    - container: ubuntu22_cuda11_aarch64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu22.04-mofed5-cuda11:5
+
 
 stages:
   - stage: Prepare
@@ -33,7 +46,7 @@ stages:
         pool:
           name: MLNX
           demands:
-          - ucx_docker -equals yes
+          - ucx_docker
         steps:
           - checkout: self
             fetchDepth: 100
@@ -47,50 +60,23 @@ stages:
             name: Result
             displayName: Check build condition
 
-  # Create an empty draft to avoid race condition in distro releases
+
   - stage: GitHubDraft
     condition: eq(dependencies.Prepare.outputs['CheckRelease.Result.Launch'], 'True')
     dependsOn: Prepare
     jobs:
-      - job: DraftRelease
-        displayName: Create draft release
-        container: centos7_cuda11
-        pool:
-          name: MLNX
-          demands:
-          - ucx_docker -equals yes
-        steps:
-        - checkout: self
-          clean: true
-          fetchDepth: 100
-          path: "we/need/to/go/deeper"
+    - template: az-GitHubDraft.yml
+      parameters:
+        name: x86_64
+        container: centos7_cuda11_x86_64
+        demands: ucx_docker
 
-        - bash: ./autogen.sh
-          displayName: Setup autotools
+    - template: az-GitHubDraft.yml
+      parameters:
+        name: aarch64
+        container: centos8_cuda11_aarch64
+        demands: ucx-arm64
 
-        - bash: |
-            set -eE
-            gcc --version
-            ./contrib/configure-release --with-java=no
-            ./contrib/buildrpm.sh -s -t -b
-          displayName: Build tarball
-
-        - task: GithubRelease@0
-          condition: eq(variables['Build.Reason'], 'IndividualCI')
-          displayName: Create/edit GitHub Draft Release
-          inputs:
-            githubConnection: release
-            repositoryName: openucx/ucx
-            action: edit
-            tag: $(Build.SourceBranchName)
-            isDraft: true
-            addChangeLog: false
-            releaseNotesSource: file
-            releaseNotesFile: NEWS
-            assetUploadMode: replace
-            assets: |
-              ./ucx-*.tar.gz
-              ./rpm-dist/ucx-*.src.rpm
 
   - stage: Build
     displayName: Build binary packages
@@ -100,6 +86,27 @@ stages:
     condition: eq(dependencies.Prepare.outputs['CheckRelease.Result.Launch'], 'True')
     jobs:
       - template: az-distro-release.yml
+        parameters:
+          name: x86_64
+          demands: ucx_docker
+
+      - template: az-distro-release.yml
+        parameters:
+          name: aarch64
+          demands: ucx-arm64
+
+      - template: jucx/jucx-build.yml
+        parameters:
+          name: amd64   # x86_64
+          container: centos7_cuda11_x86_64
+          demands: ucx_docker
+
+      - template: jucx/jucx-build.yml
+        parameters:
+          name: aarch64 # ARM
+          container: centos8_cuda11_aarch64
+          demands: ucx-arm64
+
       - template: jucx/jucx-publish.yml
         parameters:
           ${{ if eq(variables['Build.Reason'], 'IndividualCI') }}: