AZP/RELEASE: Add ARM release - part3 for v1.15.x

bindings/java/pom.xml.in

@@ -232,7 +232,7 @@
       <resource>
         <directory>resources</directory>
         <includes>
-          <include>libjucx.so</include>
+          <include>libjucx_*.so</include>
         </includes>
       </resource>
     </resources>

bindings/java/src/main/java/org/openucx/jucx/NativeLibs.java

@@ -24,7 +24,7 @@ public class NativeLibs {
         loadLibrary(UCS);   // UCS library
         loadLibrary(UCT);   // UCT library
         loadLibrary(UCP);   // UCP library
-        loadLibrary(JUCX);  // JUCX native library
+        loadLibrary(JUCX + "_" + System.getProperty("os.arch"));    // JUCX native library
     }
 
     public static void load() {

bindings/java/src/main/native/Makefile.am

@@ -48,6 +48,10 @@ $(STAMP_FILE): \
 
 $(JUCX_GENERATED_H_FILES): $(STAMP_FILE)
 
+# Return CPU architecure in Java format
+arch = $(if $(filter x86_64,$(shell uname -p)),amd64,$(shell uname -p))
+libjucx = $(objdir)/libjucx_$(arch).so
+
 lib_LTLIBRARIES = libjucx.la
 
 libjucx_la_CPPFLAGS = -I$(JDK)/include -I$(JDK)/include/linux \
@@ -74,8 +78,12 @@ libjucx_la_LIBADD = $(topdir)/src/ucs/libucs.la \
 libjucx_la_DEPENDENCIES = Makefile.am Makefile.in Makefile
 
 # Compile Java source code and pack to jar
-$(jarfile): libjucx.la
+$(libjucx): libjucx.la
+	cp $(objdir)/libjucx.so $(libjucx)
+
+$(jarfile): $(libjucx)
 	$(MVNCMD) package -DskipTests
+	jar tf $(jarfile)
 
 package : $(jarfile)
 
@@ -91,13 +99,26 @@ clean-local:
 set-version:
 	$(MVNCMD) versions:set -DnewVersion=${JUCX_VERSION}
 
+# Add multi-platform support (CI only)
+repack-jar: $(jarfile)
+	$(RM) -r $(java_build_dir)/repack_dir
+	unzip -o $(jarfile) -d $(java_build_dir)/repack_dir
+	\cp $(topdir)/libjucx_*.so/libjucx_*.so $(java_build_dir)/repack_dir
+	jar -cf $(jarfile) -C $(java_build_dir)/repack_dir .
+	jar tf $(jarfile)
+
+multi-arch:
+	@make set-version JUCX_VERSION=${JUCX_VERSION}
+	@make repack-jar
+
 # Publish JUCX jar to maven central
 publish-snapshot:
 	@make set-version JUCX_VERSION=@VERSION@-SNAPSHOT
 	@make publish
 
 publish-release:
 	@make set-version JUCX_VERSION=${JUCX_VERSION}
+	@make repack-jar
 	@make publish
 
 publish:

buildlib/az-helpers.sh

@@ -188,3 +188,16 @@ check_release_build() {
 
     echo "##vso[task.setvariable variable=Launch;isOutput=true]${launch}"
 }
+
+
+#
+# Return arch in the same format as Java System.getProperty("os.arch")
+#
+get_arch() {
+    arch=$(uname -m)
+    if [ "$arch" == "x86_64" ]; then
+        echo "amd64"
+    else
+        echo "$arch"
+    fi
+}

buildlib/azure-pipelines-release.yml

@@ -107,3 +107,17 @@ stages:
         parameters:
           arch: aarch64
           demands: ucx-arm64
+
+      - template: jucx/jucx-build.yml
+        parameters:
+          arch: amd64
+          container: centos7_cuda11_x86_64
+          demands: ucx_docker
+
+      - template: jucx/jucx-build.yml
+        parameters:
+          arch: aarch64
+          container: centos8_cuda11_aarch64
+          demands: ucx-arm64
+
+      - template: jucx/jucx-publish.yml

buildlib/jucx/jucx-build.yml

@@ -0,0 +1,39 @@
+parameters:
+  arch:
+  container:  
+  demands: []
+
+jobs:
+  - job: jucx_build_${{ parameters.arch }}
+    displayName: JUCX build ${{ parameters.arch }}
+
+    # we need to use lowest version for compatibility
+    container: ${{ parameters.container }}    
+    pool:
+      name: MLNX
+      demands: ${{ parameters.demands }}
+
+    steps:
+      - checkout: self
+        clean: true
+        fetchDepth: 100
+        path: "we/need/to/go/deeper"
+        # ^workaround agent issue with container in root path
+
+      - bash: |
+          set -eEx
+          gcc --version
+          ./autogen.sh
+          ./contrib/configure-release --with-java
+        displayName: Configure
+
+      - bash: |
+          set -eEx
+          make -s -j`nproc`
+        displayName: Build ucx
+
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathToPublish: '$(Build.Repository.LocalPath)/bindings/java/resources'
+          artifactName: libjucx_${{ parameters.arch }}.so
+        displayName: Interim artifact - push

buildlib/jucx/jucx-publish.yml

@@ -1,18 +1,21 @@
 parameters:
-  temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/tmp-settings.xml
+  temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/tmp-settings.xml
   gpg_dir: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/gpg
-  target: package
 
 jobs:
-  - job: jucx_release
+  - job: jucx_publish
+    displayName: JUCX publish
+    dependsOn:
+    - jucx_build_amd64
+    - jucx_build_aarch64
+    condition: succeeded()
 
     pool:
       name: MLNX
-      demands:
-        - harbor_registry -equals yes
+      demands: ucx_docker
 
     # we need to use lowest version for compatible
-    container: centos7_cuda11
+    container: centos7_cuda11_x86_64
 
     steps:
       - checkout: self
@@ -21,18 +24,35 @@ jobs:
         path: "we/need/to/go/deeper"
         # ^workaround agent issue with container in root path
 
+      - task: DownloadBuildArtifacts@0
+        displayName: Interim artifact - get
+        inputs:
+          downloadType: 'specific'
+          itemPattern: 'libjucx_*.so/libjucx_*.so'
+          downloadPath: '$(System.DefaultWorkingDirectory)'
+
       - bash: |
           set -eEx
           gcc --version
           ./autogen.sh
           ./contrib/configure-release --with-java
-        displayName: Configure
-
-      - bash: |
-          set -eEx
           make -s -j`nproc`
         displayName: Build ucx
 
+      - bash: |
+          source buildlib/az-helpers.sh
+          az_init_modules
+          az_module_load dev/mvn
+          # use the lowest supported Java version for compatibility:
+          az_module_load dev/jdk-1.8
+          TAG=`git describe --tags`
+          # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
+          # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
+          MAVEN_VERSION=${TAG:1}
+          make -C bindings/java/src/main/native/ multi-arch JUCX_VERSION=${MAVEN_VERSION}
+        displayName: Build multi-arch JUCX without publishing
+        condition: eq(variables['Build.Reason'], 'PullRequest')
+
       - bash: |
           set -eE
           {
@@ -42,6 +62,7 @@ jobs:
             echo -e "</server></servers></settings>"
           } > ${{ parameters.temp_cfg }}
         displayName: Generate temporary config
+        condition: eq(variables['Build.Reason'], 'IndividualCI')
 
       - task: DownloadSecureFile@1
         displayName: Download Secure file
@@ -62,7 +83,7 @@ jobs:
           # use the lowest supported Java version for compatibility:
           az_module_load dev/jdk-1.8
           mvn --version
-          mkdir ${{ parameters.gpg_dir }}
+          mkdir -p ${{ parameters.gpg_dir }}
           export GPG_TTY=`tty`
           chmod 700 ${{ parameters.gpg_dir }}
           cp $(publicKey.secureFilePath)  ${{ parameters.gpg_dir }}/pubring.gpg
@@ -72,9 +93,10 @@ jobs:
           # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
           # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
           MAVEN_VERSION=${TAG:1}
-          make -C bindings/java/src/main/native/ ${{ parameters.target }} \
+          make -C bindings/java/src/main/native/ publish-release \
               ARGS="--settings ${{ parameters.temp_cfg }}" JUCX_VERSION=${MAVEN_VERSION}
         displayName: Publish JUCX jar to maven central
+        condition: eq(variables['Build.Reason'], 'IndividualCI')
         env:
           GPG_PASSPHRASE: $(GPG_PASSPHRASE)
           SONATYPE_PASSWORD: $(SONATYPE_PASSWORD)

buildlib/jucx/jucx-test.yml

@@ -36,6 +36,7 @@ jobs:
           make -j`nproc`
           make install
         displayName: Build UCX
+
       - bash: |
           set -xeE
           source buildlib/az-helpers.sh