AZP/RELEASE: Rework JUCX verification & release processes - v1.17.x

bindings/java/pom.xml.in

@@ -117,6 +117,39 @@
         </build>
       </profile>
 
+      <profile>
+        <id>local-deploy</id>
+        <distributionManagement>
+          <repository>
+            <id>local-repo</id>
+            <name>Local Repository</name>
+            <url>file://bindings/java/src/main/native/build-java/</url>
+          </repository>
+        </distributionManagement>
+        <build>
+          <plugins>
+            <!-- Disable GPG signing -->
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-gpg-plugin</artifactId>
+              <executions>
+                <execution>
+                  <id>sign-artifacts</id>
+                  <phase>none</phase>
+                </execution>
+              </executions>
+            </plugin>
+            <!-- Skip nexus staging -->
+            <plugin>
+              <groupId>org.sonatype.plugins</groupId>
+              <artifactId>nexus-staging-maven-plugin</artifactId>
+              <configuration>
+                <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo>
+              </configuration>
+            </plugin>
+          </plugins>
+        </build>
+      </profile>
     </profiles>
 
   <issueManagement>

bindings/java/src/main/native/Makefile.am

@@ -9,7 +9,7 @@ jardir         = $(libdir)
 topdir         = $(abs_top_builddir)
 java_build_dir = $(builddir)/build-java
 maven_repo    ?= $(java_build_dir)/.deps
-jarfile        = $(java_build_dir)/jucx-@VERSION@.jar
+jarfile        = $(java_build_dir)/jucx-@VERSION@$(suffix).jar
 javadir        = $(top_srcdir)/bindings/java
 
 MVNCMD = $(MVN) -B -T 1C -f \
@@ -50,6 +50,10 @@ $(JUCX_GENERATED_H_FILES): $(STAMP_FILE)
 
 # Return CPU architecure in Java format
 arch = $(if $(filter x86_64,$(shell uname -p)),amd64,$(shell uname -p))
+
+# Append non-x86_64 arch as filename suffix
+suffix = $(if $(filter-out x86_64,$(shell uname -p)),-$(shell uname -p))
+
 libjucx = $(objdir)/libjucx_$(arch).so
 
 lib_LTLIBRARIES = libjucx.la
@@ -82,7 +86,7 @@ $(libjucx): libjucx.la
 	cp $(objdir)/libjucx.so $(libjucx)
 
 $(jarfile): $(libjucx)
-	$(MVNCMD) package -DskipTests
+	$(MVNCMD) package -DskipTests -Djar.finalName=jucx-@VERSION@$(suffix)
 	jar tf $(jarfile)
 
 package : $(jarfile)
@@ -118,17 +122,17 @@ check-jar:
 
 # Publish JUCX jar to maven central
 publish-snapshot:
-	@make set-version JUCX_VERSION=@VERSION@-SNAPSHOT
-	@make repack-jar
-	@make check-jar
+	@make set-version JUCX_VERSION=${JUCX_VERSION}-SNAPSHOT
 	@make publish
 
 publish-release:
 	@make set-version JUCX_VERSION=${JUCX_VERSION}
-	@make repack-jar
-	@make check-jar
 	@make publish
 
+publish-local:
+	@make set-version JUCX_VERSION=${JUCX_VERSION}
+	$(MVNCMD) deploy -DskipTests ${ARGS} -Plocal-deploy
+
 publish:
 	$(MVNCMD) deploy -DskipTests ${ARGS}
 

buildlib/azure-pipelines-release.yml

@@ -20,6 +20,7 @@ resources:
       options: $(DOCKER_OPT_VOLUMES)
     - container: centos8_cuda11_x86_64
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:3
+      options: $(DOCKER_OPT_VOLUMES)
     - container: ubuntu16_cuda11_x86_64
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu16.04-mofed5-cuda11:3
     - container: ubuntu18_cuda11_x86_64
@@ -41,6 +42,7 @@ resources:
     # aarch64
     - container: centos8_cuda11_aarch64
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:3
+      options: $(DOCKER_OPT_VOLUMES)
     - container: ubuntu18_cuda11_aarch64
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu18.04-mofed5-cuda11:3
     - container: ubuntu20_cuda11_aarch64
@@ -112,15 +114,13 @@ stages:
       - template: jucx/jucx-build.yml
         parameters:
           arch: amd64
-          container: centos7_cuda11_x86_64
+          container: centos8_cuda11_x86_64
           demands: ucx_docker
+          target: publish-release
 
       - template: jucx/jucx-build.yml
         parameters:
           arch: aarch64
           container: centos8_cuda11_aarch64
           demands: ucx-arm64
-
-      - template: jucx/jucx-publish.yml
-        parameters:
           target: publish-release

buildlib/azure-pipelines.yml

@@ -1,6 +1,3 @@
-# See https://aka.ms/yaml
-# This pipeline to be run on direct pushes and merges
-
 pr:
   - master
   - v*.*.x
@@ -13,12 +10,13 @@ variables:
 
 resources:
   containers:
-    - container: centos7_cuda11_x86_64
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos7-mofed5-cuda11:3
+    - container: centos8_cuda11_x86_64
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:3
       options: $(DOCKER_OPT_VOLUMES)
     - container: centos8_cuda11_aarch64
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:3
-
+      options: $(DOCKER_OPT_VOLUMES)
+
 stages:
   - stage: Prepare
     jobs:
@@ -37,25 +35,21 @@ stages:
               set -x
               check_release_build $(Build.Reason) $(Build.SourceVersion) "AZP/SNAPSHOT: "
             name: Result
+
   - stage: Build
     dependsOn: Prepare
     condition: eq(dependencies.Prepare.outputs['Check.Result.Launch'], 'True')
     jobs:
       - template: jucx/jucx-build.yml
         parameters:
           arch: amd64
-          container: centos7_cuda11_x86_64
+          container: centos8_cuda11_x86_64
           demands: ucx_docker
+          target: publish-snapshot
 
       - template: jucx/jucx-build.yml
         parameters:
           arch: aarch64
           container: centos8_cuda11_aarch64
           demands: ucx-arm64
-
-      - template: jucx/jucx-publish.yml
-        parameters:
-          ${{ if eq(variables['Build.Reason'], 'IndividualCI') }}:
-            target: publish-snapshot
-          ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-            target: package
+          target: publish-snapshot

buildlib/dockers/centos-release.Dockerfile

@@ -5,27 +5,28 @@ FROM nvidia/cuda:${CUDA_VERSION}-devel-centos${OS_VERSION}
 RUN yum install -y \
     autoconf \
     automake \
+    environment-modules \
+    ethtool \
     file \
+    fuse-libs \
     gcc-c++ \
     git \
     glibc-devel \
     libtool \
+    libusbx \
+    lsof \
     make \
     maven \
     numactl-devel \
+    pinentry \
+    python36 \
     rdma-core-devel \
     rpm-build \
     tcl \
     tcsh \
     tk \
-    wget \
-    libusbx \
-    fuse-libs \
-    python36 \
-    lsof \
-    ethtool \
-    environment-modules \
     valgrind-devel \
+    wget \
     && yum clean all
 
 # MOFED

buildlib/jucx/jucx-build.yml

@@ -2,10 +2,16 @@ parameters:
   arch:
   container:  
   demands: []
+  target:
 
 jobs:
   - job: jucx_build_${{ parameters.arch }}
     displayName: JUCX build ${{ parameters.arch }}
+    variables:
+      temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/tmp-settings.xml
+      gpg_dir: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/gpg
+      ${{ if ne(parameters.arch, 'amd64') }}:
+        SUFFIX: "-${{ parameters.arch }}"
 
     # we need to use lowest version for compatibility
     container: ${{ parameters.container }}    
@@ -31,10 +37,72 @@ jobs:
       - bash: |
           set -eEx
           make -s -j`nproc`
-        displayName: Build ucx
+          source buildlib/az-helpers.sh
+          az_init_modules
+          az_module_load dev/mvn
+          # use the lowest supported Java version for compatibility:
+          az_module_load dev/jdk-1.8
+          git fetch --unshallow
+          TAG=`git describe --tags`
+          # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
+          # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
+          MAVEN_VERSION=${TAG:1}
+          make -C bindings/java/src/main/native/ package JUCX_VERSION=${MAVEN_VERSION}
+        displayName: Build JUCX
 
-      - task: PublishBuildArtifacts@1
+      - bash: |
+          set -eE
+          {
+            echo -e "<settings><servers><server>"
+            echo -e "<id>ossrh</id><username>\${env.SONATYPE_USERNAME}</username>"
+            echo -e "<password>\${env.SONATYPE_PASSWORD}</password>"
+            echo -e "</server></servers></settings>"
+          } > $(temp_cfg)
+        displayName: Generate temporary config
+
+      - task: DownloadSecureFile@1
+        displayName: Download Secure file
+        inputs:
+          secureFile: sparkucx-secret.gpg
+        name: privateKey
+
+      - task: DownloadSecureFile@1
+        displayName: Download Secure file
         inputs:
-          pathToPublish: '$(Build.Repository.LocalPath)/bindings/java/resources'
-          artifactName: libjucx_${{ parameters.arch }}.so
-        displayName: Interim artifact - push
+          secureFile: sparkucx-public.gpg
+        name: publicKey
+
+      - bash: |
+          set -exE
+          source buildlib/az-helpers.sh
+          az_init_modules
+          az_module_load dev/mvn
+          # use the lowest supported Java version for compatibility:
+          az_module_load dev/jdk-1.8
+          mvn --version
+          mkdir -p $(gpg_dir)
+          export GPG_TTY=`tty`
+          chmod 700 $(gpg_dir)
+          cp $(publicKey.secureFilePath)  $(gpg_dir)/pubring.gpg
+          cp $(privateKey.secureFilePath) $(gpg_dir)/secring.gpg
+          export GNUPGHOME=$(gpg_dir)
+
+          # GPG agent config
+          echo "use-agent" > $GNUPGHOME/gpg.conf
+          echo "pinentry-mode loopback" >> $GNUPGHOME/gpg.conf
+          echo "allow-loopback-pinentry" > $GNUPGHOME/gpg-agent.conf
+          gpg-connect-agent reloadagent /bye || true
+
+          TAG=`git describe --tags`
+          # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
+          # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
+          MAVEN_VERSION=${TAG:1}
+          make -C bindings/java/src/main/native/ ${{ parameters.target }} \
+              ARGS="--settings $(temp_cfg)" \
+              JUCX_VERSION=${MAVEN_VERSION}${SUFFIX}
+        displayName: Publish-Maven
+        condition: eq(variables['Build.Reason'], 'IndividualCI')
+        env:
+          GPG_PASSPHRASE: $(GPG_PASSPHRASE)
+          SONATYPE_PASSWORD: $(SONATYPE_PASSWORD)
+          SONATYPE_USERNAME: $(SONATYPE_USERNAME)