diff --git a/bindings/java/pom.xml.in b/bindings/java/pom.xml.in
index 9cd193d5a39..24dde523c5a 100644
--- a/bindings/java/pom.xml.in
+++ b/bindings/java/pom.xml.in
@@ -117,6 +117,39 @@
+
+ local-deploy
+
+
+ local-repo
+ Local Repository
+ file://bindings/java/src/main/native/build-java/
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+
+
+ sign-artifacts
+ none
+
+
+
+
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+
+ true
+
+
+
+
+
diff --git a/bindings/java/src/main/native/Makefile.am b/bindings/java/src/main/native/Makefile.am
index cef45cb822c..85032bffcea 100644
--- a/bindings/java/src/main/native/Makefile.am
+++ b/bindings/java/src/main/native/Makefile.am
@@ -9,7 +9,7 @@ jardir = $(libdir)
topdir = $(abs_top_builddir)
java_build_dir = $(builddir)/build-java
maven_repo ?= $(java_build_dir)/.deps
-jarfile = $(java_build_dir)/jucx-@VERSION@.jar
+jarfile = $(java_build_dir)/jucx-@VERSION@$(suffix).jar
javadir = $(top_srcdir)/bindings/java
MVNCMD = $(MVN) -B -T 1C -f \
@@ -50,6 +50,10 @@ $(JUCX_GENERATED_H_FILES): $(STAMP_FILE)
# Return CPU architecure in Java format
arch = $(if $(filter x86_64,$(shell uname -p)),amd64,$(shell uname -p))
+
+# Append non-x86_64 arch as filename suffix
+suffix = $(if $(filter-out x86_64,$(shell uname -p)),-$(shell uname -p))
+
libjucx = $(objdir)/libjucx_$(arch).so
lib_LTLIBRARIES = libjucx.la
@@ -82,7 +86,7 @@ $(libjucx): libjucx.la
cp $(objdir)/libjucx.so $(libjucx)
$(jarfile): $(libjucx)
- $(MVNCMD) package -DskipTests
+ $(MVNCMD) package -DskipTests -Djar.finalName=jucx-@VERSION@$(suffix)
jar tf $(jarfile)
package : $(jarfile)
@@ -118,17 +122,17 @@ check-jar:
# Publish JUCX jar to maven central
publish-snapshot:
- @make set-version JUCX_VERSION=@VERSION@-SNAPSHOT
- @make repack-jar
- @make check-jar
+ @make set-version JUCX_VERSION=${JUCX_VERSION}-SNAPSHOT
@make publish
publish-release:
@make set-version JUCX_VERSION=${JUCX_VERSION}
- @make repack-jar
- @make check-jar
@make publish
+publish-local:
+ @make set-version JUCX_VERSION=${JUCX_VERSION}
+ $(MVNCMD) deploy -DskipTests ${ARGS} -Plocal-deploy
+
publish:
$(MVNCMD) deploy -DskipTests ${ARGS}
diff --git a/buildlib/azure-pipelines-release.yml b/buildlib/azure-pipelines-release.yml
index 34bc99e8dca..63f67fb7cd1 100644
--- a/buildlib/azure-pipelines-release.yml
+++ b/buildlib/azure-pipelines-release.yml
@@ -20,6 +20,7 @@ resources:
options: $(DOCKER_OPT_VOLUMES)
- container: centos8_cuda11_x86_64
image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:3
+ options: $(DOCKER_OPT_VOLUMES)
- container: ubuntu16_cuda11_x86_64
image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu16.04-mofed5-cuda11:3
- container: ubuntu18_cuda11_x86_64
@@ -41,6 +42,7 @@ resources:
# aarch64
- container: centos8_cuda11_aarch64
image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:3
+ options: $(DOCKER_OPT_VOLUMES)
- container: ubuntu18_cuda11_aarch64
image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu18.04-mofed5-cuda11:3
- container: ubuntu20_cuda11_aarch64
@@ -112,15 +114,13 @@ stages:
- template: jucx/jucx-build.yml
parameters:
arch: amd64
- container: centos7_cuda11_x86_64
+ container: centos8_cuda11_x86_64
demands: ucx_docker
+ target: publish-release
- template: jucx/jucx-build.yml
parameters:
arch: aarch64
container: centos8_cuda11_aarch64
demands: ucx-arm64
-
- - template: jucx/jucx-publish.yml
- parameters:
target: publish-release
diff --git a/buildlib/azure-pipelines.yml b/buildlib/azure-pipelines.yml
index 0a11f50a5d4..b4b3a62ab7a 100644
--- a/buildlib/azure-pipelines.yml
+++ b/buildlib/azure-pipelines.yml
@@ -1,6 +1,3 @@
-# See https://aka.ms/yaml
-# This pipeline to be run on direct pushes and merges
-
pr:
- master
- v*.*.x
@@ -13,12 +10,13 @@ variables:
resources:
containers:
- - container: centos7_cuda11_x86_64
- image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos7-mofed5-cuda11:3
+ - container: centos8_cuda11_x86_64
+ image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:3
options: $(DOCKER_OPT_VOLUMES)
- container: centos8_cuda11_aarch64
image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:3
-
+ options: $(DOCKER_OPT_VOLUMES)
+
stages:
- stage: Prepare
jobs:
@@ -37,6 +35,7 @@ stages:
set -x
check_release_build $(Build.Reason) $(Build.SourceVersion) "AZP/SNAPSHOT: "
name: Result
+
- stage: Build
dependsOn: Prepare
condition: eq(dependencies.Prepare.outputs['Check.Result.Launch'], 'True')
@@ -44,18 +43,13 @@ stages:
- template: jucx/jucx-build.yml
parameters:
arch: amd64
- container: centos7_cuda11_x86_64
+ container: centos8_cuda11_x86_64
demands: ucx_docker
+ target: publish-snapshot
- template: jucx/jucx-build.yml
parameters:
arch: aarch64
container: centos8_cuda11_aarch64
demands: ucx-arm64
-
- - template: jucx/jucx-publish.yml
- parameters:
- ${{ if eq(variables['Build.Reason'], 'IndividualCI') }}:
- target: publish-snapshot
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- target: package
+ target: publish-snapshot
diff --git a/buildlib/dockers/centos-release.Dockerfile b/buildlib/dockers/centos-release.Dockerfile
index 8f4039ee893..ab806e199f3 100644
--- a/buildlib/dockers/centos-release.Dockerfile
+++ b/buildlib/dockers/centos-release.Dockerfile
@@ -5,27 +5,28 @@ FROM nvidia/cuda:${CUDA_VERSION}-devel-centos${OS_VERSION}
RUN yum install -y \
autoconf \
automake \
+ environment-modules \
+ ethtool \
file \
+ fuse-libs \
gcc-c++ \
git \
glibc-devel \
libtool \
+ libusbx \
+ lsof \
make \
maven \
numactl-devel \
+ pinentry \
+ python36 \
rdma-core-devel \
rpm-build \
tcl \
tcsh \
tk \
- wget \
- libusbx \
- fuse-libs \
- python36 \
- lsof \
- ethtool \
- environment-modules \
valgrind-devel \
+ wget \
&& yum clean all
# MOFED
diff --git a/buildlib/jucx/jucx-build.yml b/buildlib/jucx/jucx-build.yml
index 7d5f6abfee8..a5e113d6d15 100644
--- a/buildlib/jucx/jucx-build.yml
+++ b/buildlib/jucx/jucx-build.yml
@@ -2,10 +2,16 @@ parameters:
arch:
container:
demands: []
+ target:
jobs:
- job: jucx_build_${{ parameters.arch }}
displayName: JUCX build ${{ parameters.arch }}
+ variables:
+ temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/tmp-settings.xml
+ gpg_dir: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/gpg
+ ${{ if ne(parameters.arch, 'amd64') }}:
+ SUFFIX: "-${{ parameters.arch }}"
# we need to use lowest version for compatibility
container: ${{ parameters.container }}
@@ -31,10 +37,72 @@ jobs:
- bash: |
set -eEx
make -s -j`nproc`
- displayName: Build ucx
+ source buildlib/az-helpers.sh
+ az_init_modules
+ az_module_load dev/mvn
+ # use the lowest supported Java version for compatibility:
+ az_module_load dev/jdk-1.8
+ git fetch --unshallow
+ TAG=`git describe --tags`
+ # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
+ # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
+ MAVEN_VERSION=${TAG:1}
+ make -C bindings/java/src/main/native/ package JUCX_VERSION=${MAVEN_VERSION}
+ displayName: Build JUCX
- - task: PublishBuildArtifacts@1
+ - bash: |
+ set -eE
+ {
+ echo -e ""
+ echo -e "ossrh\${env.SONATYPE_USERNAME}"
+ echo -e "\${env.SONATYPE_PASSWORD}"
+ echo -e ""
+ } > $(temp_cfg)
+ displayName: Generate temporary config
+
+ - task: DownloadSecureFile@1
+ displayName: Download Secure file
+ inputs:
+ secureFile: sparkucx-secret.gpg
+ name: privateKey
+
+ - task: DownloadSecureFile@1
+ displayName: Download Secure file
inputs:
- pathToPublish: '$(Build.Repository.LocalPath)/bindings/java/resources'
- artifactName: libjucx_${{ parameters.arch }}.so
- displayName: Interim artifact - push
+ secureFile: sparkucx-public.gpg
+ name: publicKey
+
+ - bash: |
+ set -exE
+ source buildlib/az-helpers.sh
+ az_init_modules
+ az_module_load dev/mvn
+ # use the lowest supported Java version for compatibility:
+ az_module_load dev/jdk-1.8
+ mvn --version
+ mkdir -p $(gpg_dir)
+ export GPG_TTY=`tty`
+ chmod 700 $(gpg_dir)
+ cp $(publicKey.secureFilePath) $(gpg_dir)/pubring.gpg
+ cp $(privateKey.secureFilePath) $(gpg_dir)/secring.gpg
+ export GNUPGHOME=$(gpg_dir)
+
+ # GPG agent config
+ echo "use-agent" > $GNUPGHOME/gpg.conf
+ echo "pinentry-mode loopback" >> $GNUPGHOME/gpg.conf
+ echo "allow-loopback-pinentry" > $GNUPGHOME/gpg-agent.conf
+ gpg-connect-agent reloadagent /bye || true
+
+ TAG=`git describe --tags`
+ # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
+ # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
+ MAVEN_VERSION=${TAG:1}
+ make -C bindings/java/src/main/native/ ${{ parameters.target }} \
+ ARGS="--settings $(temp_cfg)" \
+ JUCX_VERSION=${MAVEN_VERSION}${SUFFIX}
+ displayName: Publish-Maven
+ condition: eq(variables['Build.Reason'], 'IndividualCI')
+ env:
+ GPG_PASSPHRASE: $(GPG_PASSPHRASE)
+ SONATYPE_PASSWORD: $(SONATYPE_PASSWORD)
+ SONATYPE_USERNAME: $(SONATYPE_USERNAME)
diff --git a/buildlib/jucx/jucx-publish.yml b/buildlib/jucx/jucx-publish.yml
deleted file mode 100644
index 08ae11849cd..00000000000
--- a/buildlib/jucx/jucx-publish.yml
+++ /dev/null
@@ -1,105 +0,0 @@
-parameters:
- temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/tmp-settings.xml
- gpg_dir: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/gpg
- target:
-
-jobs:
- - job: jucx_publish
- displayName: JUCX publish
- dependsOn:
- - jucx_build_amd64
- - jucx_build_aarch64
- condition: succeeded()
-
- pool:
- name: MLNX
- demands: ucx_docker
-
- # we need to use lowest version for compatible
- container: centos7_cuda11_x86_64
-
- steps:
- - checkout: self
- clean: true
- fetchDepth: 300
- retryCountOnTaskFailure: 5
- path: "we/need/to/go/deeper"
- # ^workaround agent issue with container in root path
-
- - task: DownloadBuildArtifacts@0
- displayName: Interim artifact - get
- inputs:
- downloadType: 'specific'
- itemPattern: 'libjucx_*.so/libjucx_*.so'
- downloadPath: '$(System.DefaultWorkingDirectory)'
-
- - bash: |
- set -eEx
- gcc --version
- ./autogen.sh
- ./contrib/configure-release --with-java
- make -s -j`nproc`
- displayName: Build ucx
-
- - bash: |
- set -exE
- source buildlib/az-helpers.sh
- az_init_modules
- az_module_load dev/mvn
- # use the lowest supported Java version for compatibility:
- az_module_load dev/jdk-1.8
- TAG=`git describe --tags`
- # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
- # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
- MAVEN_VERSION=${TAG:1}
- make -C bindings/java/src/main/native/ multi-arch JUCX_VERSION=${MAVEN_VERSION}
- displayName: Build multi-arch JUCX without publishing
- condition: eq(variables['Build.Reason'], 'PullRequest')
-
- - bash: |
- set -eE
- {
- echo -e ""
- echo -e "ossrh\${env.SONATYPE_USERNAME}"
- echo -e "\${env.SONATYPE_PASSWORD}"
- echo -e ""
- } > ${{ parameters.temp_cfg }}
- displayName: Generate temporary config
-
- - task: DownloadSecureFile@1
- displayName: Download Secure file
- inputs:
- secureFile: sparkucx-secret.gpg
- name: privateKey
-
- - task: DownloadSecureFile@1
- displayName: Download Secure file
- inputs:
- secureFile: sparkucx-public.gpg
- name: publicKey
-
- - bash: |
- set -exE
- source buildlib/az-helpers.sh
- az_init_modules
- az_module_load dev/mvn
- # use the lowest supported Java version for compatibility:
- az_module_load dev/jdk-1.8
- mvn --version
- mkdir -p ${{ parameters.gpg_dir }}
- export GPG_TTY=`tty`
- chmod 700 ${{ parameters.gpg_dir }}
- cp $(publicKey.secureFilePath) ${{ parameters.gpg_dir }}/pubring.gpg
- cp $(privateKey.secureFilePath) ${{ parameters.gpg_dir }}/secring.gpg
- export GNUPGHOME=${{ parameters.gpg_dir }}
- TAG=`git describe --tags`
- # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,...
- # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string
- MAVEN_VERSION=${TAG:1}
- make -C bindings/java/src/main/native/ ${{ parameters.target }} \
- ARGS="--settings ${{ parameters.temp_cfg }}" JUCX_VERSION=${MAVEN_VERSION}
- displayName: Publish JUCX jar to maven central
- env:
- GPG_PASSPHRASE: $(GPG_PASSPHRASE)
- SONATYPE_PASSWORD: $(SONATYPE_PASSWORD)
- SONATYPE_USERNAME: $(SONATYPE_USERNAME)
diff --git a/buildlib/jucx/jucx-test.yml b/buildlib/jucx/jucx-test.yml
index 9b771c755f0..e5ba1e0541f 100755
--- a/buildlib/jucx/jucx-test.yml
+++ b/buildlib/jucx/jucx-test.yml
@@ -1,9 +1,11 @@
parameters:
+ arch:
name: java test
demands: []
jobs:
- - job: ${{ parameters.name }}
+ - job: jucx_test_${{ parameters.arch }}
+ displayName: JUCX Test ${{ parameters.arch }}
workspace:
clean: outputs
@@ -39,6 +41,21 @@ jobs:
make install
displayName: Build UCX
+ - bash: |
+ set -exE
+ source buildlib/az-helpers.sh
+ az_init_modules
+ az_module_load dev/mvn
+ az_module_load dev/jdk-1.8
+ git fetch --unshallow
+ TAG=`git describe --tags`
+ MAVEN_VERSION=${TAG:1}
+ make -C bindings/java/src/main/native/ publish-local \
+ JUCX_VERSION=${MAVEN_VERSION}${SUFFIX}
+ displayName: Publish-Local
+ condition: eq(variables['Build.Reason'], 'PullRequest')
+
+
- bash: |
set -xeE
source buildlib/az-helpers.sh
diff --git a/buildlib/pr/main.yml b/buildlib/pr/main.yml
index e31b54f61cf..4232bc3172b 100644
--- a/buildlib/pr/main.yml
+++ b/buildlib/pr/main.yml
@@ -356,12 +356,9 @@ stages:
jobs:
- template: ../jucx/jucx-test.yml
parameters:
- name: new
- demands: ucx_new -equals yes
- - template: ../jucx/jucx-test.yml
- parameters:
+ arch: amd64
name: gpu
- demands: ucx_gpu -equals yes
+ demands: ucx_gpu
- stage: go
dependsOn: [Static_check]