diff --git a/bindings/java/pom.xml.in b/bindings/java/pom.xml.in index 9cd193d5a39..24dde523c5a 100644 --- a/bindings/java/pom.xml.in +++ b/bindings/java/pom.xml.in @@ -117,6 +117,39 @@ + + local-deploy + + + local-repo + Local Repository + file://bindings/java/src/main/native/build-java/ + + + + + + + org.apache.maven.plugins + maven-gpg-plugin + + + sign-artifacts + none + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + + true + + + + + diff --git a/bindings/java/src/main/native/Makefile.am b/bindings/java/src/main/native/Makefile.am index cef45cb822c..85032bffcea 100644 --- a/bindings/java/src/main/native/Makefile.am +++ b/bindings/java/src/main/native/Makefile.am @@ -9,7 +9,7 @@ jardir = $(libdir) topdir = $(abs_top_builddir) java_build_dir = $(builddir)/build-java maven_repo ?= $(java_build_dir)/.deps -jarfile = $(java_build_dir)/jucx-@VERSION@.jar +jarfile = $(java_build_dir)/jucx-@VERSION@$(suffix).jar javadir = $(top_srcdir)/bindings/java MVNCMD = $(MVN) -B -T 1C -f \ @@ -50,6 +50,10 @@ $(JUCX_GENERATED_H_FILES): $(STAMP_FILE) # Return CPU architecure in Java format arch = $(if $(filter x86_64,$(shell uname -p)),amd64,$(shell uname -p)) + +# Append non-x86_64 arch as filename suffix +suffix = $(if $(filter-out x86_64,$(shell uname -p)),-$(shell uname -p)) + libjucx = $(objdir)/libjucx_$(arch).so lib_LTLIBRARIES = libjucx.la @@ -82,7 +86,7 @@ $(libjucx): libjucx.la cp $(objdir)/libjucx.so $(libjucx) $(jarfile): $(libjucx) - $(MVNCMD) package -DskipTests + $(MVNCMD) package -DskipTests -Djar.finalName=jucx-@VERSION@$(suffix) jar tf $(jarfile) package : $(jarfile) @@ -118,17 +122,17 @@ check-jar: # Publish JUCX jar to maven central publish-snapshot: - @make set-version JUCX_VERSION=@VERSION@-SNAPSHOT - @make repack-jar - @make check-jar + @make set-version JUCX_VERSION=${JUCX_VERSION}-SNAPSHOT @make publish publish-release: @make set-version JUCX_VERSION=${JUCX_VERSION} - @make repack-jar - @make check-jar @make publish +publish-local: + @make set-version JUCX_VERSION=${JUCX_VERSION} + $(MVNCMD) deploy -DskipTests ${ARGS} -Plocal-deploy + publish: $(MVNCMD) deploy -DskipTests ${ARGS} diff --git a/buildlib/azure-pipelines-release.yml b/buildlib/azure-pipelines-release.yml index 34bc99e8dca..63f67fb7cd1 100644 --- a/buildlib/azure-pipelines-release.yml +++ b/buildlib/azure-pipelines-release.yml @@ -20,6 +20,7 @@ resources: options: $(DOCKER_OPT_VOLUMES) - container: centos8_cuda11_x86_64 image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:3 + options: $(DOCKER_OPT_VOLUMES) - container: ubuntu16_cuda11_x86_64 image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu16.04-mofed5-cuda11:3 - container: ubuntu18_cuda11_x86_64 @@ -41,6 +42,7 @@ resources: # aarch64 - container: centos8_cuda11_aarch64 image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:3 + options: $(DOCKER_OPT_VOLUMES) - container: ubuntu18_cuda11_aarch64 image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/ubuntu18.04-mofed5-cuda11:3 - container: ubuntu20_cuda11_aarch64 @@ -112,15 +114,13 @@ stages: - template: jucx/jucx-build.yml parameters: arch: amd64 - container: centos7_cuda11_x86_64 + container: centos8_cuda11_x86_64 demands: ucx_docker + target: publish-release - template: jucx/jucx-build.yml parameters: arch: aarch64 container: centos8_cuda11_aarch64 demands: ucx-arm64 - - - template: jucx/jucx-publish.yml - parameters: target: publish-release diff --git a/buildlib/azure-pipelines.yml b/buildlib/azure-pipelines.yml index 0a11f50a5d4..b4b3a62ab7a 100644 --- a/buildlib/azure-pipelines.yml +++ b/buildlib/azure-pipelines.yml @@ -1,6 +1,3 @@ -# See https://aka.ms/yaml -# This pipeline to be run on direct pushes and merges - pr: - master - v*.*.x @@ -13,12 +10,13 @@ variables: resources: containers: - - container: centos7_cuda11_x86_64 - image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos7-mofed5-cuda11:3 + - container: centos8_cuda11_x86_64 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/centos8-mofed5-cuda11:3 options: $(DOCKER_OPT_VOLUMES) - container: centos8_cuda11_aarch64 image: rdmz-harbor.rdmz.labs.mlnx/ucx/aarch64/centos8-mofed5-cuda11:3 - + options: $(DOCKER_OPT_VOLUMES) + stages: - stage: Prepare jobs: @@ -37,6 +35,7 @@ stages: set -x check_release_build $(Build.Reason) $(Build.SourceVersion) "AZP/SNAPSHOT: " name: Result + - stage: Build dependsOn: Prepare condition: eq(dependencies.Prepare.outputs['Check.Result.Launch'], 'True') @@ -44,18 +43,13 @@ stages: - template: jucx/jucx-build.yml parameters: arch: amd64 - container: centos7_cuda11_x86_64 + container: centos8_cuda11_x86_64 demands: ucx_docker + target: publish-snapshot - template: jucx/jucx-build.yml parameters: arch: aarch64 container: centos8_cuda11_aarch64 demands: ucx-arm64 - - - template: jucx/jucx-publish.yml - parameters: - ${{ if eq(variables['Build.Reason'], 'IndividualCI') }}: - target: publish-snapshot - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - target: package + target: publish-snapshot diff --git a/buildlib/dockers/centos-release.Dockerfile b/buildlib/dockers/centos-release.Dockerfile index 8f4039ee893..ab806e199f3 100644 --- a/buildlib/dockers/centos-release.Dockerfile +++ b/buildlib/dockers/centos-release.Dockerfile @@ -5,27 +5,28 @@ FROM nvidia/cuda:${CUDA_VERSION}-devel-centos${OS_VERSION} RUN yum install -y \ autoconf \ automake \ + environment-modules \ + ethtool \ file \ + fuse-libs \ gcc-c++ \ git \ glibc-devel \ libtool \ + libusbx \ + lsof \ make \ maven \ numactl-devel \ + pinentry \ + python36 \ rdma-core-devel \ rpm-build \ tcl \ tcsh \ tk \ - wget \ - libusbx \ - fuse-libs \ - python36 \ - lsof \ - ethtool \ - environment-modules \ valgrind-devel \ + wget \ && yum clean all # MOFED diff --git a/buildlib/jucx/jucx-build.yml b/buildlib/jucx/jucx-build.yml index 7d5f6abfee8..a5e113d6d15 100644 --- a/buildlib/jucx/jucx-build.yml +++ b/buildlib/jucx/jucx-build.yml @@ -2,10 +2,16 @@ parameters: arch: container: demands: [] + target: jobs: - job: jucx_build_${{ parameters.arch }} displayName: JUCX build ${{ parameters.arch }} + variables: + temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/tmp-settings.xml + gpg_dir: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/gpg + ${{ if ne(parameters.arch, 'amd64') }}: + SUFFIX: "-${{ parameters.arch }}" # we need to use lowest version for compatibility container: ${{ parameters.container }} @@ -31,10 +37,72 @@ jobs: - bash: | set -eEx make -s -j`nproc` - displayName: Build ucx + source buildlib/az-helpers.sh + az_init_modules + az_module_load dev/mvn + # use the lowest supported Java version for compatibility: + az_module_load dev/jdk-1.8 + git fetch --unshallow + TAG=`git describe --tags` + # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,... + # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string + MAVEN_VERSION=${TAG:1} + make -C bindings/java/src/main/native/ package JUCX_VERSION=${MAVEN_VERSION} + displayName: Build JUCX - - task: PublishBuildArtifacts@1 + - bash: | + set -eE + { + echo -e "" + echo -e "ossrh\${env.SONATYPE_USERNAME}" + echo -e "\${env.SONATYPE_PASSWORD}" + echo -e "" + } > $(temp_cfg) + displayName: Generate temporary config + + - task: DownloadSecureFile@1 + displayName: Download Secure file + inputs: + secureFile: sparkucx-secret.gpg + name: privateKey + + - task: DownloadSecureFile@1 + displayName: Download Secure file inputs: - pathToPublish: '$(Build.Repository.LocalPath)/bindings/java/resources' - artifactName: libjucx_${{ parameters.arch }}.so - displayName: Interim artifact - push + secureFile: sparkucx-public.gpg + name: publicKey + + - bash: | + set -exE + source buildlib/az-helpers.sh + az_init_modules + az_module_load dev/mvn + # use the lowest supported Java version for compatibility: + az_module_load dev/jdk-1.8 + mvn --version + mkdir -p $(gpg_dir) + export GPG_TTY=`tty` + chmod 700 $(gpg_dir) + cp $(publicKey.secureFilePath) $(gpg_dir)/pubring.gpg + cp $(privateKey.secureFilePath) $(gpg_dir)/secring.gpg + export GNUPGHOME=$(gpg_dir) + + # GPG agent config + echo "use-agent" > $GNUPGHOME/gpg.conf + echo "pinentry-mode loopback" >> $GNUPGHOME/gpg.conf + echo "allow-loopback-pinentry" > $GNUPGHOME/gpg-agent.conf + gpg-connect-agent reloadagent /bye || true + + TAG=`git describe --tags` + # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,... + # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string + MAVEN_VERSION=${TAG:1} + make -C bindings/java/src/main/native/ ${{ parameters.target }} \ + ARGS="--settings $(temp_cfg)" \ + JUCX_VERSION=${MAVEN_VERSION}${SUFFIX} + displayName: Publish-Maven + condition: eq(variables['Build.Reason'], 'IndividualCI') + env: + GPG_PASSPHRASE: $(GPG_PASSPHRASE) + SONATYPE_PASSWORD: $(SONATYPE_PASSWORD) + SONATYPE_USERNAME: $(SONATYPE_USERNAME) diff --git a/buildlib/jucx/jucx-publish.yml b/buildlib/jucx/jucx-publish.yml deleted file mode 100644 index 08ae11849cd..00000000000 --- a/buildlib/jucx/jucx-publish.yml +++ /dev/null @@ -1,105 +0,0 @@ -parameters: - temp_cfg: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/tmp-settings.xml - gpg_dir: $(System.DefaultWorkingDirectory)/bindings/java/src/main/native/build-java/gpg - target: - -jobs: - - job: jucx_publish - displayName: JUCX publish - dependsOn: - - jucx_build_amd64 - - jucx_build_aarch64 - condition: succeeded() - - pool: - name: MLNX - demands: ucx_docker - - # we need to use lowest version for compatible - container: centos7_cuda11_x86_64 - - steps: - - checkout: self - clean: true - fetchDepth: 300 - retryCountOnTaskFailure: 5 - path: "we/need/to/go/deeper" - # ^workaround agent issue with container in root path - - - task: DownloadBuildArtifacts@0 - displayName: Interim artifact - get - inputs: - downloadType: 'specific' - itemPattern: 'libjucx_*.so/libjucx_*.so' - downloadPath: '$(System.DefaultWorkingDirectory)' - - - bash: | - set -eEx - gcc --version - ./autogen.sh - ./contrib/configure-release --with-java - make -s -j`nproc` - displayName: Build ucx - - - bash: | - set -exE - source buildlib/az-helpers.sh - az_init_modules - az_module_load dev/mvn - # use the lowest supported Java version for compatibility: - az_module_load dev/jdk-1.8 - TAG=`git describe --tags` - # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,... - # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string - MAVEN_VERSION=${TAG:1} - make -C bindings/java/src/main/native/ multi-arch JUCX_VERSION=${MAVEN_VERSION} - displayName: Build multi-arch JUCX without publishing - condition: eq(variables['Build.Reason'], 'PullRequest') - - - bash: | - set -eE - { - echo -e "" - echo -e "ossrh\${env.SONATYPE_USERNAME}" - echo -e "\${env.SONATYPE_PASSWORD}" - echo -e "" - } > ${{ parameters.temp_cfg }} - displayName: Generate temporary config - - - task: DownloadSecureFile@1 - displayName: Download Secure file - inputs: - secureFile: sparkucx-secret.gpg - name: privateKey - - - task: DownloadSecureFile@1 - displayName: Download Secure file - inputs: - secureFile: sparkucx-public.gpg - name: publicKey - - - bash: | - set -exE - source buildlib/az-helpers.sh - az_init_modules - az_module_load dev/mvn - # use the lowest supported Java version for compatibility: - az_module_load dev/jdk-1.8 - mvn --version - mkdir -p ${{ parameters.gpg_dir }} - export GPG_TTY=`tty` - chmod 700 ${{ parameters.gpg_dir }} - cp $(publicKey.secureFilePath) ${{ parameters.gpg_dir }}/pubring.gpg - cp $(privateKey.secureFilePath) ${{ parameters.gpg_dir }}/secring.gpg - export GNUPGHOME=${{ parameters.gpg_dir }} - TAG=`git describe --tags` - # Maven requires version to be of form MAJOR_VERSION.MINOR_VERSIOn,... - # ucx tags are of form v1.x.x - need to remove 'v' from the beginning of string - MAVEN_VERSION=${TAG:1} - make -C bindings/java/src/main/native/ ${{ parameters.target }} \ - ARGS="--settings ${{ parameters.temp_cfg }}" JUCX_VERSION=${MAVEN_VERSION} - displayName: Publish JUCX jar to maven central - env: - GPG_PASSPHRASE: $(GPG_PASSPHRASE) - SONATYPE_PASSWORD: $(SONATYPE_PASSWORD) - SONATYPE_USERNAME: $(SONATYPE_USERNAME) diff --git a/buildlib/jucx/jucx-test.yml b/buildlib/jucx/jucx-test.yml index 9b771c755f0..e5ba1e0541f 100755 --- a/buildlib/jucx/jucx-test.yml +++ b/buildlib/jucx/jucx-test.yml @@ -1,9 +1,11 @@ parameters: + arch: name: java test demands: [] jobs: - - job: ${{ parameters.name }} + - job: jucx_test_${{ parameters.arch }} + displayName: JUCX Test ${{ parameters.arch }} workspace: clean: outputs @@ -39,6 +41,21 @@ jobs: make install displayName: Build UCX + - bash: | + set -exE + source buildlib/az-helpers.sh + az_init_modules + az_module_load dev/mvn + az_module_load dev/jdk-1.8 + git fetch --unshallow + TAG=`git describe --tags` + MAVEN_VERSION=${TAG:1} + make -C bindings/java/src/main/native/ publish-local \ + JUCX_VERSION=${MAVEN_VERSION}${SUFFIX} + displayName: Publish-Local + condition: eq(variables['Build.Reason'], 'PullRequest') + + - bash: | set -xeE source buildlib/az-helpers.sh diff --git a/buildlib/pr/main.yml b/buildlib/pr/main.yml index e31b54f61cf..4232bc3172b 100644 --- a/buildlib/pr/main.yml +++ b/buildlib/pr/main.yml @@ -356,12 +356,9 @@ stages: jobs: - template: ../jucx/jucx-test.yml parameters: - name: new - demands: ucx_new -equals yes - - template: ../jucx/jucx-test.yml - parameters: + arch: amd64 name: gpu - demands: ucx_gpu -equals yes + demands: ucx_gpu - stage: go dependsOn: [Static_check]