diff --git a/buildlib/pr/main.yml b/buildlib/pr/main.yml index bdb7c6cf753..7360a5adec4 100644 --- a/buildlib/pr/main.yml +++ b/buildlib/pr/main.yml @@ -316,6 +316,14 @@ stages: demands: ucx_bf -equals yes test_perf: 0 + - stage: Namespace_Tests + dependsOn: [Static_check] + jobs: + - template: namespace_tests.yml + parameters: + name: new_namespace + demands: ucx_new -equals yes + - stage: io_demo dependsOn: [Static_check] jobs: diff --git a/buildlib/pr/namespace_tests.yml b/buildlib/pr/namespace_tests.yml new file mode 100644 index 00000000000..269eb1b79a0 --- /dev/null +++ b/buildlib/pr/namespace_tests.yml @@ -0,0 +1,27 @@ +jobs: + - job: tests_${{ parameters.name }} + pool: + name: MLNX + demands: ${{ parameters.demands }} + displayName: ${{ parameters.name }} on worker + timeoutInMinutes: 30 + workspace: + clean: all + steps: + - checkout: self + clean: true + fetchDepth: 100 + + - bash: | + source buildlib/az-helpers.sh + if ! unshare --user whoami; then + msg="Missing user namespace support on $(hostname)" + azure_log_error "$msg" + azure_complete_with_issues "$msg" + else + ./contrib/test_namespace.sh + fi + displayName: Run namespace test + env: + BUILD_NUMBER: "$(Build.BuildId)-$(Build.BuildNumber)" + JOB_URL: "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" diff --git a/buildlib/tools/common.sh b/buildlib/tools/common.sh index 9863a89f374..486bf435d8f 100644 --- a/buildlib/tools/common.sh +++ b/buildlib/tools/common.sh @@ -30,6 +30,52 @@ MAKE="make V=1" MAKEP="make V=1 -j${parallel_jobs}" export AUTOMAKE_JOBS=$parallel_jobs +# +# Prepare build environment +# +prepare() { + echo " ==== Prepare ====" + env + cd ${WORKSPACE} + if [ -d build-test ] + then + chmod u+rwx build-test -R + rm -rf build-test + fi + ./autogen.sh + mkdir -p build-test + cd build-test +} + +# +# Configure and build +# $1 - mode (devel|release) +# +build() { + mode=$1 + shift + + config_args="--prefix=$ucx_inst --without-java" + if [ "X$have_cuda" == "Xyes" ] + then + config_args+=" --with-iodemo-cuda" + fi + + ../contrib/configure-${mode} ${config_args} "$@" + make_clean + $MAKEP + $MAKEP install +} + +# +# Update global server port +# +step_server_port() { + # Cycle server_port between (server_port_min)..(server_port_max-1) + server_port=$((server_port + 1)) + server_port=$((server_port >= server_port_max ? server_port_min : server_port)) +} + # # cleanup ucx # @@ -42,7 +88,7 @@ make_clean() { # Prepare build environment # prepare_build() { - echo " ==== Prepare ====" + echo " ==== Prepare Build ====" env cd ${WORKSPACE} if [ -d ${ucx_build_dir} ] diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 1b95fc8abbc..22c53a6849f 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -24,6 +24,8 @@ # - COV_OPT : command line options for Coverity static checker # +source $(dirname $0)/../buildlib/tools/common.sh + WORKSPACE=${WORKSPACE:=$PWD} ucx_inst=${WORKSPACE}/install CUDA_MODULE="dev/cuda11.4" @@ -110,34 +112,6 @@ log_error() { test "x$RUNNING_IN_AZURE" = "xyes" && { azure_log_error "${msg}" ; set -x; } || echo "${msg}" } -# -# cleanup ucx -# -make_clean() { - rm -rf ${ucx_inst} - $MAKEP ${1:-clean} -} - -# -# Configure and build -# $1 - mode (devel|release) -# -build() { - mode=$1 - shift - - config_args="--prefix=$ucx_inst --without-java" - if [ "X$have_cuda" == "Xyes" ] - then - config_args+=" --with-iodemo-cuda" - fi - - ../contrib/configure-${mode} ${config_args} "$@" - make_clean - $MAKEP - $MAKEP install -} - # # Test if an environment module exists and load it if yes. # Otherwise, return error code. @@ -358,23 +332,6 @@ get_non_rdma_ip_addr() { get_ifaddr ${eth_iface} } -# -# Prepare build environment -# -prepare() { - echo " ==== Prepare ====" - env - cd ${WORKSPACE} - if [ -d build-test ] - then - chmod u+rwx build-test -R - rm -rf build-test - fi - ./autogen.sh - mkdir -p build-test - cd build-test -} - # # Expands a CPU list such as "0-3,17" to "0 1 2 3 17" (each cpu in a new line) # @@ -421,12 +378,6 @@ run_loopback_app() { wait ${pid} || true } -step_server_port() { - # Cycle server_port between (server_port_min)..(server_port_max-1) - server_port=$((server_port + 1)) - server_port=$((server_port >= server_port_max ? server_port_min : server_port)) -} - run_client_server_app() { test_exe=$1 test_args=$2 diff --git a/contrib/test_namespace.sh b/contrib/test_namespace.sh new file mode 100755 index 00000000000..bdfdac83fb3 --- /dev/null +++ b/contrib/test_namespace.sh @@ -0,0 +1,69 @@ +#!/bin/bash -eEx +# +# Testing script for UCX namespace related functionality +# +# Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2023. ALL RIGHTS RESERVED. +# +# See file LICENSE for terms. +# +# +# Environment variables set by Azure CI: +# - WORKSPACE : path to work dir +# + +source $(dirname $0)/../buildlib/tools/common.sh + +ucx_inst=${WORKSPACE}/install + +echo "==== Running namespace tests on $(hostname) ====" + +server_port_range=1000 +server_port_min=10500 +server_port_max=$((server_port_min + server_port_range)) +server_port=${server_port_min} + +test_namespace() { + # Make sure to try to use CMA when possible + # Expect fallback on SYSV + perftest="$ucx_inst/bin/ucx_perftest -t ucp_get -s 9999999 -n 5" + + echo "==== Running perftest namespace positive tests ====" + + for tls in posix cma,sysv + do + echo "==== Running perftest same non-default USER namespace test for $tls ====" + + cmd="UCX_TLS=$tls $perftest -p $server_port" + step_server_port + unshare --user bash -c "{ $cmd & sleep 3; $cmd localhost; }" + done + + for tl in posix cma + do + echo "==== Running perftest different PID namespace test for $tl ====" + + cmd="$perftest -p $server_port" + step_server_port + sudo unshare --pid --fork sudo -u $USER UCX_TLS=$tl,sysv $cmd & + sleep 3 + sudo unshare --pid --fork sudo -u $USER UCX_TLS=$tl,sysv $cmd localhost + + echo "==== Running perftest different USER namespace test for $tl ====" + cmd="$perftest -p $server_port" + step_server_port + UCX_TLS=$tl,sysv unshare --user $cmd & + sleep 3 + UCX_TLS=$tl,sysv unshare --user $cmd localhost + done + + echo "==== Running perftest different USER namespace test for posix non proc link ====" + cmd="$perftest -p $server_port" + step_server_port + UCX_TLS="posix" UCX_POSIX_USE_PROC_LINK=n unshare --user $cmd & + sleep 3 + UCX_TLS="posix" UCX_POSIX_USE_PROC_LINK=n unshare --user $cmd localhost +} + +prepare +build release +test_namespace diff --git a/src/uct/sm/mm/posix/mm_posix.c b/src/uct/sm/mm/posix/mm_posix.c index 61542c8a9ca..e9e37f0b3ac 100644 --- a/src/uct/sm/mm/posix/mm_posix.c +++ b/src/uct/sm/mm/posix/mm_posix.c @@ -136,7 +136,7 @@ uct_posix_md_query(uct_md_h tl_md, uct_md_attr_v2_t *md_attr) if (statvfs(posix_config->dir, &shm_statvfs) < 0) { ucs_error("could not stat shared memory device %s (%m)", - UCT_POSIX_SHM_OPEN_DIR); + posix_config->dir); return UCS_ERR_NO_DEVICE; } @@ -363,26 +363,36 @@ static ucs_status_t uct_posix_munmap(void *address, size_t length) } static ucs_status_t -uct_posix_mem_attach_common(uct_mm_seg_id_t seg_id, size_t length, - const char *dir, uct_mm_remote_seg_t *rseg) +uct_posix_mem_open(uct_mm_seg_id_t seg_id, const char *dir, int *fd_p) { uint64_t mmid = seg_id & UCT_POSIX_SEG_MMID_MASK; - int pid, peer_fd, fd; ucs_status_t status; - int mmap_flags; - - ucs_assert(length > 0); - rseg->cookie = (void*)length; + int pid, peer_fd; if (seg_id & UCT_POSIX_SEG_FLAG_PROCFS) { uct_posix_mmid_procfs_unpack(mmid, &pid, &peer_fd); - status = uct_posix_procfs_open(pid, peer_fd, &fd); + status = uct_posix_procfs_open(pid, peer_fd, fd_p); } else if (seg_id & UCT_POSIX_SEG_FLAG_SHM_OPEN) { - status = uct_posix_shm_open(mmid, 0, &fd); + status = uct_posix_shm_open(mmid, 0, fd_p); } else { ucs_assert(dir != NULL); /* for coverity */ - status = uct_posix_file_open(dir, mmid, 0, &fd); + status = uct_posix_file_open(dir, mmid, 0, fd_p); } + + return status; +} + +static ucs_status_t +uct_posix_mem_attach_common(uct_mm_seg_id_t seg_id, size_t length, + const char *dir, uct_mm_remote_seg_t *rseg) +{ + ucs_status_t status; + int mmap_flags, fd; + + ucs_assert(length > 0); + rseg->cookie = (void*)length; + + status = uct_posix_mem_open(seg_id, dir, &fd); if (status != UCS_OK) { return status; } @@ -403,11 +413,25 @@ static int uct_posix_is_reachable(uct_mm_md_t *md, uct_mm_seg_id_t seg_id, const void *iface_addr) { + int fd; + ucs_status_t status; + if (seg_id & UCT_POSIX_SEG_FLAG_PID_NS) { - return ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID) == *(const ucs_sys_ns_t*)iface_addr; + if (ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID) != + *(const ucs_sys_ns_t*)iface_addr) { + return 0; + } + } else if (!ucs_sys_ns_is_default(UCS_SYS_NS_TYPE_PID)) { + return 0; + } + + status = uct_posix_mem_open(seg_id, (const char*)iface_addr, &fd); + if (status != UCS_OK) { + return 0; } - return ucs_sys_ns_is_default(UCS_SYS_NS_TYPE_PID); + close(fd); + return 1; } static ucs_status_t uct_posix_mem_detach_common(const uct_mm_remote_seg_t *rseg) diff --git a/src/uct/sm/scopy/cma/cma_iface.c b/src/uct/sm/scopy/cma/cma_iface.c index 7c0190ff61b..9f4eaa7e4a4 100644 --- a/src/uct/sm/scopy/cma/cma_iface.c +++ b/src/uct/sm/scopy/cma/cma_iface.c @@ -72,7 +72,12 @@ static int uct_cma_iface_is_reachable_v2(const uct_iface_h tl_iface, const uct_iface_is_reachable_params_t *params) { + struct iovec iov = { + .iov_base = &iov, + .iov_len = sizeof(iov), + }; ucs_cma_iface_ext_device_addr_t *iface_addr; + pid_t peer_pid; if (!uct_iface_is_reachable_params_addrs_valid(params)) { return 0; @@ -91,6 +96,15 @@ uct_cma_iface_is_reachable_v2(const uct_iface_h tl_iface, return 0; } + /* Confirm reachability by actually trying to read from the remote, as + * permissions are enforced before the remote address sanity checks. + */ + peer_pid = iface_addr->super.id & ~UCT_CMA_IFACE_ADDR_FLAG_PID_NS; + if ((process_vm_readv(peer_pid, &iov, 1, &iov, 1, 0) == -1) && + (errno == EPERM)) { + return 0; + } + return uct_iface_scope_is_reachable(tl_iface, params); }