UCT/MM: Add user namespace handling for posix and cma transports

buildlib/pr/main.yml

@@ -316,6 +316,14 @@ stages:
         demands: ucx_bf -equals yes
         test_perf: 0
 
+  - stage: Namespace_Tests
+    dependsOn: [Static_check]
+    jobs:
+    - template: namespace_tests.yml
+      parameters:
+        name: new_namespace
+        demands: ucx_new -equals yes
+
   - stage: io_demo
     dependsOn: [Static_check]
     jobs:

buildlib/pr/namespace_tests.yml

@@ -0,0 +1,27 @@
+jobs:
+  - job: tests_${{ parameters.name }}
+    pool:
+      name: MLNX
+      demands: ${{ parameters.demands }}
+    displayName: ${{ parameters.name }} on worker
+    timeoutInMinutes: 30
+    workspace:
+      clean: all
+    steps:
+      - checkout: self
+        clean: true
+        fetchDepth: 100
+
+      - bash: |
+          source buildlib/az-helpers.sh
+          if ! unshare --user whoami; then
+              msg="Missing user namespace support on $(hostname)"
+              azure_log_error "$msg"
+              azure_complete_with_issues "$msg"
+          else
+              ./contrib/test_namespace.sh
+          fi
+        displayName: Run namespace test
+        env:
+          BUILD_NUMBER: "$(Build.BuildId)-$(Build.BuildNumber)"
+          JOB_URL: "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"

buildlib/tools/common.sh

@@ -30,6 +30,52 @@ MAKE="make V=1"
 MAKEP="make V=1 -j${parallel_jobs}"
 export AUTOMAKE_JOBS=$parallel_jobs
 
+#
+# Prepare build environment
+#
+prepare() {
+	echo " ==== Prepare ===="
+	env
+	cd ${WORKSPACE}
+	if [ -d build-test ]
+	then
+		chmod u+rwx build-test -R
+		rm -rf build-test
+	fi
+	./autogen.sh
+	mkdir -p build-test
+	cd build-test
+}
+
+#
+# Configure and build
+#   $1 - mode (devel|release)
+#
+build() {
+	mode=$1
+	shift
+
+	config_args="--prefix=$ucx_inst --without-java"
+	if [ "X$have_cuda" == "Xyes" ]
+	then
+		config_args+=" --with-iodemo-cuda"
+	fi
+
+	../contrib/configure-${mode} ${config_args} "$@"
+	make_clean
+	$MAKEP
+	$MAKEP install
+}
+
+#
+# Update global server port
+#
+step_server_port() {
+	# Cycle server_port between (server_port_min)..(server_port_max-1)
+	server_port=$((server_port + 1))
+	server_port=$((server_port >= server_port_max ? server_port_min : server_port))
+}
+
 #
 # cleanup ucx
 #
@@ -42,7 +88,7 @@ make_clean() {
 # Prepare build environment
 #
 prepare_build() {
-	echo " ==== Prepare ===="
+	echo " ==== Prepare Build ===="
 	env
 	cd ${WORKSPACE}
 	if [ -d ${ucx_build_dir} ]

contrib/test_jenkins.sh

@@ -24,6 +24,8 @@
 #  - COV_OPT  : command line options for Coverity static checker
 #
 
+source $(dirname $0)/../buildlib/tools/common.sh
+
 WORKSPACE=${WORKSPACE:=$PWD}
 ucx_inst=${WORKSPACE}/install
 CUDA_MODULE="dev/cuda11.4"
@@ -110,34 +112,6 @@ log_error() {
 	test "x$RUNNING_IN_AZURE" = "xyes" && { azure_log_error "${msg}" ; set -x; } || echo "${msg}"
 }
 
-#
-# cleanup ucx
-#
-make_clean() {
-	rm -rf ${ucx_inst}
-	$MAKEP ${1:-clean}
-}
-
-#
-# Configure and build
-#   $1 - mode (devel|release)
-#
-build() {
-	mode=$1
-	shift
-
-	config_args="--prefix=$ucx_inst --without-java"
-	if [ "X$have_cuda" == "Xyes" ]
-	then
-		config_args+=" --with-iodemo-cuda"
-	fi
-
-	../contrib/configure-${mode} ${config_args} "$@"
-	make_clean
-	$MAKEP
-	$MAKEP install
-}
-
 #
 # Test if an environment module exists and load it if yes.
 # Otherwise, return error code.
@@ -358,23 +332,6 @@ get_non_rdma_ip_addr() {
 	get_ifaddr ${eth_iface}
 }
 
-#
-# Prepare build environment
-#
-prepare() {
-	echo " ==== Prepare ===="
-	env
-	cd ${WORKSPACE}
-	if [ -d build-test ]
-	then
-		chmod u+rwx build-test -R
-		rm -rf build-test
-	fi
-	./autogen.sh
-	mkdir -p build-test
-	cd build-test
-}
-
 #
 # Expands a CPU list such as "0-3,17" to "0 1 2 3 17" (each cpu in a new line)
 #
@@ -421,12 +378,6 @@ run_loopback_app() {
 	wait ${pid} || true
 }
 
-step_server_port() {
-	# Cycle server_port between (server_port_min)..(server_port_max-1)
-	server_port=$((server_port + 1))
-	server_port=$((server_port >= server_port_max ? server_port_min : server_port))
-}
-
 run_client_server_app() {
 	test_exe=$1
 	test_args=$2

contrib/test_namespace.sh

@@ -0,0 +1,69 @@
+#!/bin/bash -eEx
+#
+# Testing script for UCX namespace related functionality
+#
+# Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2023. ALL RIGHTS RESERVED.
+#
+# See file LICENSE for terms.
+#
+#
+# Environment variables set by Azure CI:
+#  - WORKSPACE           : path to work dir
+#
+
+source $(dirname $0)/../buildlib/tools/common.sh
+
+ucx_inst=${WORKSPACE}/install
+
+echo "==== Running namespace tests on $(hostname) ===="
+
+server_port_range=1000
+server_port_min=10500
+server_port_max=$((server_port_min + server_port_range))
+server_port=${server_port_min}
+
+test_namespace() {
+	# Make sure to try to use CMA when possible
+	# Expect fallback on SYSV
+	perftest="$ucx_inst/bin/ucx_perftest -t ucp_get -s 9999999 -n 5"
+
+	echo "==== Running perftest namespace positive tests ===="
+
+	for tls in posix cma,sysv
+	do
+		echo "==== Running perftest same non-default USER namespace test for $tls ===="
+
+		cmd="UCX_TLS=$tls $perftest -p $server_port"
+		step_server_port
+		unshare --user bash -c "{ $cmd & sleep 3; $cmd localhost; }"
+	done
+
+	for tl in posix cma
+	do
+		echo "==== Running perftest different PID namespace test for $tl ===="
+
+		cmd="$perftest -p $server_port"
+		step_server_port
+		sudo unshare --pid --fork sudo -u $USER UCX_TLS=$tl,sysv $cmd &
+		sleep 3
+		sudo unshare --pid --fork sudo -u $USER UCX_TLS=$tl,sysv $cmd localhost
+
+		echo "==== Running perftest different USER namespace test for $tl ===="
+		cmd="$perftest -p $server_port"
+		step_server_port
+		UCX_TLS=$tl,sysv unshare --user $cmd &
+		sleep 3
+		UCX_TLS=$tl,sysv unshare --user $cmd localhost
+	done
+
+	echo "==== Running perftest different USER namespace test for posix non proc link ===="
+	cmd="$perftest -p $server_port"
+	step_server_port
+	UCX_TLS="posix" UCX_POSIX_USE_PROC_LINK=n unshare --user $cmd &
+	sleep 3
+	UCX_TLS="posix" UCX_POSIX_USE_PROC_LINK=n unshare --user $cmd localhost
+}
+
+prepare
+build release
+test_namespace

src/uct/sm/mm/posix/mm_posix.c

@@ -136,7 +136,7 @@ uct_posix_md_query(uct_md_h tl_md, uct_md_attr_v2_t *md_attr)
 
     if (statvfs(posix_config->dir, &shm_statvfs) < 0) {
         ucs_error("could not stat shared memory device %s (%m)",
-                  UCT_POSIX_SHM_OPEN_DIR);
+                  posix_config->dir);
         return UCS_ERR_NO_DEVICE;
     }
 
@@ -363,26 +363,36 @@ static ucs_status_t uct_posix_munmap(void *address, size_t length)
 }
 
 static ucs_status_t
-uct_posix_mem_attach_common(uct_mm_seg_id_t seg_id, size_t length,
-                            const char *dir, uct_mm_remote_seg_t *rseg)
+uct_posix_mem_open(uct_mm_seg_id_t seg_id, const char *dir, int *fd_p)
 {
     uint64_t mmid = seg_id & UCT_POSIX_SEG_MMID_MASK;
-    int pid, peer_fd, fd;
     ucs_status_t status;
-    int mmap_flags;
-
-    ucs_assert(length > 0);
-    rseg->cookie = (void*)length;
+    int pid, peer_fd;
 
     if (seg_id & UCT_POSIX_SEG_FLAG_PROCFS) {
         uct_posix_mmid_procfs_unpack(mmid, &pid, &peer_fd);
-        status = uct_posix_procfs_open(pid, peer_fd, &fd);
+        status = uct_posix_procfs_open(pid, peer_fd, fd_p);
     } else if (seg_id & UCT_POSIX_SEG_FLAG_SHM_OPEN) {
-        status = uct_posix_shm_open(mmid, 0, &fd);
+        status = uct_posix_shm_open(mmid, 0, fd_p);
     } else {
         ucs_assert(dir != NULL); /* for coverity */
-        status = uct_posix_file_open(dir, mmid, 0, &fd);
+        status = uct_posix_file_open(dir, mmid, 0, fd_p);
     }
+
+    return status;
+}
+
+static ucs_status_t
+uct_posix_mem_attach_common(uct_mm_seg_id_t seg_id, size_t length,
+                            const char *dir, uct_mm_remote_seg_t *rseg)
+{
+    ucs_status_t status;
+    int mmap_flags, fd;
+
+    ucs_assert(length > 0);
+    rseg->cookie = (void*)length;
+
+    status = uct_posix_mem_open(seg_id, dir, &fd);
     if (status != UCS_OK) {
         return status;
     }
@@ -403,11 +413,25 @@ static int
 uct_posix_is_reachable(uct_mm_md_t *md, uct_mm_seg_id_t seg_id,
                        const void *iface_addr)
 {
+    int fd;
+    ucs_status_t status;
+
     if (seg_id & UCT_POSIX_SEG_FLAG_PID_NS) {
-        return ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID) == *(const ucs_sys_ns_t*)iface_addr;
+        if (ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID) !=
+            *(const ucs_sys_ns_t*)iface_addr) {
+            return 0;
+        }
+    } else if (!ucs_sys_ns_is_default(UCS_SYS_NS_TYPE_PID)) {
+        return 0;
+    }
+
+    status = uct_posix_mem_open(seg_id, (const char*)iface_addr, &fd);
+    if (status != UCS_OK) {
+        return 0;
     }
 
-    return ucs_sys_ns_is_default(UCS_SYS_NS_TYPE_PID);
+    close(fd);
+    return 1;
 }
 
 static ucs_status_t uct_posix_mem_detach_common(const uct_mm_remote_seg_t *rseg)

src/uct/sm/scopy/cma/cma_iface.c

@@ -72,7 +72,12 @@ static int
 uct_cma_iface_is_reachable_v2(const uct_iface_h tl_iface,
                               const uct_iface_is_reachable_params_t *params)
 {
+    struct iovec iov = {
+        .iov_base = &iov,
+        .iov_len  = sizeof(iov),
+    };
     ucs_cma_iface_ext_device_addr_t *iface_addr;
+    pid_t peer_pid;
 
     if (!uct_iface_is_reachable_params_addrs_valid(params)) {
         return 0;
@@ -91,6 +96,15 @@ uct_cma_iface_is_reachable_v2(const uct_iface_h tl_iface,
         return 0;
     }
 
+    /* Confirm reachability by actually trying to read from the remote, as
+     * permissions are enforced before the remote address sanity checks.
+     */
+    peer_pid = iface_addr->super.id & ~UCT_CMA_IFACE_ADDR_FLAG_PID_NS;
+    if ((process_vm_readv(peer_pid, &iov, 1, &iov, 1, 0) == -1) &&
+        (errno == EPERM)) {
+        return 0;
+    }
+
     return uct_iface_scope_is_reachable(tl_iface, params);
 }