UCT/ROCM: don't try to lock host pinned memory

newer ROCm releases don't like locking host pinned memory (e.g. hipHostMalloc). avoid the lock operation in the rocm_copy path.
openucx · May 15, 2023 · 745df30 · 745df30
1 parent f82d4cb
commit 745df30
Showing 1 changed file with 14 additions and 8 deletions.
diff --git a/src/uct/rocm/copy/rocm_copy_ep.c b/src/uct/rocm/copy/rocm_copy_ep.c
@@ -108,11 +108,14 @@ ucs_status_t uct_rocm_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr,
         return UCS_ERR_IO_ERROR;
     }
 
-    if ((remote_addr_mem_type == HSA_EXT_POINTER_TYPE_HSA) &&
-        (dev_type == HSA_DEVICE_TYPE_GPU)) {
-        /* UCS_MEMORY_TYPE_ROCM */
+    if (remote_addr_mem_type == HSA_EXT_POINTER_TYPE_HSA) {
         remote_addr_mod = (void*)remote_addr;
-        agent           = tmp_agent;
+        if (dev_type == HSA_DEVICE_TYPE_GPU) {
+            /* UCS_MEMORY_TYPE_ROCM */
+            agent = tmp_agent;
+        } else {
+            remote_addr_is_host = 1;
+        }
     } else if ((remote_addr_mem_type == HSA_EXT_POINTER_TYPE_LOCKED) &&
                (size == dev_size)) {
         /* locked host memory, e.g. hipHostRegister, OR previously registered */
@@ -141,11 +144,14 @@ ucs_status_t uct_rocm_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr,
         return UCS_ERR_IO_ERROR;
     }
 
-    if ((iov_buffer_mem_type == HSA_EXT_POINTER_TYPE_HSA) &&
-        (dev_type == HSA_DEVICE_TYPE_GPU)) {
-        /* UCS_MEMORY_TYPE_ROCM */
+    if (iov_buffer_mem_type == HSA_EXT_POINTER_TYPE_HSA) {
         iov_buffer_mod = iov->buffer;
-        agent          = tmp_agent;
+        if (dev_type == HSA_DEVICE_TYPE_GPU) {
+            /* UCS_MEMORY_TYPE_ROCM */
+            agent = tmp_agent;
+        } else {
+            iov_buffer_is_host = 1;
+        }
     } else if ((iov_buffer_mem_type == HSA_EXT_POINTER_TYPE_LOCKED) &&
                (size == dev_size)) {
         /* locked host memory (e.g. hipHostRegister) OR previously registered */