Merge pull request openucx#13 from xinzhao3/topic/cuda-md

Reduce size of domain-specific config array to actual number of addre…
xinzhao3 · Sep 19, 2017 · 6783916 · 6783916
2 parents 8c2ac85 + f99b592
commit 6783916
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 9 deletions.
diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c
@@ -859,6 +859,7 @@ void ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config)
     size_t it;
     size_t max_rndv_thresh;
     size_t max_am_rndv_thresh;
+    size_t dn_num;
 
     /* Default settings */
     for (it = 0; it < UCP_MAX_IOV; ++it) {
@@ -957,15 +958,21 @@ void ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config)
     }
 
     /* Configuration for memory domains */
-    config->dn = ucs_malloc(sizeof(ucp_ep_dn_config_t) * context->num_mds, "domain-specific configuration");
+    dn_num = 0;
     for (it = 0; it < context->num_mds; it++) {
         if (context->tl_mds[it].attr.cap.flags & UCT_MD_FLAG_ADDR_DN) {
-            config->dn[it].tag.eager.max_short = context->tl_mds[it].attr.cap.eager.max_short;
-            memset(config->dn[it].tag.eager.zcopy_thresh, 0, UCP_MAX_IOV * sizeof(size_t));
-        } else {
-            config->dn[it].tag.eager.max_short = config->tag.eager.max_short;
-            memcpy(config->dn[it].tag.eager.zcopy_thresh, config->tag.eager.zcopy_thresh,
-                   UCP_MAX_IOV * sizeof(size_t));
+            dn_num++;
+        }
+    }
+
+    config->dn = ucs_malloc(sizeof(ucp_ep_dn_config_t) * dn_num, "domain-specific configuration");
+
+    dn_num = 0;
+    for (it = 0; it < context->num_mds; it++) {
+        if (context->tl_mds[it].attr.cap.flags & UCT_MD_FLAG_ADDR_DN) {
+            config->dn[dn_num].tag.eager.max_short = context->tl_mds[it].attr.cap.eager.max_short;
+            memset(config->dn[dn_num].tag.eager.zcopy_thresh, 0, UCP_MAX_IOV * sizeof(size_t));
+            dn_num++;
         }
     }
 

diff --git a/src/ucp/tag/tag_send.c b/src/ucp/tag/tag_send.c
@@ -280,8 +280,8 @@ UCS_PROFILE_FUNC(ucs_status_ptr_t, ucp_tag_send_nb,
     }
 
     ret = ucp_tag_send_req(req, count,
-                           ucp_ep_config(ep)->dn[md_index].tag.eager.max_short,
-                           ucp_ep_config(ep)->dn[md_index].tag.eager.zcopy_thresh,
+                           addr_dn.mask ? ucp_ep_config(ep)->dn[md_index].tag.eager.max_short : ucp_ep_config(ep)->tag.eager.max_short,
+                           addr_dn.mask ? ucp_ep_config(ep)->dn[md_index].tag.eager.zcopy_thresh : ucp_ep_config(ep)->tag.eager.zcopy_thresh,
                            ucp_ep_config(ep)->tag.rndv.rma_thresh,
                            ucp_ep_config(ep)->tag.rndv.am_thresh,
                            cb, ucp_ep_config(ep)->tag.proto);