UCP/EP: Add eager multi-fragment overhead

openucx · Feb 20, 2022 · e7b8cfc · e7b8cfc
1 parent 821b793
commit e7b8cfc
Showing 1 changed file with 14 additions and 5 deletions.
diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c
@@ -1714,7 +1714,8 @@ int ucp_ep_config_is_equal(const ucp_ep_config_key_t *key1,
 static void ucp_ep_config_calc_params(ucp_worker_h worker,
                                       const ucp_ep_config_t *config,
                                       const ucp_lane_index_t *lanes,
-                                      ucp_ep_thresh_params_t *params)
+                                      ucp_ep_thresh_params_t *params,
+                                      int eager)
 {
     ucp_context_h context = worker->context;
     ucp_md_map_t md_map   = 0;
@@ -1723,6 +1724,7 @@ static void ucp_ep_config_calc_params(ucp_worker_h worker,
     ucp_md_index_t md_index;
     uct_md_attr_t *md_attr;
     uct_iface_attr_t *iface_attr;
+    double bw;
     int i;
 
     memset(params, 0, sizeof(*params));
@@ -1749,7 +1751,14 @@ static void ucp_ep_config_calc_params(ucp_worker_h worker,
             }
         }
 
-        params->bw += ucp_tl_iface_bandwidth(context, &iface_attr->bandwidth);
+        bw = ucp_tl_iface_bandwidth(context, &iface_attr->bandwidth);
+
+        if (eager) {
+            params->bw += 1.0 / ((1.0 / bw) + (iface_attr->overhead /
+                                               iface_attr->cap.am.max_bcopy));
+        } else {
+            params->bw += bw;
+        }
     }
 }
 
@@ -1771,8 +1780,8 @@ static size_t ucp_ep_config_calc_rndv_thresh(ucp_worker_t *worker,
     /* All formulas and descriptions are listed at
      * https://github.com/openucx/ucx/wiki/Rendezvous-Protocol-threshold-for-multilane-mode */
 
-    ucp_ep_config_calc_params(worker, config, eager_lanes, &eager_zcopy);
-    ucp_ep_config_calc_params(worker, config, rndv_lanes, &rndv);
+    ucp_ep_config_calc_params(worker, config, eager_lanes, &eager_zcopy, 1);
+    ucp_ep_config_calc_params(worker, config, rndv_lanes, &rndv, 0);
 
     if ((eager_zcopy.bw == 0) || (rndv.bw == 0)) {
         goto fallback;
@@ -1826,7 +1835,7 @@ static size_t ucp_ep_config_calc_rma_zcopy_thresh(ucp_worker_t *worker,
     double numerator, denumerator;
     double reg_overhead, reg_growth;
 
-    ucp_ep_config_calc_params(worker, config, rma_lanes, &rma);
+    ucp_ep_config_calc_params(worker, config, rma_lanes, &rma, 0);
 
     if (rma.bw == 0) {
         goto fallback;