UCT/UD: Don't wake up remote peer for every ACKREQ packet

Introduce a minimal interval for sending packets with SOLICITED flag, to avoid excessive interrupts on the remote peer which have a significant performance overhead.
openucx · Jan 27, 2021 · 8599b3f · 8599b3f
1 parent 99f60ee
commit 8599b3f
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 1 deletion.
diff --git a/src/uct/ib/ud/base/ud_ep.c b/src/uct/ib/ud/base/ud_ep.c
@@ -1308,7 +1308,9 @@ static void uct_ud_ep_send_ack(uct_ud_iface_t *iface, uct_ud_ep_t *ep)
     skb->neth->packet_type = ep->dest_ep_id;
     if (uct_ud_ep_ctl_op_check(ep, UCT_UD_EP_OP_ACK_REQ)) {
         skb->neth->packet_type |= UCT_UD_PACKET_FLAG_ACK_REQ;
-        ctl_flags              |= UCT_UD_IFACE_SEND_CTL_FLAG_SOLICITED;
+        if (ep->tx.tick >= iface->config.min_poke_time) {
+            ctl_flags |= UCT_UD_IFACE_SEND_CTL_FLAG_SOLICITED;
+        }
     }
 
     if (uct_ud_ep_ctl_op_check(ep, UCT_UD_EP_OP_NACK)) {

diff --git a/src/uct/ib/ud/base/ud_iface.c b/src/uct/ib/ud/base/ud_iface.c
@@ -460,6 +460,7 @@ UCS_CLASS_INIT_FUNC(uct_ud_iface_t, uct_ud_iface_ops_t *ops, uct_md_h md,
     self->rx.quota               = 0;
     self->config.tx_qp_len       = config->super.tx.queue_len;
     self->config.peer_timeout    = ucs_time_from_sec(config->peer_timeout);
+    self->config.min_poke_time   = ucs_time_from_sec(config->min_poke_time);
     self->config.check_grh_dgid  = config->dgid_check &&
                                    uct_ib_iface_is_roce(&self->super);
 
@@ -623,15 +624,26 @@ ucs_config_field_t uct_ud_iface_config_table[] = {
 
     {"TIMEOUT", "5.0m", "Transport timeout",
      ucs_offsetof(uct_ud_iface_config_t, peer_timeout), UCS_CONFIG_TYPE_TIME},
+
     {"TIMER_TICK", "10ms", "Initial timeout for retransmissions",
      ucs_offsetof(uct_ud_iface_config_t, timer_tick), UCS_CONFIG_TYPE_TIME},
+
     {"TIMER_BACKOFF", "2.0",
      "Timeout multiplier for resending trigger (must be >= "
      UCS_PP_MAKE_STRING(UCT_UD_MIN_TIMER_TIMER_BACKOFF) ")",
      ucs_offsetof(uct_ud_iface_config_t, timer_backoff),
                   UCS_CONFIG_TYPE_DOUBLE},
+
     {"ASYNC_TIMER_TICK", "100ms", "Resolution for async timer",
      ucs_offsetof(uct_ud_iface_config_t, event_timer_tick), UCS_CONFIG_TYPE_TIME},
+
+    {"MIN_POKE_TIME", "250ms",
+     "Minimal internal to send ACK request with solicited flag, to wake up\n"
+     "the remote peer in case it is not actively calling progress.\n"
+     "Smaller values may incur performance overhead, while extermely large\n"
+     "values can cause delays in presence of packet drops.",
+     ucs_offsetof(uct_ud_iface_config_t, min_poke_time), UCS_CONFIG_TYPE_TIME},
+
     {"ETH_DGID_CHECK", "y",
      "Enable checking destination GID for incoming packets of Ethernet network.\n"
      "Mismatched packets are silently dropped.",

diff --git a/src/uct/ib/ud/base/ud_iface.h b/src/uct/ib/ud/base/ud_iface.h
@@ -53,6 +53,7 @@ typedef struct uct_ud_iface_config {
     uct_ib_iface_config_t         super;
     uct_ud_iface_common_config_t  ud_common;
     double                        peer_timeout;
+    double                        min_poke_time;
     double                        timer_tick;
     double                        timer_backoff;
     double                        event_timer_tick;
@@ -179,6 +180,7 @@ struct uct_ud_iface {
     } tx;
     struct {
         ucs_time_t           peer_timeout;
+        ucs_time_t           min_poke_time;
         unsigned             tx_qp_len;
         unsigned             max_inline;
         int                  check_grh_dgid;