From 8599b3f0421e5a2c99e806de2208632b2508adf4 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Wed, 27 Jan 2021 09:46:04 -0800 Subject: [PATCH] UCT/UD: Don't wake up remote peer for every ACKREQ packet Introduce a minimal interval for sending packets with SOLICITED flag, to avoid excessive interrupts on the remote peer which have a significant performance overhead. --- src/uct/ib/ud/base/ud_ep.c | 4 +++- src/uct/ib/ud/base/ud_iface.c | 12 ++++++++++++ src/uct/ib/ud/base/ud_iface.h | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/uct/ib/ud/base/ud_ep.c b/src/uct/ib/ud/base/ud_ep.c index 192de1bfb285..780e1ebee507 100644 --- a/src/uct/ib/ud/base/ud_ep.c +++ b/src/uct/ib/ud/base/ud_ep.c @@ -1308,7 +1308,9 @@ static void uct_ud_ep_send_ack(uct_ud_iface_t *iface, uct_ud_ep_t *ep) skb->neth->packet_type = ep->dest_ep_id; if (uct_ud_ep_ctl_op_check(ep, UCT_UD_EP_OP_ACK_REQ)) { skb->neth->packet_type |= UCT_UD_PACKET_FLAG_ACK_REQ; - ctl_flags |= UCT_UD_IFACE_SEND_CTL_FLAG_SOLICITED; + if (ep->tx.tick >= iface->config.min_poke_time) { + ctl_flags |= UCT_UD_IFACE_SEND_CTL_FLAG_SOLICITED; + } } if (uct_ud_ep_ctl_op_check(ep, UCT_UD_EP_OP_NACK)) { diff --git a/src/uct/ib/ud/base/ud_iface.c b/src/uct/ib/ud/base/ud_iface.c index 912a1d082388..69dbfc1b49da 100644 --- a/src/uct/ib/ud/base/ud_iface.c +++ b/src/uct/ib/ud/base/ud_iface.c @@ -460,6 +460,7 @@ UCS_CLASS_INIT_FUNC(uct_ud_iface_t, uct_ud_iface_ops_t *ops, uct_md_h md, self->rx.quota = 0; self->config.tx_qp_len = config->super.tx.queue_len; self->config.peer_timeout = ucs_time_from_sec(config->peer_timeout); + self->config.min_poke_time = ucs_time_from_sec(config->min_poke_time); self->config.check_grh_dgid = config->dgid_check && uct_ib_iface_is_roce(&self->super); @@ -623,15 +624,26 @@ ucs_config_field_t uct_ud_iface_config_table[] = { {"TIMEOUT", "5.0m", "Transport timeout", ucs_offsetof(uct_ud_iface_config_t, peer_timeout), UCS_CONFIG_TYPE_TIME}, + {"TIMER_TICK", "10ms", "Initial timeout for retransmissions", ucs_offsetof(uct_ud_iface_config_t, timer_tick), UCS_CONFIG_TYPE_TIME}, + {"TIMER_BACKOFF", "2.0", "Timeout multiplier for resending trigger (must be >= " UCS_PP_MAKE_STRING(UCT_UD_MIN_TIMER_TIMER_BACKOFF) ")", ucs_offsetof(uct_ud_iface_config_t, timer_backoff), UCS_CONFIG_TYPE_DOUBLE}, + {"ASYNC_TIMER_TICK", "100ms", "Resolution for async timer", ucs_offsetof(uct_ud_iface_config_t, event_timer_tick), UCS_CONFIG_TYPE_TIME}, + + {"MIN_POKE_TIME", "250ms", + "Minimal internal to send ACK request with solicited flag, to wake up\n" + "the remote peer in case it is not actively calling progress.\n" + "Smaller values may incur performance overhead, while extermely large\n" + "values can cause delays in presence of packet drops.", + ucs_offsetof(uct_ud_iface_config_t, min_poke_time), UCS_CONFIG_TYPE_TIME}, + {"ETH_DGID_CHECK", "y", "Enable checking destination GID for incoming packets of Ethernet network.\n" "Mismatched packets are silently dropped.", diff --git a/src/uct/ib/ud/base/ud_iface.h b/src/uct/ib/ud/base/ud_iface.h index deef3fb08805..be6079e29bad 100644 --- a/src/uct/ib/ud/base/ud_iface.h +++ b/src/uct/ib/ud/base/ud_iface.h @@ -53,6 +53,7 @@ typedef struct uct_ud_iface_config { uct_ib_iface_config_t super; uct_ud_iface_common_config_t ud_common; double peer_timeout; + double min_poke_time; double timer_tick; double timer_backoff; double event_timer_tick; @@ -179,6 +180,7 @@ struct uct_ud_iface { } tx; struct { ucs_time_t peer_timeout; + ucs_time_t min_poke_time; unsigned tx_qp_len; unsigned max_inline; int check_grh_dgid;