From e2651d9ec1746faca3058d0ad60808101eb33bfa Mon Sep 17 00:00:00 2001
From: Mikhail Brinskii <mikhailb@mellanox.com>
Date: Thu, 19 Jul 2018 17:03:28 +0300
Subject: [PATCH] UCT/DC: Fix DCI pending allocation scheme

---
 src/uct/ib/dc/base/dc_ep.c   |  8 +++----
 test/gtest/uct/ib/test_dc.cc | 44 +++++++++++++++++++++++++++++++++++-
 2 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/src/uct/ib/dc/base/dc_ep.c b/src/uct/ib/dc/base/dc_ep.c
index 8b1175dcf2b..b57e4bfec10 100644
--- a/src/uct/ib/dc/base/dc_ep.c
+++ b/src/uct/ib/dc/base/dc_ep.c
@@ -129,11 +129,9 @@ uct_dc_iface_dci_do_pending_wait(ucs_arbiter_t *arbiter,
     uct_dc_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem), uct_dc_ep_t, arb_group);
     uct_dc_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_dc_iface_t);
 
-    /**
-     * stop if dci can not be allocated
-     * else move group to the dci arbiter
-     */
-    ucs_assert_always(ep->dci == UCT_DC_EP_NO_DCI);
+    if (ep->dci != UCT_DC_EP_NO_DCI) {
+        return UCS_ARBITER_CB_RESULT_DESCHED_GROUP;
+    }
 
     if (!uct_dc_iface_dci_can_alloc(iface)) {
         return UCS_ARBITER_CB_RESULT_STOP;
diff --git a/test/gtest/uct/ib/test_dc.cc b/test/gtest/uct/ib/test_dc.cc
index 442c59330b8..c4130aab452 100644
--- a/test/gtest/uct/ib/test_dc.cc
+++ b/test/gtest/uct/ib/test_dc.cc
@@ -306,7 +306,7 @@ UCS_TEST_P(test_dc, dcs_ep_flush_pending) {
     preq.uct_req.func = uct_pending_flush;
     status = uct_ep_pending_add(m_e1->ep(0), &preq.uct_req);
     EXPECT_UCS_OK(status);
-    
+
     /* progress till ep is flushed */
     do {
         progress();
@@ -319,6 +319,48 @@ UCS_TEST_P(test_dc, dcs_ep_flush_pending) {
     EXPECT_EQ(0, iface->tx.stack_top);
 }
 
+/* Check that the following sequnce works ok:
+ * - Add some pending request to DCI wait queue
+ * - Try to send something from this ep. This will force ep to take free DCI
+ *   (the send will not succeed anyway)
+ * - Progress all pendings
+ * - Make sure that there is no any assertion and everyting is ok
+ *   (just send something).
+ * */
+UCS_TEST_P(test_dc, dcs_ep_am_pending) {
+
+    ucs_status_t status;
+    uct_dc_iface_t *iface;
+
+    m_e1->connect_to_iface(0, *m_e2);
+    m_e1->connect_to_iface(1, *m_e2);
+
+    /* use all iface resources */
+    iface = dc_iface(m_e1);
+    iface->super.tx.cq_available = 8;
+    do {
+        status = uct_ep_am_short(m_e1->ep(1), 0, 0, NULL, 0);
+    } while (status == UCS_OK);
+
+    EXPECT_EQ(UCS_ERR_NO_RESOURCE, status);
+
+    /* put AM op on pending */
+    preq.e            = m_e1;
+    preq.uct_req.func = uct_pending_flush;
+    status            = uct_ep_pending_add(m_e1->ep(0), &preq.uct_req);
+    EXPECT_UCS_OK(status);
+
+    status = uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0);
+    EXPECT_EQ(UCS_ERR_NO_RESOURCE, status);
+
+    flush();
+
+    status = uct_ep_am_short(m_e1->ep(0), 0, 0, NULL, 0);
+    EXPECT_EQ(UCS_OK, status);
+
+    flush();
+}
+
 /* check that ep does not hold dci after
  * purge
  */