net/smc: don't send CDC/LLC message if link not ready

We found smc_llc_send_link_delete_all() sometimes wait
for 2s timeout when testing with RDMA link up/down.
It is possible when a smc_link is in ACTIVATING state,
the underlaying QP is still in RESET or RTR state, which
cannot send any messages out.

smc_llc_send_link_delete_all() use smc_link_usable() to
checks whether the link is usable, if the QP is still in
RESET or RTR state, but the smc_link is in ACTIVATING, this
LLC message will always fail without any CQE entering the
CQ, and we will always wait 2s before timeout.

Since we cannot send any messages through the QP before
the QP enter RTS. I add a wrapper smc_link_sendable()
which checks the state of QP along with the link state.
And replace smc_link_usable() with smc_link_sendable()
in all LLC & CDC message sending routine.

Fixes: 5f08318f617b ("smc: connection data control (CDC)")
Signed-off-by: Dust Li <dust.li@linux.alibaba.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 387d28b2f8dd055a3b3ceea6024ffb231f115877..55ca175e8d5786213853a29da6f935bbb9ea2e5e 100644 (file)
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -647,7 +647,7 @@ static void smcr_lgr_link_deactivate_all(struct smc_link_group *lgr)
        for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
                struct smc_link *lnk = &lgr->lnk[i];
 
-               if (smc_link_usable(lnk))
+               if (smc_link_sendable(lnk))
                        lnk->state = SMC_LNK_INACTIVE;
        }
        wake_up_all(&lgr->llc_msg_waiter);

diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 59cef3b830d835d262d806058821f6ca84df2bff..d63b08274197eb28959f099c90548938baa6eefd 100644 (file)
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -415,6 +415,12 @@ static inline bool smc_link_usable(struct smc_link *lnk)
        return true;
 }
 
+static inline bool smc_link_sendable(struct smc_link *lnk)
+{
+       return smc_link_usable(lnk) &&
+               lnk->qp_attr.cur_qp_state == IB_QPS_RTS;
+}
+
 static inline bool smc_link_active(struct smc_link *lnk)
 {
        return lnk->state == SMC_LNK_ACTIVE;

diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
index b102680296b898ec8cd0491f5546630e1c20c505..3e9fd8a3124c5dc8df18cf0e963450ad3a916e33 100644 (file)
--- a/net/smc/smc_llc.c
+++ b/net/smc/smc_llc.c
@@ -1630,7 +1630,7 @@ void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
        delllc.reason = htonl(rsn);
 
        for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
-               if (!smc_link_usable(&lgr->lnk[i]))
+               if (!smc_link_sendable(&lgr->lnk[i]))
                        continue;
                if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
                        break;

diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c
index 79a7431f534e2cb2d4fd1719fd530426b9d0da7f..df1dc225cbab60612d81f839933d3f0a8333dc01 100644 (file)
--- a/net/smc/smc_wr.c
+++ b/net/smc/smc_wr.c
@@ -188,7 +188,7 @@ void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
 {
        *idx = link->wr_tx_cnt;
-       if (!smc_link_usable(link))
+       if (!smc_link_sendable(link))
                return -ENOLINK;
        for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
                if (!test_and_set_bit(*idx, link->wr_tx_mask))
@@ -231,7 +231,7 @@ int smc_wr_tx_get_free_slot(struct smc_link *link,
        } else {
                rc = wait_event_interruptible_timeout(
                        link->wr_tx_wait,
-                       !smc_link_usable(link) ||
+                       !smc_link_sendable(link) ||
                        lgr->terminating ||
                        (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
                        SMC_WR_TX_WAIT_FREE_SLOT_TIME);

diff --git a/net/smc/smc_wr.h b/net/smc/smc_wr.h
index f353311e6f84b1e86fc950dba781d71d970805f7..48ed9b08ac7add07f626b57dec0425596cf16316 100644 (file)
--- a/net/smc/smc_wr.h
+++ b/net/smc/smc_wr.h
@@ -62,7 +62,7 @@ static inline void smc_wr_tx_set_wr_id(atomic_long_t *wr_tx_id, long val)
 
 static inline bool smc_wr_tx_link_hold(struct smc_link *link)
 {
-       if (!smc_link_usable(link))
+       if (!smc_link_sendable(link))
                return false;
        atomic_inc(&link->wr_tx_refcnt);
        return true;