net/smc: improve delete link processing

Send an orderly DELETE LINK request before termination of a link group,
add support for client triggered DELETE LINK processing. And send a
disorderly DELETE LINK before module is unloaded.

Signed-off-by: Karsten Graul <kgraul@linux.ibm.com>
Signed-off-by: Ursula Braun <ubraun@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 90c10ae9ae09de8cb11537efc55f8e3a89fd67ab..a46418f45ecdea396ba2f2e8ac76e2614c383c3c 100644 (file)
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -30,6 +30,7 @@
 #define SMC_LGR_NUM_INCR               256
 #define SMC_LGR_FREE_DELAY_SERV                (600 * HZ)
 #define SMC_LGR_FREE_DELAY_CLNT                (SMC_LGR_FREE_DELAY_SERV + 10 * HZ)
+#define SMC_LGR_FREE_DELAY_FAST                (8 * HZ)
 
 static struct smc_lgr_list smc_lgr_list = {    /* established link groups */
        .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
@@ -51,6 +52,11 @@ static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
                         SMC_LGR_FREE_DELAY_CLNT : SMC_LGR_FREE_DELAY_SERV);
 }
 
+void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr)
+{
+       mod_delayed_work(system_wq, &lgr->free_work, SMC_LGR_FREE_DELAY_FAST);
+}
+
 /* Register connection's alert token in our lookup structure.
  * To use rbtrees we have to implement our own insert core.
  * Requires @conns_lock
@@ -133,6 +139,20 @@ static void smc_lgr_unregister_conn(struct smc_connection *conn)
        smc_lgr_schedule_free_work(lgr);
 }
 
+/* Send delete link, either as client to request the initiation
+ * of the DELETE LINK sequence from server; or as server to
+ * initiate the delete processing. See smc_llc_rx_delete_link().
+ */
+static int smc_link_send_delete(struct smc_link *lnk)
+{
+       if (lnk->state == SMC_LNK_ACTIVE &&
+           !smc_llc_send_delete_link(lnk, SMC_LLC_REQ, true)) {
+               smc_llc_link_deleting(lnk);
+               return 0;
+       }
+       return -ENOTCONN;
+}
+
 static void smc_lgr_free_work(struct work_struct *work)
 {
        struct smc_link_group *lgr = container_of(to_delayed_work(work),
@@ -153,10 +173,21 @@ static void smc_lgr_free_work(struct work_struct *work)
        list_del_init(&lgr->list); /* remove from smc_lgr_list */
 free:
        spin_unlock_bh(&smc_lgr_list.lock);
+
+       if (!lgr->is_smcd && !lgr->terminating) {
+               /* try to send del link msg, on error free lgr immediately */
+               if (!smc_link_send_delete(&lgr->lnk[SMC_SINGLE_LINK])) {
+                       /* reschedule in case we never receive a response */
+                       smc_lgr_schedule_free_work(lgr);
+                       return;
+               }
+       }
+
        if (!delayed_work_pending(&lgr->free_work)) {
-               if (!lgr->is_smcd &&
-                   lgr->lnk[SMC_SINGLE_LINK].state != SMC_LNK_INACTIVE)
-                       smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]);
+               struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
+
+               if (!lgr->is_smcd && lnk->state != SMC_LNK_INACTIVE)
+                       smc_llc_link_inactive(lnk);
                smc_lgr_free(lgr);
        }
 }
@@ -984,8 +1015,14 @@ void smc_core_exit(void)
        spin_unlock_bh(&smc_lgr_list.lock);
        list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) {
                list_del_init(&lgr->list);
-               if (!lgr->is_smcd)
-                       smc_llc_link_inactive(&lgr->lnk[SMC_SINGLE_LINK]);
+               if (!lgr->is_smcd) {
+                       struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
+
+                       if (lnk->state == SMC_LNK_ACTIVE)
+                               smc_llc_send_delete_link(lnk, SMC_LLC_REQ,
+                                                        false);
+                       smc_llc_link_inactive(lnk);
+               }
                cancel_delayed_work_sync(&lgr->free_work);
                smc_lgr_free(lgr); /* free link group */
        }

diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index a4f0cc4e0270119806f8d22ef38309d53914cd1b..c156674733c9dcb37af68e47a660785d140231b3 100644 (file)
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -34,7 +34,8 @@ enum smc_lgr_role {           /* possible roles of a link group */
 enum smc_link_state {                  /* possible states of a link */
        SMC_LNK_INACTIVE,       /* link is inactive */
        SMC_LNK_ACTIVATING,     /* link is being activated */
-       SMC_LNK_ACTIVE          /* link is active */
+       SMC_LNK_ACTIVE,         /* link is active */
+       SMC_LNK_DELETING,       /* link is being deleted */
 };
 
 #define SMC_WR_BUF_SIZE                48      /* size of work request buffer */
@@ -265,6 +266,7 @@ int smc_conn_create(struct smc_sock *smc, bool is_smcd, int srv_first_contact,
                    struct smc_clc_msg_local *lcl, struct smcd_dev *smcd,
                    u64 peer_gid);
 void smcd_conn_free(struct smc_connection *conn);
+void smc_lgr_schedule_free_work_fast(struct smc_link_group *lgr);
 void smc_core_exit(void);
 
 static inline struct smc_link_group *smc_get_lgr(struct smc_link *link)

diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
index a88c01029fa6fc589e4b78532b2e0d3e40414bc6..9c916c709ca71129d52d3e2d695e5a8076244458 100644 (file)
--- a/net/smc/smc_llc.c
+++ b/net/smc/smc_llc.c
@@ -278,7 +278,7 @@ int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 /* prepare a delete link message */
 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
                                     struct smc_link *link,
-                                    enum smc_llc_reqresp reqresp)
+                                    enum smc_llc_reqresp reqresp, bool orderly)
 {
        memset(delllc, 0, sizeof(*delllc));
        delllc->hd.common.type = SMC_LLC_DELETE_LINK;
@@ -287,13 +287,14 @@ static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
                delllc->hd.flags |= SMC_LLC_FLAG_RESP;
        /* DEL_LINK_ALL because only 1 link supported */
        delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
-       delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
+       if (orderly)
+               delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
        delllc->link_num = link->link_id;
 }
 
 /* send DELETE LINK request or response */
 int smc_llc_send_delete_link(struct smc_link *link,
-                            enum smc_llc_reqresp reqresp)
+                            enum smc_llc_reqresp reqresp, bool orderly)
 {
        struct smc_llc_msg_del_link *delllc;
        struct smc_wr_tx_pend_priv *pend;
@@ -304,7 +305,7 @@ int smc_llc_send_delete_link(struct smc_link *link,
        if (rc)
                return rc;
        delllc = (struct smc_llc_msg_del_link *)wr_buf;
-       smc_llc_prep_delete_link(delllc, link, reqresp);
+       smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
        /* send llc message */
        rc = smc_wr_tx_send(link, pend);
        return rc;
@@ -438,17 +439,19 @@ static void smc_llc_rx_delete_link(struct smc_link *link,
 
        if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
                if (lgr->role == SMC_SERV)
-                       smc_lgr_terminate(lgr);
+                       smc_lgr_schedule_free_work_fast(lgr);
        } else {
+               smc_lgr_forget(lgr);
+               smc_llc_link_deleting(link);
                if (lgr->role == SMC_SERV) {
-                       smc_lgr_forget(lgr);
-                       smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ);
-                       smc_llc_send_message(link, llc, sizeof(*llc));
+                       /* client asks to delete this link, send request */
+                       smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
                } else {
-                       smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP);
-                       smc_llc_send_message(link, llc, sizeof(*llc));
-                       smc_lgr_terminate(lgr);
+                       /* server requests to delete this link, send response */
+                       smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
                }
+               smc_llc_send_message(link, llc, sizeof(*llc));
+               smc_lgr_schedule_free_work_fast(lgr);
        }
 }
 
@@ -622,6 +625,11 @@ void smc_llc_link_active(struct smc_link *link, int testlink_time)
        }
 }
 
+void smc_llc_link_deleting(struct smc_link *link)
+{
+       link->state = SMC_LNK_DELETING;
+}
+
 /* called in tasklet context */
 void smc_llc_link_inactive(struct smc_link *link)
 {

diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
index 95a7f3662e5914fa2d2c75057b0818e2b40b4dcb..9e2ff088e30188e08c1658d9904838318b18be12 100644 (file)
--- a/net/smc/smc_llc.h
+++ b/net/smc/smc_llc.h
@@ -41,9 +41,10 @@ int smc_llc_send_confirm_link(struct smc_link *lnk,
 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
                          enum smc_llc_reqresp reqresp);
 int smc_llc_send_delete_link(struct smc_link *link,
-                            enum smc_llc_reqresp reqresp);
+                            enum smc_llc_reqresp reqresp, bool orderly);
 int smc_llc_link_init(struct smc_link *link);
 void smc_llc_link_active(struct smc_link *link, int testlink_time);
+void smc_llc_link_deleting(struct smc_link *link);
 void smc_llc_link_inactive(struct smc_link *link);
 void smc_llc_link_clear(struct smc_link *link);
 int smc_llc_do_confirm_rkey(struct smc_link *link,

diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c
index b6df69756bef6967616f6acaa4def86ad79e8fac..f856b8402b3fe527c0dbf80225d501bcab9336ce 100644 (file)
--- a/net/smc/smc_wr.c
+++ b/net/smc/smc_wr.c
@@ -182,17 +182,14 @@ int smc_wr_tx_get_free_slot(struct smc_link *link,
                if (rc)
                        return rc;
        } else {
-               struct smc_link_group *lgr;
-
-               lgr = smc_get_lgr(link);
                rc = wait_event_timeout(
                        link->wr_tx_wait,
-                       list_empty(&lgr->list) || /* lgr terminated */
+                       link->state == SMC_LNK_INACTIVE ||
                        (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
                        SMC_WR_TX_WAIT_FREE_SLOT_TIME);
                if (!rc) {
                        /* timeout - terminate connections */
-                       smc_lgr_terminate(lgr);
+                       smc_lgr_terminate(smc_get_lgr(link));
                        return -EPIPE;
                }
                if (idx == link->wr_tx_cnt)