xprtrdma: Wake RPCs directly in rpcrdma_wc_send path

[linux.git] / net / sunrpc / xprtrdma / rpc_rdma.c

diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 33b6e6a03f68b341975eba2bad7f6a2fdc3d5a2c..caf0b1950d7635215f4be572272203b7be04cd3b 100644 (file)
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -511,6 +511,16 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
        return 0;
 }
 
+static void rpcrdma_sendctx_done(struct kref *kref)
+{
+       struct rpcrdma_req *req =
+               container_of(kref, struct rpcrdma_req, rl_kref);
+       struct rpcrdma_rep *rep = req->rl_reply;
+
+       rpcrdma_complete_rqst(rep);
+       rep->rr_rxprt->rx_stats.reply_waits_for_send++;
+}
+
 /**
  * rpcrdma_sendctx_unmap - DMA-unmap Send buffer
  * @sc: sendctx containing SGEs to unmap
@@ -520,6 +530,9 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
 {
        struct ib_sge *sge;
 
+       if (!sc->sc_unmap_count)
+               return;
+
        /* The first two SGEs contain the transport header and
         * the inline buffer. These are always left mapped so
         * they can be cheaply re-used.
@@ -529,9 +542,7 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
                ib_dma_unmap_page(sc->sc_device, sge->addr, sge->length,
                                  DMA_TO_DEVICE);
 
-       if (test_and_clear_bit(RPCRDMA_REQ_F_TX_RESOURCES,
-                              &sc->sc_req->rl_flags))
-               wake_up_bit(&sc->sc_req->rl_flags, RPCRDMA_REQ_F_TX_RESOURCES);
+       kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done);
 }
 
 /* Prepare an SGE for the RPC-over-RDMA transport header.
@@ -666,7 +677,7 @@ static bool rpcrdma_prepare_msg_sges(struct rpcrdma_xprt *r_xprt,
 out:
        sc->sc_wr.num_sge += sge_no;
        if (sc->sc_unmap_count)
-               __set_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags);
+               kref_get(&req->rl_kref);
        return true;
 
 out_regbuf:
@@ -708,7 +719,7 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
        req->rl_sendctx->sc_wr.num_sge = 0;
        req->rl_sendctx->sc_unmap_count = 0;
        req->rl_sendctx->sc_req = req;
-       __clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags);
+       kref_init(&req->rl_kref);
 
        ret = -EIO;
        if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))
@@ -1268,36 +1279,12 @@ void rpcrdma_complete_rqst(struct rpcrdma_rep *rep)
        goto out;
 }
 
-/* Ensure that any DMA mapped pages associated with
- * the Send of the RPC Call have been unmapped before
- * allowing the RPC to complete. This protects argument
- * memory not controlled by the RPC client from being
- * re-used before we're done with it.
- */
-static void rpcrdma_release_tx(struct rpcrdma_xprt *r_xprt,
-                              struct rpcrdma_req *req)
+static void rpcrdma_reply_done(struct kref *kref)
 {
-       if (test_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags)) {
-               r_xprt->rx_stats.reply_waits_for_send++;
-               out_of_line_wait_on_bit(&req->rl_flags,
-                                       RPCRDMA_REQ_F_TX_RESOURCES,
-                                       bit_wait,
-                                       TASK_UNINTERRUPTIBLE);
-       }
-}
+       struct rpcrdma_req *req =
+               container_of(kref, struct rpcrdma_req, rl_kref);
 
-/**
- * rpcrdma_release_rqst - Release hardware resources
- * @r_xprt: controlling transport instance
- * @req: request with resources to release
- *
- */
-void rpcrdma_release_rqst(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
-{
-       if (!list_empty(&req->rl_registered))
-               frwr_unmap_sync(r_xprt, req);
-
-       rpcrdma_release_tx(r_xprt, req);
+       rpcrdma_complete_rqst(req->rl_reply);
 }
 
 /**
@@ -1367,13 +1354,11 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep)
 
        if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE)
                frwr_reminv(rep, &req->rl_registered);
-       if (!list_empty(&req->rl_registered)) {
+       if (!list_empty(&req->rl_registered))
                frwr_unmap_async(r_xprt, req);
                /* LocalInv completion will complete the RPC */
-       } else {
-               rpcrdma_release_tx(r_xprt, req);
-               rpcrdma_complete_rqst(rep);
-       }
+       else
+               kref_put(&req->rl_kref, rpcrdma_reply_done);
        return;
 
 out_badversion: