xprtrdma: Wake RPCs directly in rpcrdma_wc_send path

[linux.git] / net / sunrpc / xprtrdma / xprt_rdma.h

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index d1e0749bcbc49a859a4e93af098987ea9124b104..5475f0dff22a6e5fc940386a4c016d1ccd12b387 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -44,7 +44,8 @@
 
 #include <linux/wait.h>                /* wait_queue_head_t, etc */
 #include <linux/spinlock.h>            /* spinlock_t, etc */
-#include <linux/atomic.h>                      /* atomic_t, etc */
+#include <linux/atomic.h>              /* atomic_t, etc */
+#include <linux/kref.h>                        /* struct kref */
 #include <linux/workqueue.h>           /* struct work_struct */
 
 #include <rdma/rdma_cm.h>              /* RDMA connection api */
@@ -202,10 +203,9 @@ struct rpcrdma_rep {
        bool                    rr_temp;
        struct rpcrdma_regbuf   *rr_rdmabuf;
        struct rpcrdma_xprt     *rr_rxprt;
-       struct work_struct      rr_work;
+       struct rpc_rqst         *rr_rqst;
        struct xdr_buf          rr_hdrbuf;
        struct xdr_stream       rr_stream;
-       struct rpc_rqst         *rr_rqst;
        struct list_head        rr_list;
        struct ib_recv_wr       rr_recv_wr;
 };
@@ -240,18 +240,12 @@ struct rpcrdma_sendctx {
  * An external memory region is any buffer or page that is registered
  * on the fly (ie, not pre-registered).
  */
-enum rpcrdma_frwr_state {
-       FRWR_IS_INVALID,        /* ready to be used */
-       FRWR_IS_VALID,          /* in use */
-       FRWR_FLUSHED_FR,        /* flushed FASTREG WR */
-       FRWR_FLUSHED_LI,        /* flushed LOCALINV WR */
-};
-
+struct rpcrdma_req;
 struct rpcrdma_frwr {
        struct ib_mr                    *fr_mr;
        struct ib_cqe                   fr_cqe;
-       enum rpcrdma_frwr_state         fr_state;
        struct completion               fr_linv_done;
+       struct rpcrdma_req              *fr_req;
        union {
                struct ib_reg_wr        fr_regwr;
                struct ib_send_wr       fr_invwr;
@@ -336,18 +330,12 @@ struct rpcrdma_req {
        struct rpcrdma_regbuf   *rl_recvbuf;    /* rq_rcv_buf */
 
        struct list_head        rl_all;
-       unsigned long           rl_flags;
+       struct kref             rl_kref;
 
        struct list_head        rl_registered;  /* registered segments */
        struct rpcrdma_mr_seg   rl_segments[RPCRDMA_MAX_SEGS];
 };
 
-/* rl_flags */
-enum {
-       RPCRDMA_REQ_F_PENDING = 0,
-       RPCRDMA_REQ_F_TX_RESOURCES,
-};
-
 static inline struct rpcrdma_req *
 rpcr_to_rdmar(const struct rpc_rqst *rqst)
 {
@@ -391,22 +379,15 @@ struct rpcrdma_buffer {
        struct list_head        rb_recv_bufs;
        struct list_head        rb_allreqs;
 
-       unsigned long           rb_flags;
        u32                     rb_max_requests;
        u32                     rb_credits;     /* most recent credit grant */
 
        u32                     rb_bc_srv_max_requests;
        u32                     rb_bc_max_requests;
 
-       struct workqueue_struct *rb_completion_wq;
        struct delayed_work     rb_refresh_worker;
 };
 
-/* rb_flags */
-enum {
-       RPCRDMA_BUF_F_EMPTY_SCQ = 0,
-};
-
 /*
  * Statistics for RPCRDMA
  */
@@ -564,6 +545,7 @@ rpcrdma_data_dir(bool writing)
 /* Memory registration calls xprtrdma/frwr_ops.c
  */
 bool frwr_is_supported(struct ib_device *device);
+void frwr_reset(struct rpcrdma_req *req);
 int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep);
 int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr);
 void frwr_release_mr(struct rpcrdma_mr *mr);
@@ -574,8 +556,8 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
                                struct rpcrdma_mr **mr);
 int frwr_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req);
 void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs);
-void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt,
-                    struct list_head *mrs);
+void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
+void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
 
 /*
  * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
@@ -598,9 +580,6 @@ int rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst);
 void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
 void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);
 void rpcrdma_reply_handler(struct rpcrdma_rep *rep);
-void rpcrdma_release_rqst(struct rpcrdma_xprt *r_xprt,
-                         struct rpcrdma_req *req);
-void rpcrdma_deferred_completion(struct work_struct *work);
 
 static inline void rpcrdma_set_xdrlen(struct xdr_buf *xdr, size_t len)
 {