xprtrdma: rpcrdma_regbuf alignment

Allocate the struct rpcrdma_regbuf separately from the I/O buffer
to better guarantee the alignment of the I/O buffer and eliminate
the wasted space between the rpcrdma_regbuf metadata and the buffer
itself.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c
index 713961a63c49284bd2348ec944c45d17c2d7fc2e..6170ec7ba504c87d911c4fe457f6565fbe6d8f98 100644 (file)
--- a/net/sunrpc/xprtrdma/backchannel.c
+++ b/net/sunrpc/xprtrdma/backchannel.c
@@ -45,10 +45,10 @@ static int rpcrdma_bc_setup_reqs(struct rpcrdma_xprt *r_xprt,
 
                size = r_xprt->rx_data.inline_rsize;
                rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
-               if (IS_ERR(rb))
+               if (!rb)
                        goto out_fail;
                req->rl_sendbuf = rb;
-               xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base,
+               xdr_buf_init(&rqst->rq_snd_buf, rdmab_data(rb),
                             min_t(size_t, size, PAGE_SIZE));
        }
        return 0;
@@ -123,7 +123,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst)
 
        rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);
        xdr_init_encode(&req->rl_stream, &req->rl_hdrbuf,
-                       req->rl_rdmabuf->rg_base, rqst);
+                       rdmab_data(req->rl_rdmabuf), rqst);
 
        p = xdr_reserve_space(&req->rl_stream, 28);
        if (unlikely(!p))

diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index b759b169dadfb20164c36e922d915158ec5a8bb0..cf99c55add1b90687bf62d1d45f5376845c506c0 100644 (file)
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -747,8 +747,8 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst)
        int ret;
 
        rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);
-       xdr_init_encode(xdr, &req->rl_hdrbuf,
-                       req->rl_rdmabuf->rg_base, rqst);
+       xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf),
+                       rqst);
 
        /* Fixed header fields */
        ret = -EMSGSIZE;

diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index 7e73abe01cfe2aad12a29b03d5725f420904b15d..ced9812940f77266d0eb890e8cc1ae4bb050c6ca 100644 (file)
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -595,7 +595,7 @@ rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
                return true;
 
        rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags);
-       if (IS_ERR(rb))
+       if (!rb)
                return false;
 
        rpcrdma_free_regbuf(req->rl_sendbuf);
@@ -625,7 +625,7 @@ rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
                return true;
 
        rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags);
-       if (IS_ERR(rb))
+       if (!rb)
                return false;
 
        rpcrdma_free_regbuf(req->rl_recvbuf);
@@ -660,8 +660,8 @@ xprt_rdma_allocate(struct rpc_task *task)
        if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags))
                goto out_fail;
 
-       rqst->rq_buffer = req->rl_sendbuf->rg_base;
-       rqst->rq_rbuffer = req->rl_recvbuf->rg_base;
+       rqst->rq_buffer = rdmab_data(req->rl_sendbuf);
+       rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf);
        trace_xprtrdma_op_allocate(task, req);
        return 0;
 

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index caa6a5df12b0e0e23567bf018b8c462d58da39e7..f88fd3934f5613ad7ced8e3612047d5d4992142e 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -1013,12 +1013,12 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, gfp_t flags)
                return NULL;
 
        rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, flags);
-       if (IS_ERR(rb)) {
+       if (!rb) {
                kfree(req);
                return NULL;
        }
        req->rl_rdmabuf = rb;
-       xdr_buf_init(&req->rl_hdrbuf, rb->rg_base, rdmab_length(rb));
+       xdr_buf_init(&req->rl_hdrbuf, rdmab_data(rb), rdmab_length(rb));
        req->rl_buffer = buffer;
        INIT_LIST_HEAD(&req->rl_registered);
 
@@ -1040,9 +1040,9 @@ static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp)
 
        rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize,
                                               DMA_FROM_DEVICE, GFP_KERNEL);
-       if (IS_ERR(rep->rr_rdmabuf))
+       if (!rep->rr_rdmabuf)
                goto out_free;
-       xdr_buf_init(&rep->rr_hdrbuf, rep->rr_rdmabuf->rg_base,
+       xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),
                     rdmab_length(rep->rr_rdmabuf));
 
        rep->rr_cqe.done = rpcrdma_wc_receive;
@@ -1354,8 +1354,7 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
  * @direction: direction of data movement
  * @flags: GFP flags
  *
- * Returns an ERR_PTR, or a pointer to a regbuf, a buffer that
- * can be persistently DMA-mapped for I/O.
+ * Returns a pointer to a rpcrdma_regbuf object, or NULL.
  *
  * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
  * receiving the payload of RDMA RECV operations. During Long Calls
@@ -1367,14 +1366,18 @@ rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction,
 {
        struct rpcrdma_regbuf *rb;
 
-       rb = kmalloc(sizeof(*rb) + size, flags);
-       if (rb == NULL)
-               return ERR_PTR(-ENOMEM);
+       rb = kmalloc(sizeof(*rb), flags);
+       if (!rb)
+               return NULL;
+       rb->rg_data = kmalloc(size, flags);
+       if (!rb->rg_data) {
+               kfree(rb);
+               return NULL;
+       }
 
        rb->rg_device = NULL;
        rb->rg_direction = direction;
        rb->rg_iov.length = size;
-
        return rb;
 }
 
@@ -1392,7 +1395,7 @@ __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
                return false;
 
        rb->rg_iov.addr = ib_dma_map_single(device,
-                                           (void *)rb->rg_base,
+                                           rdmab_data(rb),
                                            rdmab_length(rb),
                                            rb->rg_direction);
        if (ib_dma_mapping_error(device, rdmab_addr(rb))) {
@@ -1427,6 +1430,8 @@ void
 rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb)
 {
        rpcrdma_dma_unmap_regbuf(rb);
+       if (rb)
+               kfree(rb->rg_data);
        kfree(rb);
 }
 

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 539558fc9c623d2a210f9968c19b8e9450acdaec..1af9674572bd0a7b5198e43d1e38c12314143eed 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -121,33 +121,34 @@ struct rpcrdma_regbuf {
        struct ib_sge           rg_iov;
        struct ib_device        *rg_device;
        enum dma_data_direction rg_direction;
-       __be32                  rg_base[0] __attribute__ ((aligned(256)));
+       void                    *rg_data;
 };
 
-static inline u64
-rdmab_addr(struct rpcrdma_regbuf *rb)
+static inline u64 rdmab_addr(struct rpcrdma_regbuf *rb)
 {
        return rb->rg_iov.addr;
 }
 
-static inline u32
-rdmab_length(struct rpcrdma_regbuf *rb)
+static inline u32 rdmab_length(struct rpcrdma_regbuf *rb)
 {
        return rb->rg_iov.length;
 }
 
-static inline u32
-rdmab_lkey(struct rpcrdma_regbuf *rb)
+static inline u32 rdmab_lkey(struct rpcrdma_regbuf *rb)
 {
        return rb->rg_iov.lkey;
 }
 
-static inline struct ib_device *
-rdmab_device(struct rpcrdma_regbuf *rb)
+static inline struct ib_device *rdmab_device(struct rpcrdma_regbuf *rb)
 {
        return rb->rg_device;
 }
 
+static inline void *rdmab_data(const struct rpcrdma_regbuf *rb)
+{
+       return rb->rg_data;
+}
+
 #define RPCRDMA_DEF_GFP                (GFP_NOIO | __GFP_NOWARN)
 
 /* To ensure a transport can always make forward progress,