net/sunrpc/xprtrdma/svc_rdma_rw.c

   1 /*
   2  * Copyright (c) 2016 Oracle.  All rights reserved.
   3  *
   4  * Use the core R/W API to move RPC-over-RDMA Read and Write chunks.
   5  */
   6
   7 #include <linux/sunrpc/rpc_rdma.h>
   8 #include <linux/sunrpc/svc_rdma.h>
   9 #include <linux/sunrpc/debug.h>
  10
  11 #include <rdma/rw.h>
  12
  13 #define RPCDBG_FACILITY RPCDBG_SVCXPRT
  14
  15 /* Each R/W context contains state for one chain of RDMA Read or
  16  * Write Work Requests.
  17  *
  18  * Each WR chain handles a single contiguous server-side buffer,
  19  * because scatterlist entries after the first have to start on
  20  * page alignment. xdr_buf iovecs cannot guarantee alignment.
  21  *
  22  * Each WR chain handles only one R_key. Each RPC-over-RDMA segment
  23  * from a client may contain a unique R_key, so each WR chain moves
  24  * up to one segment at a time.
  25  *
  26  * The scatterlist makes this data structure over 4KB in size. To
  27  * make it less likely to fail, and to handle the allocation for
  28  * smaller I/O requests without disabling bottom-halves, these
  29  * contexts are created on demand, but cached and reused until the
  30  * controlling svcxprt_rdma is destroyed.
  31  */
  32 struct svc_rdma_rw_ctxt {
  33         struct list_head        rw_list;
  34         struct rdma_rw_ctx      rw_ctx;
  35         int                     rw_nents;
  36         struct sg_table         rw_sg_table;
  37         struct scatterlist      rw_first_sgl[0];
  38 };
  39
  40 static inline struct svc_rdma_rw_ctxt *
  41 svc_rdma_next_ctxt(struct list_head *list)
  42 {
  43         return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt,
  44                                         rw_list);
  45 }
  46
  47 static struct svc_rdma_rw_ctxt *
  48 svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges)
  49 {
  50         struct svc_rdma_rw_ctxt *ctxt;
  51
  52         spin_lock(&rdma->sc_rw_ctxt_lock);
  53
  54         ctxt = svc_rdma_next_ctxt(&rdma->sc_rw_ctxts);
  55         if (ctxt) {
  56                 list_del(&ctxt->rw_list);
  57                 spin_unlock(&rdma->sc_rw_ctxt_lock);
  58         } else {
  59                 spin_unlock(&rdma->sc_rw_ctxt_lock);
  60                 ctxt = kmalloc(sizeof(*ctxt) +
  61                                SG_CHUNK_SIZE * sizeof(struct scatterlist),
  62                                GFP_KERNEL);
  63                 if (!ctxt)
  64                         goto out;
  65                 INIT_LIST_HEAD(&ctxt->rw_list);
  66         }
  67
  68         ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl;
  69         if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
  70                                    ctxt->rw_sg_table.sgl)) {
  71                 kfree(ctxt);
  72                 ctxt = NULL;
  73         }
  74 out:
  75         return ctxt;
  76 }
  77
  78 static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma,
  79                                  struct svc_rdma_rw_ctxt *ctxt)
  80 {
  81         sg_free_table_chained(&ctxt->rw_sg_table, true);
  82
  83         spin_lock(&rdma->sc_rw_ctxt_lock);
  84         list_add(&ctxt->rw_list, &rdma->sc_rw_ctxts);
  85         spin_unlock(&rdma->sc_rw_ctxt_lock);
  86 }
  87
  88 /**
  89  * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts
  90  * @rdma: transport about to be destroyed
  91  *
  92  */
  93 void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma)
  94 {
  95         struct svc_rdma_rw_ctxt *ctxt;
  96
  97         while ((ctxt = svc_rdma_next_ctxt(&rdma->sc_rw_ctxts)) != NULL) {
  98                 list_del(&ctxt->rw_list);
  99                 kfree(ctxt);
 100         }
 101 }
 102
 103 /* A chunk context tracks all I/O for moving one Read or Write
 104  * chunk. This is a a set of rdma_rw's that handle data movement
 105  * for all segments of one chunk.
 106  *
 107  * These are small, acquired with a single allocator call, and
 108  * no more than one is needed per chunk. They are allocated on
 109  * demand, and not cached.
 110  */
 111 struct svc_rdma_chunk_ctxt {
 112         struct ib_cqe           cc_cqe;
 113         struct svcxprt_rdma     *cc_rdma;
 114         struct list_head        cc_rwctxts;
 115         int                     cc_sqecount;
 116         enum dma_data_direction cc_dir;
 117 };
 118
 119 static void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
 120                              struct svc_rdma_chunk_ctxt *cc,
 121                              enum dma_data_direction dir)
 122 {
 123         cc->cc_rdma = rdma;
 124         svc_xprt_get(&rdma->sc_xprt);
 125
 126         INIT_LIST_HEAD(&cc->cc_rwctxts);
 127         cc->cc_sqecount = 0;
 128         cc->cc_dir = dir;
 129 }
 130
 131 static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc)
 132 {
 133         struct svcxprt_rdma *rdma = cc->cc_rdma;
 134         struct svc_rdma_rw_ctxt *ctxt;
 135
 136         while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) {
 137                 list_del(&ctxt->rw_list);
 138
 139                 rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp,
 140                                     rdma->sc_port_num, ctxt->rw_sg_table.sgl,
 141                                     ctxt->rw_nents, cc->cc_dir);
 142                 svc_rdma_put_rw_ctxt(rdma, ctxt);
 143         }
 144         svc_xprt_put(&rdma->sc_xprt);
 145 }
 146
 147 /* State for sending a Write or Reply chunk.
 148  *  - Tracks progress of writing one chunk over all its segments
 149  *  - Stores arguments for the SGL constructor functions
 150  */
 151 struct svc_rdma_write_info {
 152         /* write state of this chunk */
 153         unsigned int            wi_seg_off;
 154         unsigned int            wi_seg_no;
 155         unsigned int            wi_nsegs;
 156         __be32                  *wi_segs;
 157
 158         /* SGL constructor arguments */
 159         struct xdr_buf          *wi_xdr;
 160         unsigned char           *wi_base;
 161         unsigned int            wi_next_off;
 162
 163         struct svc_rdma_chunk_ctxt      wi_cc;
 164 };
 165
 166 static struct svc_rdma_write_info *
 167 svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, __be32 *chunk)
 168 {
 169         struct svc_rdma_write_info *info;
 170
 171         info = kmalloc(sizeof(*info), GFP_KERNEL);
 172         if (!info)
 173                 return info;
 174
 175         info->wi_seg_off = 0;
 176         info->wi_seg_no = 0;
 177         info->wi_nsegs = be32_to_cpup(++chunk);
 178         info->wi_segs = ++chunk;
 179         svc_rdma_cc_init(rdma, &info->wi_cc, DMA_TO_DEVICE);
 180         return info;
 181 }
 182
 183 static void svc_rdma_write_info_free(struct svc_rdma_write_info *info)
 184 {
 185         svc_rdma_cc_release(&info->wi_cc);
 186         kfree(info);
 187 }
 188
 189 /**
 190  * svc_rdma_write_done - Write chunk completion
 191  * @cq: controlling Completion Queue
 192  * @wc: Work Completion
 193  *
 194  * Pages under I/O are freed by a subsequent Send completion.
 195  */
 196 static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
 197 {
 198         struct ib_cqe *cqe = wc->wr_cqe;
 199         struct svc_rdma_chunk_ctxt *cc =
 200                         container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
 201         struct svcxprt_rdma *rdma = cc->cc_rdma;
 202         struct svc_rdma_write_info *info =
 203                         container_of(cc, struct svc_rdma_write_info, wi_cc);
 204
 205         atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
 206         wake_up(&rdma->sc_send_wait);
 207
 208         if (unlikely(wc->status != IB_WC_SUCCESS)) {
 209                 set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
 210                 if (wc->status != IB_WC_WR_FLUSH_ERR)
 211                         pr_err("svcrdma: write ctx: %s (%u/0x%x)\n",
 212                                ib_wc_status_msg(wc->status),
 213                                wc->status, wc->vendor_err);
 214         }
 215
 216         svc_rdma_write_info_free(info);
 217 }
 218
 219 /* This function sleeps when the transport's Send Queue is congested.
 220  *
 221  * Assumptions:
 222  * - If ib_post_send() succeeds, only one completion is expected,
 223  *   even if one or more WRs are flushed. This is true when posting
 224  *   an rdma_rw_ctx or when posting a single signaled WR.
 225  */
 226 static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc)
 227 {
 228         struct svcxprt_rdma *rdma = cc->cc_rdma;
 229         struct svc_xprt *xprt = &rdma->sc_xprt;
 230         struct ib_send_wr *first_wr, *bad_wr;
 231         struct list_head *tmp;
 232         struct ib_cqe *cqe;
 233         int ret;
 234
 235         if (cc->cc_sqecount > rdma->sc_sq_depth)
 236                 return -EINVAL;
 237
 238         first_wr = NULL;
 239         cqe = &cc->cc_cqe;
 240         list_for_each(tmp, &cc->cc_rwctxts) {
 241                 struct svc_rdma_rw_ctxt *ctxt;
 242
 243                 ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list);
 244                 first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp,
 245                                            rdma->sc_port_num, cqe, first_wr);
 246                 cqe = NULL;
 247         }
 248
 249         do {
 250                 if (atomic_sub_return(cc->cc_sqecount,
 251                                       &rdma->sc_sq_avail) > 0) {
 252                         ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr);
 253                         if (ret)
 254                                 break;
 255                         return 0;
 256                 }
 257
 258                 atomic_inc(&rdma_stat_sq_starve);
 259                 atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
 260                 wait_event(rdma->sc_send_wait,
 261                            atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount);
 262         } while (1);
 263
 264         pr_err("svcrdma: ib_post_send failed (%d)\n", ret);
 265         set_bit(XPT_CLOSE, &xprt->xpt_flags);
 266
 267         /* If even one was posted, there will be a completion. */
 268         if (bad_wr != first_wr)
 269                 return 0;
 270
 271         atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
 272         wake_up(&rdma->sc_send_wait);
 273         return -ENOTCONN;
 274 }
 275
 276 /* Build and DMA-map an SGL that covers one kvec in an xdr_buf
 277  */
 278 static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info,
 279                                unsigned int len,
 280                                struct svc_rdma_rw_ctxt *ctxt)
 281 {
 282         struct scatterlist *sg = ctxt->rw_sg_table.sgl;
 283
 284         sg_set_buf(&sg[0], info->wi_base, len);
 285         info->wi_base += len;
 286
 287         ctxt->rw_nents = 1;
 288 }
 289
 290 /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist.
 291  */
 292 static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info,
 293                                     unsigned int remaining,
 294                                     struct svc_rdma_rw_ctxt *ctxt)
 295 {
 296         unsigned int sge_no, sge_bytes, page_off, page_no;
 297         struct xdr_buf *xdr = info->wi_xdr;
 298         struct scatterlist *sg;
 299         struct page **page;
 300
 301         page_off = (info->wi_next_off + xdr->page_base) & ~PAGE_MASK;
 302         page_no = (info->wi_next_off + xdr->page_base) >> PAGE_SHIFT;
 303         page = xdr->pages + page_no;
 304         info->wi_next_off += remaining;
 305         sg = ctxt->rw_sg_table.sgl;
 306         sge_no = 0;
 307         do {
 308                 sge_bytes = min_t(unsigned int, remaining,
 309                                   PAGE_SIZE - page_off);
 310                 sg_set_page(sg, *page, sge_bytes, page_off);
 311
 312                 remaining -= sge_bytes;
 313                 sg = sg_next(sg);
 314                 page_off = 0;
 315                 sge_no++;
 316                 page++;
 317         } while (remaining);
 318
 319         ctxt->rw_nents = sge_no;
 320 }
 321
 322 /* Construct RDMA Write WRs to send a portion of an xdr_buf containing
 323  * an RPC Reply.
 324  */
 325 static int
 326 svc_rdma_build_writes(struct svc_rdma_write_info *info,
 327                       void (*constructor)(struct svc_rdma_write_info *info,
 328                                           unsigned int len,
 329                                           struct svc_rdma_rw_ctxt *ctxt),
 330                       unsigned int remaining)
 331 {
 332         struct svc_rdma_chunk_ctxt *cc = &info->wi_cc;
 333         struct svcxprt_rdma *rdma = cc->cc_rdma;
 334         struct svc_rdma_rw_ctxt *ctxt;
 335         __be32 *seg;
 336         int ret;
 337
 338         cc->cc_cqe.done = svc_rdma_write_done;
 339         seg = info->wi_segs + info->wi_seg_no * rpcrdma_segment_maxsz;
 340         do {
 341                 unsigned int write_len;
 342                 u32 seg_length, seg_handle;
 343                 u64 seg_offset;
 344
 345                 if (info->wi_seg_no >= info->wi_nsegs)
 346                         goto out_overflow;
 347
 348                 seg_handle = be32_to_cpup(seg);
 349                 seg_length = be32_to_cpup(seg + 1);
 350                 xdr_decode_hyper(seg + 2, &seg_offset);
 351                 seg_offset += info->wi_seg_off;
 352
 353                 write_len = min(remaining, seg_length - info->wi_seg_off);
 354                 ctxt = svc_rdma_get_rw_ctxt(rdma,
 355                                             (write_len >> PAGE_SHIFT) + 2);
 356                 if (!ctxt)
 357                         goto out_noctx;
 358
 359                 constructor(info, write_len, ctxt);
 360                 ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp,
 361                                        rdma->sc_port_num, ctxt->rw_sg_table.sgl,
 362                                        ctxt->rw_nents, 0, seg_offset,
 363                                        seg_handle, DMA_TO_DEVICE);
 364                 if (ret < 0)
 365                         goto out_initerr;
 366
 367                 list_add(&ctxt->rw_list, &cc->cc_rwctxts);
 368                 cc->cc_sqecount += ret;
 369                 if (write_len == seg_length - info->wi_seg_off) {
 370                         seg += 4;
 371                         info->wi_seg_no++;
 372                         info->wi_seg_off = 0;
 373                 } else {
 374                         info->wi_seg_off += write_len;
 375                 }
 376                 remaining -= write_len;
 377         } while (remaining);
 378
 379         return 0;
 380
 381 out_overflow:
 382         dprintk("svcrdma: inadequate space in Write chunk (%u)\n",
 383                 info->wi_nsegs);
 384         return -E2BIG;
 385
 386 out_noctx:
 387         dprintk("svcrdma: no R/W ctxs available\n");
 388         return -ENOMEM;
 389
 390 out_initerr:
 391         svc_rdma_put_rw_ctxt(rdma, ctxt);
 392         pr_err("svcrdma: failed to map pagelist (%d)\n", ret);
 393         return -EIO;
 394 }
 395
 396 /* Send one of an xdr_buf's kvecs by itself. To send a Reply
 397  * chunk, the whole RPC Reply is written back to the client.
 398  * This function writes either the head or tail of the xdr_buf
 399  * containing the Reply.
 400  */
 401 static int svc_rdma_send_xdr_kvec(struct svc_rdma_write_info *info,
 402                                   struct kvec *vec)
 403 {
 404         info->wi_base = vec->iov_base;
 405         return svc_rdma_build_writes(info, svc_rdma_vec_to_sg,
 406                                      vec->iov_len);
 407 }
 408
 409 /* Send an xdr_buf's page list by itself. A Write chunk is
 410  * just the page list. a Reply chunk is the head, page list,
 411  * and tail. This function is shared between the two types
 412  * of chunk.
 413  */
 414 static int svc_rdma_send_xdr_pagelist(struct svc_rdma_write_info *info,
 415                                       struct xdr_buf *xdr)
 416 {
 417         info->wi_xdr = xdr;
 418         info->wi_next_off = 0;
 419         return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg,
 420                                      xdr->page_len);
 421 }
 422
 423 /**
 424  * svc_rdma_send_write_chunk - Write all segments in a Write chunk
 425  * @rdma: controlling RDMA transport
 426  * @wr_ch: Write chunk provided by client
 427  * @xdr: xdr_buf containing the data payload
 428  *
 429  * Returns a non-negative number of bytes the chunk consumed, or
 430  *      %-E2BIG if the payload was larger than the Write chunk,
 431  *      %-EINVAL if client provided too many segments,
 432  *      %-ENOMEM if rdma_rw context pool was exhausted,
 433  *      %-ENOTCONN if posting failed (connection is lost),
 434  *      %-EIO if rdma_rw initialization failed (DMA mapping, etc).
 435  */
 436 int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, __be32 *wr_ch,
 437                               struct xdr_buf *xdr)
 438 {
 439         struct svc_rdma_write_info *info;
 440         int ret;
 441
 442         if (!xdr->page_len)
 443                 return 0;
 444
 445         info = svc_rdma_write_info_alloc(rdma, wr_ch);
 446         if (!info)
 447                 return -ENOMEM;
 448
 449         ret = svc_rdma_send_xdr_pagelist(info, xdr);
 450         if (ret < 0)
 451                 goto out_err;
 452
 453         ret = svc_rdma_post_chunk_ctxt(&info->wi_cc);
 454         if (ret < 0)
 455                 goto out_err;
 456         return xdr->page_len;
 457
 458 out_err:
 459         svc_rdma_write_info_free(info);
 460         return ret;
 461 }
 462
 463 /**
 464  * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk
 465  * @rdma: controlling RDMA transport
 466  * @rp_ch: Reply chunk provided by client
 467  * @writelist: true if client provided a Write list
 468  * @xdr: xdr_buf containing an RPC Reply
 469  *
 470  * Returns a non-negative number of bytes the chunk consumed, or
 471  *      %-E2BIG if the payload was larger than the Reply chunk,
 472  *      %-EINVAL if client provided too many segments,
 473  *      %-ENOMEM if rdma_rw context pool was exhausted,
 474  *      %-ENOTCONN if posting failed (connection is lost),
 475  *      %-EIO if rdma_rw initialization failed (DMA mapping, etc).
 476  */
 477 int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, __be32 *rp_ch,
 478                               bool writelist, struct xdr_buf *xdr)
 479 {
 480         struct svc_rdma_write_info *info;
 481         int consumed, ret;
 482
 483         info = svc_rdma_write_info_alloc(rdma, rp_ch);
 484         if (!info)
 485                 return -ENOMEM;
 486
 487         ret = svc_rdma_send_xdr_kvec(info, &xdr->head[0]);
 488         if (ret < 0)
 489                 goto out_err;
 490         consumed = xdr->head[0].iov_len;
 491
 492         /* Send the page list in the Reply chunk only if the
 493          * client did not provide Write chunks.
 494          */
 495         if (!writelist && xdr->page_len) {
 496                 ret = svc_rdma_send_xdr_pagelist(info, xdr);
 497                 if (ret < 0)
 498                         goto out_err;
 499                 consumed += xdr->page_len;
 500         }
 501
 502         if (xdr->tail[0].iov_len) {
 503                 ret = svc_rdma_send_xdr_kvec(info, &xdr->tail[0]);
 504                 if (ret < 0)
 505                         goto out_err;
 506                 consumed += xdr->tail[0].iov_len;
 507         }
 508
 509         ret = svc_rdma_post_chunk_ctxt(&info->wi_cc);
 510         if (ret < 0)
 511                 goto out_err;
 512         return consumed;
 513
 514 out_err:
 515         svc_rdma_write_info_free(info);
 516         return ret;
 517 }