drivers/nvme/host/rdma.c

   1 /*
   2  * NVMe over Fabrics RDMA host code.
   3  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  15 #include <linux/module.h>
  16 #include <linux/init.h>
  17 #include <linux/slab.h>
  18 #include <linux/err.h>
  19 #include <linux/string.h>
  20 #include <linux/atomic.h>
  21 #include <linux/blk-mq.h>
  22 #include <linux/types.h>
  23 #include <linux/list.h>
  24 #include <linux/mutex.h>
  25 #include <linux/scatterlist.h>
  26 #include <linux/nvme.h>
  27 #include <asm/unaligned.h>
  28
  29 #include <rdma/ib_verbs.h>
  30 #include <rdma/rdma_cm.h>
  31 #include <linux/nvme-rdma.h>
  32
  33 #include "nvme.h"
  34 #include "fabrics.h"
  35
  36
  37 #define NVME_RDMA_CONNECT_TIMEOUT_MS    3000            /* 3 second */
  38
  39 #define NVME_RDMA_MAX_SEGMENT_SIZE      0xffffff        /* 24-bit SGL field */
  40
  41 #define NVME_RDMA_MAX_SEGMENTS          256
  42
  43 #define NVME_RDMA_MAX_INLINE_SEGMENTS   1
  44
  45 /*
  46  * We handle AEN commands ourselves and don't even let the
  47  * block layer know about them.
  48  */
  49 #define NVME_RDMA_NR_AEN_COMMANDS      1
  50 #define NVME_RDMA_AQ_BLKMQ_DEPTH       \
  51         (NVMF_AQ_DEPTH - NVME_RDMA_NR_AEN_COMMANDS)
  52
  53 struct nvme_rdma_device {
  54         struct ib_device       *dev;
  55         struct ib_pd           *pd;
  56         struct kref             ref;
  57         struct list_head        entry;
  58 };
  59
  60 struct nvme_rdma_qe {
  61         struct ib_cqe           cqe;
  62         void                    *data;
  63         u64                     dma;
  64 };
  65
  66 struct nvme_rdma_queue;
  67 struct nvme_rdma_request {
  68         struct nvme_request     req;
  69         struct ib_mr            *mr;
  70         struct nvme_rdma_qe     sqe;
  71         struct ib_sge           sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
  72         u32                     num_sge;
  73         int                     nents;
  74         bool                    inline_data;
  75         struct ib_reg_wr        reg_wr;
  76         struct ib_cqe           reg_cqe;
  77         struct nvme_rdma_queue  *queue;
  78         struct sg_table         sg_table;
  79         struct scatterlist      first_sgl[];
  80 };
  81
  82 enum nvme_rdma_queue_flags {
  83         NVME_RDMA_Q_LIVE                = 0,
  84         NVME_RDMA_Q_DELETING            = 1,
  85 };
  86
  87 struct nvme_rdma_queue {
  88         struct nvme_rdma_qe     *rsp_ring;
  89         u8                      sig_count;
  90         int                     queue_size;
  91         size_t                  cmnd_capsule_len;
  92         struct nvme_rdma_ctrl   *ctrl;
  93         struct nvme_rdma_device *device;
  94         struct ib_cq            *ib_cq;
  95         struct ib_qp            *qp;
  96
  97         unsigned long           flags;
  98         struct rdma_cm_id       *cm_id;
  99         int                     cm_error;
 100         struct completion       cm_done;
 101 };
 102
 103 struct nvme_rdma_ctrl {
 104         /* read only in the hot path */
 105         struct nvme_rdma_queue  *queues;
 106         u32                     queue_count;
 107
 108         /* other member variables */
 109         struct blk_mq_tag_set   tag_set;
 110         struct work_struct      delete_work;
 111         struct work_struct      reset_work;
 112         struct work_struct      err_work;
 113
 114         struct nvme_rdma_qe     async_event_sqe;
 115
 116         struct delayed_work     reconnect_work;
 117
 118         struct list_head        list;
 119
 120         struct blk_mq_tag_set   admin_tag_set;
 121         struct nvme_rdma_device *device;
 122
 123         u64                     cap;
 124         u32                     max_fr_pages;
 125
 126         struct sockaddr_storage addr;
 127         struct sockaddr_storage src_addr;
 128
 129         struct nvme_ctrl        ctrl;
 130 };
 131
 132 static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
 133 {
 134         return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
 135 }
 136
 137 static LIST_HEAD(device_list);
 138 static DEFINE_MUTEX(device_list_mutex);
 139
 140 static LIST_HEAD(nvme_rdma_ctrl_list);
 141 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
 142
 143 /*
 144  * Disabling this option makes small I/O goes faster, but is fundamentally
 145  * unsafe.  With it turned off we will have to register a global rkey that
 146  * allows read and write access to all physical memory.
 147  */
 148 static bool register_always = true;
 149 module_param(register_always, bool, 0444);
 150 MODULE_PARM_DESC(register_always,
 151          "Use memory registration even for contiguous memory regions");
 152
 153 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
 154                 struct rdma_cm_event *event);
 155 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 156
 157 /* XXX: really should move to a generic header sooner or later.. */
 158 static inline void put_unaligned_le24(u32 val, u8 *p)
 159 {
 160         *p++ = val;
 161         *p++ = val >> 8;
 162         *p++ = val >> 16;
 163 }
 164
 165 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
 166 {
 167         return queue - queue->ctrl->queues;
 168 }
 169
 170 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
 171 {
 172         return queue->cmnd_capsule_len - sizeof(struct nvme_command);
 173 }
 174
 175 static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
 176                 size_t capsule_size, enum dma_data_direction dir)
 177 {
 178         ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
 179         kfree(qe->data);
 180 }
 181
 182 static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
 183                 size_t capsule_size, enum dma_data_direction dir)
 184 {
 185         qe->data = kzalloc(capsule_size, GFP_KERNEL);
 186         if (!qe->data)
 187                 return -ENOMEM;
 188
 189         qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
 190         if (ib_dma_mapping_error(ibdev, qe->dma)) {
 191                 kfree(qe->data);
 192                 return -ENOMEM;
 193         }
 194
 195         return 0;
 196 }
 197
 198 static void nvme_rdma_free_ring(struct ib_device *ibdev,
 199                 struct nvme_rdma_qe *ring, size_t ib_queue_size,
 200                 size_t capsule_size, enum dma_data_direction dir)
 201 {
 202         int i;
 203
 204         for (i = 0; i < ib_queue_size; i++)
 205                 nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
 206         kfree(ring);
 207 }
 208
 209 static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
 210                 size_t ib_queue_size, size_t capsule_size,
 211                 enum dma_data_direction dir)
 212 {
 213         struct nvme_rdma_qe *ring;
 214         int i;
 215
 216         ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
 217         if (!ring)
 218                 return NULL;
 219
 220         for (i = 0; i < ib_queue_size; i++) {
 221                 if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
 222                         goto out_free_ring;
 223         }
 224
 225         return ring;
 226
 227 out_free_ring:
 228         nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
 229         return NULL;
 230 }
 231
 232 static void nvme_rdma_qp_event(struct ib_event *event, void *context)
 233 {
 234         pr_debug("QP event %s (%d)\n",
 235                  ib_event_msg(event->event), event->event);
 236
 237 }
 238
 239 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
 240 {
 241         wait_for_completion_interruptible_timeout(&queue->cm_done,
 242                         msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
 243         return queue->cm_error;
 244 }
 245
 246 static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
 247 {
 248         struct nvme_rdma_device *dev = queue->device;
 249         struct ib_qp_init_attr init_attr;
 250         int ret;
 251
 252         memset(&init_attr, 0, sizeof(init_attr));
 253         init_attr.event_handler = nvme_rdma_qp_event;
 254         /* +1 for drain */
 255         init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
 256         /* +1 for drain */
 257         init_attr.cap.max_recv_wr = queue->queue_size + 1;
 258         init_attr.cap.max_recv_sge = 1;
 259         init_attr.cap.max_send_sge = 1 + NVME_RDMA_MAX_INLINE_SEGMENTS;
 260         init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 261         init_attr.qp_type = IB_QPT_RC;
 262         init_attr.send_cq = queue->ib_cq;
 263         init_attr.recv_cq = queue->ib_cq;
 264
 265         ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
 266
 267         queue->qp = queue->cm_id->qp;
 268         return ret;
 269 }
 270
 271 static int nvme_rdma_reinit_request(void *data, struct request *rq)
 272 {
 273         struct nvme_rdma_ctrl *ctrl = data;
 274         struct nvme_rdma_device *dev = ctrl->device;
 275         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 276         int ret = 0;
 277
 278         if (!req->mr->need_inval)
 279                 goto out;
 280
 281         ib_dereg_mr(req->mr);
 282
 283         req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
 284                         ctrl->max_fr_pages);
 285         if (IS_ERR(req->mr)) {
 286                 ret = PTR_ERR(req->mr);
 287                 req->mr = NULL;
 288                 goto out;
 289         }
 290
 291         req->mr->need_inval = false;
 292
 293 out:
 294         return ret;
 295 }
 296
 297 static void nvme_rdma_exit_request(struct blk_mq_tag_set *set,
 298                 struct request *rq, unsigned int hctx_idx)
 299 {
 300         struct nvme_rdma_ctrl *ctrl = set->driver_data;
 301         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 302         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 303         struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
 304         struct nvme_rdma_device *dev = queue->device;
 305
 306         if (req->mr)
 307                 ib_dereg_mr(req->mr);
 308
 309         nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
 310                         DMA_TO_DEVICE);
 311 }
 312
 313 static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
 314                 struct request *rq, unsigned int hctx_idx,
 315                 unsigned int numa_node)
 316 {
 317         struct nvme_rdma_ctrl *ctrl = set->driver_data;
 318         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 319         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 320         struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
 321         struct nvme_rdma_device *dev = queue->device;
 322         struct ib_device *ibdev = dev->dev;
 323         int ret;
 324
 325         ret = nvme_rdma_alloc_qe(ibdev, &req->sqe, sizeof(struct nvme_command),
 326                         DMA_TO_DEVICE);
 327         if (ret)
 328                 return ret;
 329
 330         req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
 331                         ctrl->max_fr_pages);
 332         if (IS_ERR(req->mr)) {
 333                 ret = PTR_ERR(req->mr);
 334                 goto out_free_qe;
 335         }
 336
 337         req->queue = queue;
 338
 339         return 0;
 340
 341 out_free_qe:
 342         nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
 343                         DMA_TO_DEVICE);
 344         return -ENOMEM;
 345 }
 346
 347 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 348                 unsigned int hctx_idx)
 349 {
 350         struct nvme_rdma_ctrl *ctrl = data;
 351         struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
 352
 353         BUG_ON(hctx_idx >= ctrl->queue_count);
 354
 355         hctx->driver_data = queue;
 356         return 0;
 357 }
 358
 359 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 360                 unsigned int hctx_idx)
 361 {
 362         struct nvme_rdma_ctrl *ctrl = data;
 363         struct nvme_rdma_queue *queue = &ctrl->queues[0];
 364
 365         BUG_ON(hctx_idx != 0);
 366
 367         hctx->driver_data = queue;
 368         return 0;
 369 }
 370
 371 static void nvme_rdma_free_dev(struct kref *ref)
 372 {
 373         struct nvme_rdma_device *ndev =
 374                 container_of(ref, struct nvme_rdma_device, ref);
 375
 376         mutex_lock(&device_list_mutex);
 377         list_del(&ndev->entry);
 378         mutex_unlock(&device_list_mutex);
 379
 380         ib_dealloc_pd(ndev->pd);
 381         kfree(ndev);
 382 }
 383
 384 static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
 385 {
 386         kref_put(&dev->ref, nvme_rdma_free_dev);
 387 }
 388
 389 static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
 390 {
 391         return kref_get_unless_zero(&dev->ref);
 392 }
 393
 394 static struct nvme_rdma_device *
 395 nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
 396 {
 397         struct nvme_rdma_device *ndev;
 398
 399         mutex_lock(&device_list_mutex);
 400         list_for_each_entry(ndev, &device_list, entry) {
 401                 if (ndev->dev->node_guid == cm_id->device->node_guid &&
 402                     nvme_rdma_dev_get(ndev))
 403                         goto out_unlock;
 404         }
 405
 406         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 407         if (!ndev)
 408                 goto out_err;
 409
 410         ndev->dev = cm_id->device;
 411         kref_init(&ndev->ref);
 412
 413         ndev->pd = ib_alloc_pd(ndev->dev,
 414                 register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
 415         if (IS_ERR(ndev->pd))
 416                 goto out_free_dev;
 417
 418         if (!(ndev->dev->attrs.device_cap_flags &
 419               IB_DEVICE_MEM_MGT_EXTENSIONS)) {
 420                 dev_err(&ndev->dev->dev,
 421                         "Memory registrations not supported.\n");
 422                 goto out_free_pd;
 423         }
 424
 425         list_add(&ndev->entry, &device_list);
 426 out_unlock:
 427         mutex_unlock(&device_list_mutex);
 428         return ndev;
 429
 430 out_free_pd:
 431         ib_dealloc_pd(ndev->pd);
 432 out_free_dev:
 433         kfree(ndev);
 434 out_err:
 435         mutex_unlock(&device_list_mutex);
 436         return NULL;
 437 }
 438
 439 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
 440 {
 441         struct nvme_rdma_device *dev;
 442         struct ib_device *ibdev;
 443
 444         dev = queue->device;
 445         ibdev = dev->dev;
 446         rdma_destroy_qp(queue->cm_id);
 447         ib_free_cq(queue->ib_cq);
 448
 449         nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
 450                         sizeof(struct nvme_completion), DMA_FROM_DEVICE);
 451
 452         nvme_rdma_dev_put(dev);
 453 }
 454
 455 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
 456 {
 457         struct ib_device *ibdev;
 458         const int send_wr_factor = 3;                   /* MR, SEND, INV */
 459         const int cq_factor = send_wr_factor + 1;       /* + RECV */
 460         int comp_vector, idx = nvme_rdma_queue_idx(queue);
 461         int ret;
 462
 463         queue->device = nvme_rdma_find_get_device(queue->cm_id);
 464         if (!queue->device) {
 465                 dev_err(queue->cm_id->device->dev.parent,
 466                         "no client data found!\n");
 467                 return -ECONNREFUSED;
 468         }
 469         ibdev = queue->device->dev;
 470
 471         /*
 472          * The admin queue is barely used once the controller is live, so don't
 473          * bother to spread it out.
 474          */
 475         if (idx == 0)
 476                 comp_vector = 0;
 477         else
 478                 comp_vector = idx % ibdev->num_comp_vectors;
 479
 480
 481         /* +1 for ib_stop_cq */
 482         queue->ib_cq = ib_alloc_cq(ibdev, queue,
 483                                 cq_factor * queue->queue_size + 1,
 484                                 comp_vector, IB_POLL_SOFTIRQ);
 485         if (IS_ERR(queue->ib_cq)) {
 486                 ret = PTR_ERR(queue->ib_cq);
 487                 goto out_put_dev;
 488         }
 489
 490         ret = nvme_rdma_create_qp(queue, send_wr_factor);
 491         if (ret)
 492                 goto out_destroy_ib_cq;
 493
 494         queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
 495                         sizeof(struct nvme_completion), DMA_FROM_DEVICE);
 496         if (!queue->rsp_ring) {
 497                 ret = -ENOMEM;
 498                 goto out_destroy_qp;
 499         }
 500
 501         return 0;
 502
 503 out_destroy_qp:
 504         ib_destroy_qp(queue->qp);
 505 out_destroy_ib_cq:
 506         ib_free_cq(queue->ib_cq);
 507 out_put_dev:
 508         nvme_rdma_dev_put(queue->device);
 509         return ret;
 510 }
 511
 512 static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
 513                 int idx, size_t queue_size)
 514 {
 515         struct nvme_rdma_queue *queue;
 516         struct sockaddr *src_addr = NULL;
 517         int ret;
 518
 519         queue = &ctrl->queues[idx];
 520         queue->ctrl = ctrl;
 521         init_completion(&queue->cm_done);
 522
 523         if (idx > 0)
 524                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
 525         else
 526                 queue->cmnd_capsule_len = sizeof(struct nvme_command);
 527
 528         queue->queue_size = queue_size;
 529
 530         queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
 531                         RDMA_PS_TCP, IB_QPT_RC);
 532         if (IS_ERR(queue->cm_id)) {
 533                 dev_info(ctrl->ctrl.device,
 534                         "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
 535                 return PTR_ERR(queue->cm_id);
 536         }
 537
 538         if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
 539                 src_addr = (struct sockaddr *)&ctrl->src_addr;
 540
 541         queue->cm_error = -ETIMEDOUT;
 542         ret = rdma_resolve_addr(queue->cm_id, src_addr,
 543                         (struct sockaddr *)&ctrl->addr,
 544                         NVME_RDMA_CONNECT_TIMEOUT_MS);
 545         if (ret) {
 546                 dev_info(ctrl->ctrl.device,
 547                         "rdma_resolve_addr failed (%d).\n", ret);
 548                 goto out_destroy_cm_id;
 549         }
 550
 551         ret = nvme_rdma_wait_for_cm(queue);
 552         if (ret) {
 553                 dev_info(ctrl->ctrl.device,
 554                         "rdma_resolve_addr wait failed (%d).\n", ret);
 555                 goto out_destroy_cm_id;
 556         }
 557
 558         clear_bit(NVME_RDMA_Q_DELETING, &queue->flags);
 559
 560         return 0;
 561
 562 out_destroy_cm_id:
 563         rdma_destroy_id(queue->cm_id);
 564         return ret;
 565 }
 566
 567 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 568 {
 569         rdma_disconnect(queue->cm_id);
 570         ib_drain_qp(queue->qp);
 571 }
 572
 573 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
 574 {
 575         nvme_rdma_destroy_queue_ib(queue);
 576         rdma_destroy_id(queue->cm_id);
 577 }
 578
 579 static void nvme_rdma_stop_and_free_queue(struct nvme_rdma_queue *queue)
 580 {
 581         if (test_and_set_bit(NVME_RDMA_Q_DELETING, &queue->flags))
 582                 return;
 583         nvme_rdma_stop_queue(queue);
 584         nvme_rdma_free_queue(queue);
 585 }
 586
 587 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
 588 {
 589         int i;
 590
 591         for (i = 1; i < ctrl->queue_count; i++)
 592                 nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
 593 }
 594
 595 static int nvme_rdma_connect_io_queues(struct nvme_rdma_ctrl *ctrl)
 596 {
 597         int i, ret = 0;
 598
 599         for (i = 1; i < ctrl->queue_count; i++) {
 600                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
 601                 if (ret) {
 602                         dev_info(ctrl->ctrl.device,
 603                                 "failed to connect i/o queue: %d\n", ret);
 604                         goto out_free_queues;
 605                 }
 606                 set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
 607         }
 608
 609         return 0;
 610
 611 out_free_queues:
 612         nvme_rdma_free_io_queues(ctrl);
 613         return ret;
 614 }
 615
 616 static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
 617 {
 618         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
 619         unsigned int nr_io_queues;
 620         int i, ret;
 621
 622         nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
 623         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
 624         if (ret)
 625                 return ret;
 626
 627         ctrl->queue_count = nr_io_queues + 1;
 628         if (ctrl->queue_count < 2)
 629                 return 0;
 630
 631         dev_info(ctrl->ctrl.device,
 632                 "creating %d I/O queues.\n", nr_io_queues);
 633
 634         for (i = 1; i < ctrl->queue_count; i++) {
 635                 ret = nvme_rdma_init_queue(ctrl, i,
 636                                            ctrl->ctrl.opts->queue_size);
 637                 if (ret) {
 638                         dev_info(ctrl->ctrl.device,
 639                                 "failed to initialize i/o queue: %d\n", ret);
 640                         goto out_free_queues;
 641                 }
 642         }
 643
 644         return 0;
 645
 646 out_free_queues:
 647         for (i--; i >= 1; i--)
 648                 nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
 649
 650         return ret;
 651 }
 652
 653 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
 654 {
 655         nvme_rdma_free_qe(ctrl->queues[0].device->dev, &ctrl->async_event_sqe,
 656                         sizeof(struct nvme_command), DMA_TO_DEVICE);
 657         nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
 658         blk_cleanup_queue(ctrl->ctrl.admin_q);
 659         blk_mq_free_tag_set(&ctrl->admin_tag_set);
 660         nvme_rdma_dev_put(ctrl->device);
 661 }
 662
 663 static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
 664 {
 665         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
 666
 667         if (list_empty(&ctrl->list))
 668                 goto free_ctrl;
 669
 670         mutex_lock(&nvme_rdma_ctrl_mutex);
 671         list_del(&ctrl->list);
 672         mutex_unlock(&nvme_rdma_ctrl_mutex);
 673
 674         kfree(ctrl->queues);
 675         nvmf_free_options(nctrl->opts);
 676 free_ctrl:
 677         kfree(ctrl);
 678 }
 679
 680 static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
 681 {
 682         /* If we are resetting/deleting then do nothing */
 683         if (ctrl->ctrl.state != NVME_CTRL_RECONNECTING) {
 684                 WARN_ON_ONCE(ctrl->ctrl.state == NVME_CTRL_NEW ||
 685                         ctrl->ctrl.state == NVME_CTRL_LIVE);
 686                 return;
 687         }
 688
 689         if (nvmf_should_reconnect(&ctrl->ctrl)) {
 690                 dev_info(ctrl->ctrl.device, "Reconnecting in %d seconds...\n",
 691                         ctrl->ctrl.opts->reconnect_delay);
 692                 queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
 693                                 ctrl->ctrl.opts->reconnect_delay * HZ);
 694         } else {
 695                 dev_info(ctrl->ctrl.device, "Removing controller...\n");
 696                 queue_work(nvme_wq, &ctrl->delete_work);
 697         }
 698 }
 699
 700 static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 701 {
 702         struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
 703                         struct nvme_rdma_ctrl, reconnect_work);
 704         bool changed;
 705         int ret;
 706
 707         ++ctrl->ctrl.nr_reconnects;
 708
 709         if (ctrl->queue_count > 1) {
 710                 nvme_rdma_free_io_queues(ctrl);
 711
 712                 ret = blk_mq_reinit_tagset(&ctrl->tag_set);
 713                 if (ret)
 714                         goto requeue;
 715         }
 716
 717         nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
 718
 719         ret = blk_mq_reinit_tagset(&ctrl->admin_tag_set);
 720         if (ret)
 721                 goto requeue;
 722
 723         ret = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
 724         if (ret)
 725                 goto requeue;
 726
 727         ret = nvmf_connect_admin_queue(&ctrl->ctrl);
 728         if (ret)
 729                 goto requeue;
 730
 731         set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
 732
 733         ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
 734         if (ret)
 735                 goto requeue;
 736
 737         nvme_start_keep_alive(&ctrl->ctrl);
 738
 739         if (ctrl->queue_count > 1) {
 740                 ret = nvme_rdma_init_io_queues(ctrl);
 741                 if (ret)
 742                         goto requeue;
 743
 744                 ret = nvme_rdma_connect_io_queues(ctrl);
 745                 if (ret)
 746                         goto requeue;
 747         }
 748
 749         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
 750         WARN_ON_ONCE(!changed);
 751         ctrl->ctrl.nr_reconnects = 0;
 752
 753         if (ctrl->queue_count > 1) {
 754                 nvme_queue_scan(&ctrl->ctrl);
 755                 nvme_queue_async_events(&ctrl->ctrl);
 756         }
 757
 758         dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
 759
 760         return;
 761
 762 requeue:
 763         dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
 764                         ctrl->ctrl.nr_reconnects);
 765         nvme_rdma_reconnect_or_remove(ctrl);
 766 }
 767
 768 static void nvme_rdma_error_recovery_work(struct work_struct *work)
 769 {
 770         struct nvme_rdma_ctrl *ctrl = container_of(work,
 771                         struct nvme_rdma_ctrl, err_work);
 772         int i;
 773
 774         nvme_stop_keep_alive(&ctrl->ctrl);
 775
 776         for (i = 0; i < ctrl->queue_count; i++)
 777                 clear_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
 778
 779         if (ctrl->queue_count > 1)
 780                 nvme_stop_queues(&ctrl->ctrl);
 781         blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
 782
 783         /* We must take care of fastfail/requeue all our inflight requests */
 784         if (ctrl->queue_count > 1)
 785                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
 786                                         nvme_cancel_request, &ctrl->ctrl);
 787         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
 788                                 nvme_cancel_request, &ctrl->ctrl);
 789
 790         /*
 791          * queues are not a live anymore, so restart the queues to fail fast
 792          * new IO
 793          */
 794         blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true);
 795         nvme_start_queues(&ctrl->ctrl);
 796
 797         nvme_rdma_reconnect_or_remove(ctrl);
 798 }
 799
 800 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
 801 {
 802         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RECONNECTING))
 803                 return;
 804
 805         queue_work(nvme_wq, &ctrl->err_work);
 806 }
 807
 808 static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
 809                 const char *op)
 810 {
 811         struct nvme_rdma_queue *queue = cq->cq_context;
 812         struct nvme_rdma_ctrl *ctrl = queue->ctrl;
 813
 814         if (ctrl->ctrl.state == NVME_CTRL_LIVE)
 815                 dev_info(ctrl->ctrl.device,
 816                              "%s for CQE 0x%p failed with status %s (%d)\n",
 817                              op, wc->wr_cqe,
 818                              ib_wc_status_msg(wc->status), wc->status);
 819         nvme_rdma_error_recovery(ctrl);
 820 }
 821
 822 static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
 823 {
 824         if (unlikely(wc->status != IB_WC_SUCCESS))
 825                 nvme_rdma_wr_error(cq, wc, "MEMREG");
 826 }
 827
 828 static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
 829 {
 830         if (unlikely(wc->status != IB_WC_SUCCESS))
 831                 nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
 832 }
 833
 834 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
 835                 struct nvme_rdma_request *req)
 836 {
 837         struct ib_send_wr *bad_wr;
 838         struct ib_send_wr wr = {
 839                 .opcode             = IB_WR_LOCAL_INV,
 840                 .next               = NULL,
 841                 .num_sge            = 0,
 842                 .send_flags         = 0,
 843                 .ex.invalidate_rkey = req->mr->rkey,
 844         };
 845
 846         req->reg_cqe.done = nvme_rdma_inv_rkey_done;
 847         wr.wr_cqe = &req->reg_cqe;
 848
 849         return ib_post_send(queue->qp, &wr, &bad_wr);
 850 }
 851
 852 static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
 853                 struct request *rq)
 854 {
 855         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 856         struct nvme_rdma_ctrl *ctrl = queue->ctrl;
 857         struct nvme_rdma_device *dev = queue->device;
 858         struct ib_device *ibdev = dev->dev;
 859         int res;
 860
 861         if (!blk_rq_bytes(rq))
 862                 return;
 863
 864         if (req->mr->need_inval) {
 865                 res = nvme_rdma_inv_rkey(queue, req);
 866                 if (res < 0) {
 867                         dev_err(ctrl->ctrl.device,
 868                                 "Queueing INV WR for rkey %#x failed (%d)\n",
 869                                 req->mr->rkey, res);
 870                         nvme_rdma_error_recovery(queue->ctrl);
 871                 }
 872         }
 873
 874         ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
 875                         req->nents, rq_data_dir(rq) ==
 876                                     WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 877
 878         nvme_cleanup_cmd(rq);
 879         sg_free_table_chained(&req->sg_table, true);
 880 }
 881
 882 static int nvme_rdma_set_sg_null(struct nvme_command *c)
 883 {
 884         struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 885
 886         sg->addr = 0;
 887         put_unaligned_le24(0, sg->length);
 888         put_unaligned_le32(0, sg->key);
 889         sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
 890         return 0;
 891 }
 892
 893 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
 894                 struct nvme_rdma_request *req, struct nvme_command *c)
 895 {
 896         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
 897
 898         req->sge[1].addr = sg_dma_address(req->sg_table.sgl);
 899         req->sge[1].length = sg_dma_len(req->sg_table.sgl);
 900         req->sge[1].lkey = queue->device->pd->local_dma_lkey;
 901
 902         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
 903         sg->length = cpu_to_le32(sg_dma_len(req->sg_table.sgl));
 904         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
 905
 906         req->inline_data = true;
 907         req->num_sge++;
 908         return 0;
 909 }
 910
 911 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
 912                 struct nvme_rdma_request *req, struct nvme_command *c)
 913 {
 914         struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 915
 916         sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
 917         put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
 918         put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
 919         sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
 920         return 0;
 921 }
 922
 923 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
 924                 struct nvme_rdma_request *req, struct nvme_command *c,
 925                 int count)
 926 {
 927         struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 928         int nr;
 929
 930         nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, PAGE_SIZE);
 931         if (nr < count) {
 932                 if (nr < 0)
 933                         return nr;
 934                 return -EINVAL;
 935         }
 936
 937         ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
 938
 939         req->reg_cqe.done = nvme_rdma_memreg_done;
 940         memset(&req->reg_wr, 0, sizeof(req->reg_wr));
 941         req->reg_wr.wr.opcode = IB_WR_REG_MR;
 942         req->reg_wr.wr.wr_cqe = &req->reg_cqe;
 943         req->reg_wr.wr.num_sge = 0;
 944         req->reg_wr.mr = req->mr;
 945         req->reg_wr.key = req->mr->rkey;
 946         req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
 947                              IB_ACCESS_REMOTE_READ |
 948                              IB_ACCESS_REMOTE_WRITE;
 949
 950         req->mr->need_inval = true;
 951
 952         sg->addr = cpu_to_le64(req->mr->iova);
 953         put_unaligned_le24(req->mr->length, sg->length);
 954         put_unaligned_le32(req->mr->rkey, sg->key);
 955         sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
 956                         NVME_SGL_FMT_INVALIDATE;
 957
 958         return 0;
 959 }
 960
 961 static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
 962                 struct request *rq, struct nvme_command *c)
 963 {
 964         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 965         struct nvme_rdma_device *dev = queue->device;
 966         struct ib_device *ibdev = dev->dev;
 967         int count, ret;
 968
 969         req->num_sge = 1;
 970         req->inline_data = false;
 971         req->mr->need_inval = false;
 972
 973         c->common.flags |= NVME_CMD_SGL_METABUF;
 974
 975         if (!blk_rq_bytes(rq))
 976                 return nvme_rdma_set_sg_null(c);
 977
 978         req->sg_table.sgl = req->first_sgl;
 979         ret = sg_alloc_table_chained(&req->sg_table,
 980                         blk_rq_nr_phys_segments(rq), req->sg_table.sgl);
 981         if (ret)
 982                 return -ENOMEM;
 983
 984         req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
 985
 986         count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents,
 987                     rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 988         if (unlikely(count <= 0)) {
 989                 sg_free_table_chained(&req->sg_table, true);
 990                 return -EIO;
 991         }
 992
 993         if (count == 1) {
 994                 if (rq_data_dir(rq) == WRITE && nvme_rdma_queue_idx(queue) &&
 995                     blk_rq_payload_bytes(rq) <=
 996                                 nvme_rdma_inline_data_size(queue))
 997                         return nvme_rdma_map_sg_inline(queue, req, c);
 998
 999                 if (dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)
1000                         return nvme_rdma_map_sg_single(queue, req, c);
1001         }
1002
1003         return nvme_rdma_map_sg_fr(queue, req, c, count);
1004 }
1005
1006 static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1007 {
1008         if (unlikely(wc->status != IB_WC_SUCCESS))
1009                 nvme_rdma_wr_error(cq, wc, "SEND");
1010 }
1011
1012 static inline int nvme_rdma_queue_sig_limit(struct nvme_rdma_queue *queue)
1013 {
1014         int sig_limit;
1015
1016         /*
1017          * We signal completion every queue depth/2 and also handle the
1018          * degenerated case of a  device with queue_depth=1, where we
1019          * would need to signal every message.
1020          */
1021         sig_limit = max(queue->queue_size / 2, 1);
1022         return (++queue->sig_count % sig_limit) == 0;
1023 }
1024
1025 static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1026                 struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
1027                 struct ib_send_wr *first, bool flush)
1028 {
1029         struct ib_send_wr wr, *bad_wr;
1030         int ret;
1031
1032         sge->addr   = qe->dma;
1033         sge->length = sizeof(struct nvme_command),
1034         sge->lkey   = queue->device->pd->local_dma_lkey;
1035
1036         qe->cqe.done = nvme_rdma_send_done;
1037
1038         wr.next       = NULL;
1039         wr.wr_cqe     = &qe->cqe;
1040         wr.sg_list    = sge;
1041         wr.num_sge    = num_sge;
1042         wr.opcode     = IB_WR_SEND;
1043         wr.send_flags = 0;
1044
1045         /*
1046          * Unsignalled send completions are another giant desaster in the
1047          * IB Verbs spec:  If we don't regularly post signalled sends
1048          * the send queue will fill up and only a QP reset will rescue us.
1049          * Would have been way to obvious to handle this in hardware or
1050          * at least the RDMA stack..
1051          *
1052          * Always signal the flushes. The magic request used for the flush
1053          * sequencer is not allocated in our driver's tagset and it's
1054          * triggered to be freed by blk_cleanup_queue(). So we need to
1055          * always mark it as signaled to ensure that the "wr_cqe", which is
1056          * embedded in request's payload, is not freed when __ib_process_cq()
1057          * calls wr_cqe->done().
1058          */
1059         if (nvme_rdma_queue_sig_limit(queue) || flush)
1060                 wr.send_flags |= IB_SEND_SIGNALED;
1061
1062         if (first)
1063                 first->next = &wr;
1064         else
1065                 first = &wr;
1066
1067         ret = ib_post_send(queue->qp, first, &bad_wr);
1068         if (ret) {
1069                 dev_err(queue->ctrl->ctrl.device,
1070                              "%s failed with error code %d\n", __func__, ret);
1071         }
1072         return ret;
1073 }
1074
1075 static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1076                 struct nvme_rdma_qe *qe)
1077 {
1078         struct ib_recv_wr wr, *bad_wr;
1079         struct ib_sge list;
1080         int ret;
1081
1082         list.addr   = qe->dma;
1083         list.length = sizeof(struct nvme_completion);
1084         list.lkey   = queue->device->pd->local_dma_lkey;
1085
1086         qe->cqe.done = nvme_rdma_recv_done;
1087
1088         wr.next     = NULL;
1089         wr.wr_cqe   = &qe->cqe;
1090         wr.sg_list  = &list;
1091         wr.num_sge  = 1;
1092
1093         ret = ib_post_recv(queue->qp, &wr, &bad_wr);
1094         if (ret) {
1095                 dev_err(queue->ctrl->ctrl.device,
1096                         "%s failed with error code %d\n", __func__, ret);
1097         }
1098         return ret;
1099 }
1100
1101 static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1102 {
1103         u32 queue_idx = nvme_rdma_queue_idx(queue);
1104
1105         if (queue_idx == 0)
1106                 return queue->ctrl->admin_tag_set.tags[queue_idx];
1107         return queue->ctrl->tag_set.tags[queue_idx - 1];
1108 }
1109
1110 static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
1111 {
1112         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1113         struct nvme_rdma_queue *queue = &ctrl->queues[0];
1114         struct ib_device *dev = queue->device->dev;
1115         struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1116         struct nvme_command *cmd = sqe->data;
1117         struct ib_sge sge;
1118         int ret;
1119
1120         if (WARN_ON_ONCE(aer_idx != 0))
1121                 return;
1122
1123         ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1124
1125         memset(cmd, 0, sizeof(*cmd));
1126         cmd->common.opcode = nvme_admin_async_event;
1127         cmd->common.command_id = NVME_RDMA_AQ_BLKMQ_DEPTH;
1128         cmd->common.flags |= NVME_CMD_SGL_METABUF;
1129         nvme_rdma_set_sg_null(cmd);
1130
1131         ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1132                         DMA_TO_DEVICE);
1133
1134         ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL, false);
1135         WARN_ON_ONCE(ret);
1136 }
1137
1138 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1139                 struct nvme_completion *cqe, struct ib_wc *wc, int tag)
1140 {
1141         struct request *rq;
1142         struct nvme_rdma_request *req;
1143         int ret = 0;
1144
1145         rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
1146         if (!rq) {
1147                 dev_err(queue->ctrl->ctrl.device,
1148                         "tag 0x%x on QP %#x not found\n",
1149                         cqe->command_id, queue->qp->qp_num);
1150                 nvme_rdma_error_recovery(queue->ctrl);
1151                 return ret;
1152         }
1153         req = blk_mq_rq_to_pdu(rq);
1154
1155         if (rq->tag == tag)
1156                 ret = 1;
1157
1158         if ((wc->wc_flags & IB_WC_WITH_INVALIDATE) &&
1159             wc->ex.invalidate_rkey == req->mr->rkey)
1160                 req->mr->need_inval = false;
1161
1162         nvme_end_request(rq, cqe->status, cqe->result);
1163         return ret;
1164 }
1165
1166 static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
1167 {
1168         struct nvme_rdma_qe *qe =
1169                 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1170         struct nvme_rdma_queue *queue = cq->cq_context;
1171         struct ib_device *ibdev = queue->device->dev;
1172         struct nvme_completion *cqe = qe->data;
1173         const size_t len = sizeof(struct nvme_completion);
1174         int ret = 0;
1175
1176         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1177                 nvme_rdma_wr_error(cq, wc, "RECV");
1178                 return 0;
1179         }
1180
1181         ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1182         /*
1183          * AEN requests are special as they don't time out and can
1184          * survive any kind of queue freeze and often don't respond to
1185          * aborts.  We don't even bother to allocate a struct request
1186          * for them but rather special case them here.
1187          */
1188         if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1189                         cqe->command_id >= NVME_RDMA_AQ_BLKMQ_DEPTH))
1190                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1191                                 &cqe->result);
1192         else
1193                 ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
1194         ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1195
1196         nvme_rdma_post_recv(queue, qe);
1197         return ret;
1198 }
1199
1200 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1201 {
1202         __nvme_rdma_recv_done(cq, wc, -1);
1203 }
1204
1205 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1206 {
1207         int ret, i;
1208
1209         for (i = 0; i < queue->queue_size; i++) {
1210                 ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1211                 if (ret)
1212                         goto out_destroy_queue_ib;
1213         }
1214
1215         return 0;
1216
1217 out_destroy_queue_ib:
1218         nvme_rdma_destroy_queue_ib(queue);
1219         return ret;
1220 }
1221
1222 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1223                 struct rdma_cm_event *ev)
1224 {
1225         struct rdma_cm_id *cm_id = queue->cm_id;
1226         int status = ev->status;
1227         const char *rej_msg;
1228         const struct nvme_rdma_cm_rej *rej_data;
1229         u8 rej_data_len;
1230
1231         rej_msg = rdma_reject_msg(cm_id, status);
1232         rej_data = rdma_consumer_reject_data(cm_id, ev, &rej_data_len);
1233
1234         if (rej_data && rej_data_len >= sizeof(u16)) {
1235                 u16 sts = le16_to_cpu(rej_data->sts);
1236
1237                 dev_err(queue->ctrl->ctrl.device,
1238                       "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1239                       status, rej_msg, sts, nvme_rdma_cm_msg(sts));
1240         } else {
1241                 dev_err(queue->ctrl->ctrl.device,
1242                         "Connect rejected: status %d (%s).\n", status, rej_msg);
1243         }
1244
1245         return -ECONNRESET;
1246 }
1247
1248 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1249 {
1250         int ret;
1251
1252         ret = nvme_rdma_create_queue_ib(queue);
1253         if (ret)
1254                 return ret;
1255
1256         ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1257         if (ret) {
1258                 dev_err(queue->ctrl->ctrl.device,
1259                         "rdma_resolve_route failed (%d).\n",
1260                         queue->cm_error);
1261                 goto out_destroy_queue;
1262         }
1263
1264         return 0;
1265
1266 out_destroy_queue:
1267         nvme_rdma_destroy_queue_ib(queue);
1268         return ret;
1269 }
1270
1271 static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1272 {
1273         struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1274         struct rdma_conn_param param = { };
1275         struct nvme_rdma_cm_req priv = { };
1276         int ret;
1277
1278         param.qp_num = queue->qp->qp_num;
1279         param.flow_control = 1;
1280
1281         param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
1282         /* maximum retry count */
1283         param.retry_count = 7;
1284         param.rnr_retry_count = 7;
1285         param.private_data = &priv;
1286         param.private_data_len = sizeof(priv);
1287
1288         priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1289         priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
1290         /*
1291          * set the admin queue depth to the minimum size
1292          * specified by the Fabrics standard.
1293          */
1294         if (priv.qid == 0) {
1295                 priv.hrqsize = cpu_to_le16(NVMF_AQ_DEPTH);
1296                 priv.hsqsize = cpu_to_le16(NVMF_AQ_DEPTH - 1);
1297         } else {
1298                 /*
1299                  * current interpretation of the fabrics spec
1300                  * is at minimum you make hrqsize sqsize+1, or a
1301                  * 1's based representation of sqsize.
1302                  */
1303                 priv.hrqsize = cpu_to_le16(queue->queue_size);
1304                 priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
1305         }
1306
1307         ret = rdma_connect(queue->cm_id, &param);
1308         if (ret) {
1309                 dev_err(ctrl->ctrl.device,
1310                         "rdma_connect failed (%d).\n", ret);
1311                 goto out_destroy_queue_ib;
1312         }
1313
1314         return 0;
1315
1316 out_destroy_queue_ib:
1317         nvme_rdma_destroy_queue_ib(queue);
1318         return ret;
1319 }
1320
1321 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1322                 struct rdma_cm_event *ev)
1323 {
1324         struct nvme_rdma_queue *queue = cm_id->context;
1325         int cm_error = 0;
1326
1327         dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1328                 rdma_event_msg(ev->event), ev->event,
1329                 ev->status, cm_id);
1330
1331         switch (ev->event) {
1332         case RDMA_CM_EVENT_ADDR_RESOLVED:
1333                 cm_error = nvme_rdma_addr_resolved(queue);
1334                 break;
1335         case RDMA_CM_EVENT_ROUTE_RESOLVED:
1336                 cm_error = nvme_rdma_route_resolved(queue);
1337                 break;
1338         case RDMA_CM_EVENT_ESTABLISHED:
1339                 queue->cm_error = nvme_rdma_conn_established(queue);
1340                 /* complete cm_done regardless of success/failure */
1341                 complete(&queue->cm_done);
1342                 return 0;
1343         case RDMA_CM_EVENT_REJECTED:
1344                 nvme_rdma_destroy_queue_ib(queue);
1345                 cm_error = nvme_rdma_conn_rejected(queue, ev);
1346                 break;
1347         case RDMA_CM_EVENT_ROUTE_ERROR:
1348         case RDMA_CM_EVENT_CONNECT_ERROR:
1349         case RDMA_CM_EVENT_UNREACHABLE:
1350                 nvme_rdma_destroy_queue_ib(queue);
1351         case RDMA_CM_EVENT_ADDR_ERROR:
1352                 dev_dbg(queue->ctrl->ctrl.device,
1353                         "CM error event %d\n", ev->event);
1354                 cm_error = -ECONNRESET;
1355                 break;
1356         case RDMA_CM_EVENT_DISCONNECTED:
1357         case RDMA_CM_EVENT_ADDR_CHANGE:
1358         case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1359                 dev_dbg(queue->ctrl->ctrl.device,
1360                         "disconnect received - connection closed\n");
1361                 nvme_rdma_error_recovery(queue->ctrl);
1362                 break;
1363         case RDMA_CM_EVENT_DEVICE_REMOVAL:
1364                 /* device removal is handled via the ib_client API */
1365                 break;
1366         default:
1367                 dev_err(queue->ctrl->ctrl.device,
1368                         "Unexpected RDMA CM event (%d)\n", ev->event);
1369                 nvme_rdma_error_recovery(queue->ctrl);
1370                 break;
1371         }
1372
1373         if (cm_error) {
1374                 queue->cm_error = cm_error;
1375                 complete(&queue->cm_done);
1376         }
1377
1378         return 0;
1379 }
1380
1381 static enum blk_eh_timer_return
1382 nvme_rdma_timeout(struct request *rq, bool reserved)
1383 {
1384         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1385
1386         /* queue error recovery */
1387         nvme_rdma_error_recovery(req->queue->ctrl);
1388
1389         /* fail with DNR on cmd timeout */
1390         nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR;
1391
1392         return BLK_EH_HANDLED;
1393 }
1394
1395 /*
1396  * We cannot accept any other command until the Connect command has completed.
1397  */
1398 static inline blk_status_t
1399 nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue, struct request *rq)
1400 {
1401         if (unlikely(!test_bit(NVME_RDMA_Q_LIVE, &queue->flags))) {
1402                 struct nvme_command *cmd = nvme_req(rq)->cmd;
1403
1404                 if (!blk_rq_is_passthrough(rq) ||
1405                     cmd->common.opcode != nvme_fabrics_command ||
1406                     cmd->fabrics.fctype != nvme_fabrics_type_connect) {
1407                         /*
1408                          * reconnecting state means transport disruption, which
1409                          * can take a long time and even might fail permanently,
1410                          * so we can't let incoming I/O be requeued forever.
1411                          * fail it fast to allow upper layers a chance to
1412                          * failover.
1413                          */
1414                         if (queue->ctrl->ctrl.state == NVME_CTRL_RECONNECTING)
1415                                 return BLK_STS_IOERR;
1416                         return BLK_STS_RESOURCE; /* try again later */
1417                 }
1418         }
1419
1420         return 0;
1421 }
1422
1423 static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
1424                 const struct blk_mq_queue_data *bd)
1425 {
1426         struct nvme_ns *ns = hctx->queue->queuedata;
1427         struct nvme_rdma_queue *queue = hctx->driver_data;
1428         struct request *rq = bd->rq;
1429         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1430         struct nvme_rdma_qe *sqe = &req->sqe;
1431         struct nvme_command *c = sqe->data;
1432         bool flush = false;
1433         struct ib_device *dev;
1434         blk_status_t ret;
1435         int err;
1436
1437         WARN_ON_ONCE(rq->tag < 0);
1438
1439         ret = nvme_rdma_queue_is_ready(queue, rq);
1440         if (unlikely(ret))
1441                 return ret;
1442
1443         dev = queue->device->dev;
1444         ib_dma_sync_single_for_cpu(dev, sqe->dma,
1445                         sizeof(struct nvme_command), DMA_TO_DEVICE);
1446
1447         ret = nvme_setup_cmd(ns, rq, c);
1448         if (ret)
1449                 return ret;
1450
1451         blk_mq_start_request(rq);
1452
1453         err = nvme_rdma_map_data(queue, rq, c);
1454         if (err < 0) {
1455                 dev_err(queue->ctrl->ctrl.device,
1456                              "Failed to map data (%d)\n", err);
1457                 nvme_cleanup_cmd(rq);
1458                 goto err;
1459         }
1460
1461         ib_dma_sync_single_for_device(dev, sqe->dma,
1462                         sizeof(struct nvme_command), DMA_TO_DEVICE);
1463
1464         if (req_op(rq) == REQ_OP_FLUSH)
1465                 flush = true;
1466         err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1467                         req->mr->need_inval ? &req->reg_wr.wr : NULL, flush);
1468         if (err) {
1469                 nvme_rdma_unmap_data(queue, rq);
1470                 goto err;
1471         }
1472
1473         return BLK_STS_OK;
1474 err:
1475         if (err == -ENOMEM || err == -EAGAIN)
1476                 return BLK_STS_RESOURCE;
1477         return BLK_STS_IOERR;
1478 }
1479
1480 static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
1481 {
1482         struct nvme_rdma_queue *queue = hctx->driver_data;
1483         struct ib_cq *cq = queue->ib_cq;
1484         struct ib_wc wc;
1485         int found = 0;
1486
1487         while (ib_poll_cq(cq, 1, &wc) > 0) {
1488                 struct ib_cqe *cqe = wc.wr_cqe;
1489
1490                 if (cqe) {
1491                         if (cqe->done == nvme_rdma_recv_done)
1492                                 found |= __nvme_rdma_recv_done(cq, &wc, tag);
1493                         else
1494                                 cqe->done(cq, &wc);
1495                 }
1496         }
1497
1498         return found;
1499 }
1500
1501 static void nvme_rdma_complete_rq(struct request *rq)
1502 {
1503         struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1504
1505         nvme_rdma_unmap_data(req->queue, rq);
1506         nvme_complete_rq(rq);
1507 }
1508
1509 static const struct blk_mq_ops nvme_rdma_mq_ops = {
1510         .queue_rq       = nvme_rdma_queue_rq,
1511         .complete       = nvme_rdma_complete_rq,
1512         .init_request   = nvme_rdma_init_request,
1513         .exit_request   = nvme_rdma_exit_request,
1514         .reinit_request = nvme_rdma_reinit_request,
1515         .init_hctx      = nvme_rdma_init_hctx,
1516         .poll           = nvme_rdma_poll,
1517         .timeout        = nvme_rdma_timeout,
1518 };
1519
1520 static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
1521         .queue_rq       = nvme_rdma_queue_rq,
1522         .complete       = nvme_rdma_complete_rq,
1523         .init_request   = nvme_rdma_init_request,
1524         .exit_request   = nvme_rdma_exit_request,
1525         .reinit_request = nvme_rdma_reinit_request,
1526         .init_hctx      = nvme_rdma_init_admin_hctx,
1527         .timeout        = nvme_rdma_timeout,
1528 };
1529
1530 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl)
1531 {
1532         int error;
1533
1534         error = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
1535         if (error)
1536                 return error;
1537
1538         ctrl->device = ctrl->queues[0].device;
1539
1540         /*
1541          * We need a reference on the device as long as the tag_set is alive,
1542          * as the MRs in the request structures need a valid ib_device.
1543          */
1544         error = -EINVAL;
1545         if (!nvme_rdma_dev_get(ctrl->device))
1546                 goto out_free_queue;
1547
1548         ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
1549                 ctrl->device->dev->attrs.max_fast_reg_page_list_len);
1550
1551         memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
1552         ctrl->admin_tag_set.ops = &nvme_rdma_admin_mq_ops;
1553         ctrl->admin_tag_set.queue_depth = NVME_RDMA_AQ_BLKMQ_DEPTH;
1554         ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
1555         ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
1556         ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1557                 SG_CHUNK_SIZE * sizeof(struct scatterlist);
1558         ctrl->admin_tag_set.driver_data = ctrl;
1559         ctrl->admin_tag_set.nr_hw_queues = 1;
1560         ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
1561
1562         error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
1563         if (error)
1564                 goto out_put_dev;
1565
1566         ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
1567         if (IS_ERR(ctrl->ctrl.admin_q)) {
1568                 error = PTR_ERR(ctrl->ctrl.admin_q);
1569                 goto out_free_tagset;
1570         }
1571
1572         error = nvmf_connect_admin_queue(&ctrl->ctrl);
1573         if (error)
1574                 goto out_cleanup_queue;
1575
1576         set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
1577
1578         error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);
1579         if (error) {
1580                 dev_err(ctrl->ctrl.device,
1581                         "prop_get NVME_REG_CAP failed\n");
1582                 goto out_cleanup_queue;
1583         }
1584
1585         ctrl->ctrl.sqsize =
1586                 min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->ctrl.sqsize);
1587
1588         error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
1589         if (error)
1590                 goto out_cleanup_queue;
1591
1592         ctrl->ctrl.max_hw_sectors =
1593                 (ctrl->max_fr_pages - 1) << (PAGE_SHIFT - 9);
1594
1595         error = nvme_init_identify(&ctrl->ctrl);
1596         if (error)
1597                 goto out_cleanup_queue;
1598
1599         error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
1600                         &ctrl->async_event_sqe, sizeof(struct nvme_command),
1601                         DMA_TO_DEVICE);
1602         if (error)
1603                 goto out_cleanup_queue;
1604
1605         nvme_start_keep_alive(&ctrl->ctrl);
1606
1607         return 0;
1608
1609 out_cleanup_queue:
1610         blk_cleanup_queue(ctrl->ctrl.admin_q);
1611 out_free_tagset:
1612         /* disconnect and drain the queue before freeing the tagset */
1613         nvme_rdma_stop_queue(&ctrl->queues[0]);
1614         blk_mq_free_tag_set(&ctrl->admin_tag_set);
1615 out_put_dev:
1616         nvme_rdma_dev_put(ctrl->device);
1617 out_free_queue:
1618         nvme_rdma_free_queue(&ctrl->queues[0]);
1619         return error;
1620 }
1621
1622 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
1623 {
1624         nvme_stop_keep_alive(&ctrl->ctrl);
1625         cancel_work_sync(&ctrl->err_work);
1626         cancel_delayed_work_sync(&ctrl->reconnect_work);
1627
1628         if (ctrl->queue_count > 1) {
1629                 nvme_stop_queues(&ctrl->ctrl);
1630                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
1631                                         nvme_cancel_request, &ctrl->ctrl);
1632                 nvme_rdma_free_io_queues(ctrl);
1633         }
1634
1635         if (test_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags))
1636                 nvme_shutdown_ctrl(&ctrl->ctrl);
1637
1638         blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
1639         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1640                                 nvme_cancel_request, &ctrl->ctrl);
1641         nvme_rdma_destroy_admin_queue(ctrl);
1642 }
1643
1644 static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1645 {
1646         nvme_uninit_ctrl(&ctrl->ctrl);
1647         if (shutdown)
1648                 nvme_rdma_shutdown_ctrl(ctrl);
1649
1650         if (ctrl->ctrl.tagset) {
1651                 blk_cleanup_queue(ctrl->ctrl.connect_q);
1652                 blk_mq_free_tag_set(&ctrl->tag_set);
1653                 nvme_rdma_dev_put(ctrl->device);
1654         }
1655
1656         nvme_put_ctrl(&ctrl->ctrl);
1657 }
1658
1659 static void nvme_rdma_del_ctrl_work(struct work_struct *work)
1660 {
1661         struct nvme_rdma_ctrl *ctrl = container_of(work,
1662                                 struct nvme_rdma_ctrl, delete_work);
1663
1664         __nvme_rdma_remove_ctrl(ctrl, true);
1665 }
1666
1667 static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
1668 {
1669         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
1670                 return -EBUSY;
1671
1672         if (!queue_work(nvme_wq, &ctrl->delete_work))
1673                 return -EBUSY;
1674
1675         return 0;
1676 }
1677
1678 static int nvme_rdma_del_ctrl(struct nvme_ctrl *nctrl)
1679 {
1680         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1681         int ret = 0;
1682
1683         /*
1684          * Keep a reference until all work is flushed since
1685          * __nvme_rdma_del_ctrl can free the ctrl mem
1686          */
1687         if (!kref_get_unless_zero(&ctrl->ctrl.kref))
1688                 return -EBUSY;
1689         ret = __nvme_rdma_del_ctrl(ctrl);
1690         if (!ret)
1691                 flush_work(&ctrl->delete_work);
1692         nvme_put_ctrl(&ctrl->ctrl);
1693         return ret;
1694 }
1695
1696 static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
1697 {
1698         struct nvme_rdma_ctrl *ctrl = container_of(work,
1699                                 struct nvme_rdma_ctrl, delete_work);
1700
1701         __nvme_rdma_remove_ctrl(ctrl, false);
1702 }
1703
1704 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1705 {
1706         struct nvme_rdma_ctrl *ctrl = container_of(work,
1707                                         struct nvme_rdma_ctrl, reset_work);
1708         int ret;
1709         bool changed;
1710
1711         nvme_rdma_shutdown_ctrl(ctrl);
1712
1713         ret = nvme_rdma_configure_admin_queue(ctrl);
1714         if (ret) {
1715                 /* ctrl is already shutdown, just remove the ctrl */
1716                 INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
1717                 goto del_dead_ctrl;
1718         }
1719
1720         if (ctrl->queue_count > 1) {
1721                 ret = blk_mq_reinit_tagset(&ctrl->tag_set);
1722                 if (ret)
1723                         goto del_dead_ctrl;
1724
1725                 ret = nvme_rdma_init_io_queues(ctrl);
1726                 if (ret)
1727                         goto del_dead_ctrl;
1728
1729                 ret = nvme_rdma_connect_io_queues(ctrl);
1730                 if (ret)
1731                         goto del_dead_ctrl;
1732         }
1733
1734         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1735         WARN_ON_ONCE(!changed);
1736
1737         if (ctrl->queue_count > 1) {
1738                 nvme_start_queues(&ctrl->ctrl);
1739                 nvme_queue_scan(&ctrl->ctrl);
1740                 nvme_queue_async_events(&ctrl->ctrl);
1741         }
1742
1743         return;
1744
1745 del_dead_ctrl:
1746         /* Deleting this dead controller... */
1747         dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1748         WARN_ON(!queue_work(nvme_wq, &ctrl->delete_work));
1749 }
1750
1751 static int nvme_rdma_reset_ctrl(struct nvme_ctrl *nctrl)
1752 {
1753         struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1754
1755         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
1756                 return -EBUSY;
1757
1758         if (!queue_work(nvme_wq, &ctrl->reset_work))
1759                 return -EBUSY;
1760
1761         flush_work(&ctrl->reset_work);
1762
1763         return 0;
1764 }
1765
1766 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1767         .name                   = "rdma",
1768         .module                 = THIS_MODULE,
1769         .flags                  = NVME_F_FABRICS,
1770         .reg_read32             = nvmf_reg_read32,
1771         .reg_read64             = nvmf_reg_read64,
1772         .reg_write32            = nvmf_reg_write32,
1773         .reset_ctrl             = nvme_rdma_reset_ctrl,
1774         .free_ctrl              = nvme_rdma_free_ctrl,
1775         .submit_async_event     = nvme_rdma_submit_async_event,
1776         .delete_ctrl            = nvme_rdma_del_ctrl,
1777         .get_subsysnqn          = nvmf_get_subsysnqn,
1778         .get_address            = nvmf_get_address,
1779 };
1780
1781 static int nvme_rdma_create_io_queues(struct nvme_rdma_ctrl *ctrl)
1782 {
1783         int ret;
1784
1785         ret = nvme_rdma_init_io_queues(ctrl);
1786         if (ret)
1787                 return ret;
1788
1789         /*
1790          * We need a reference on the device as long as the tag_set is alive,
1791          * as the MRs in the request structures need a valid ib_device.
1792          */
1793         ret = -EINVAL;
1794         if (!nvme_rdma_dev_get(ctrl->device))
1795                 goto out_free_io_queues;
1796
1797         memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
1798         ctrl->tag_set.ops = &nvme_rdma_mq_ops;
1799         ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
1800         ctrl->tag_set.reserved_tags = 1; /* fabric connect */
1801         ctrl->tag_set.numa_node = NUMA_NO_NODE;
1802         ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1803         ctrl->tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1804                 SG_CHUNK_SIZE * sizeof(struct scatterlist);
1805         ctrl->tag_set.driver_data = ctrl;
1806         ctrl->tag_set.nr_hw_queues = ctrl->queue_count - 1;
1807         ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
1808
1809         ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
1810         if (ret)
1811                 goto out_put_dev;
1812         ctrl->ctrl.tagset = &ctrl->tag_set;
1813
1814         ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
1815         if (IS_ERR(ctrl->ctrl.connect_q)) {
1816                 ret = PTR_ERR(ctrl->ctrl.connect_q);
1817                 goto out_free_tag_set;
1818         }
1819
1820         ret = nvme_rdma_connect_io_queues(ctrl);
1821         if (ret)
1822                 goto out_cleanup_connect_q;
1823
1824         return 0;
1825
1826 out_cleanup_connect_q:
1827         blk_cleanup_queue(ctrl->ctrl.connect_q);
1828 out_free_tag_set:
1829         blk_mq_free_tag_set(&ctrl->tag_set);
1830 out_put_dev:
1831         nvme_rdma_dev_put(ctrl->device);
1832 out_free_io_queues:
1833         nvme_rdma_free_io_queues(ctrl);
1834         return ret;
1835 }
1836
1837 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
1838                 struct nvmf_ctrl_options *opts)
1839 {
1840         struct nvme_rdma_ctrl *ctrl;
1841         int ret;
1842         bool changed;
1843         char *port;
1844
1845         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1846         if (!ctrl)
1847                 return ERR_PTR(-ENOMEM);
1848         ctrl->ctrl.opts = opts;
1849         INIT_LIST_HEAD(&ctrl->list);
1850
1851         if (opts->mask & NVMF_OPT_TRSVCID)
1852                 port = opts->trsvcid;
1853         else
1854                 port = __stringify(NVME_RDMA_IP_PORT);
1855
1856         ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1857                         opts->traddr, port, &ctrl->addr);
1858         if (ret) {
1859                 pr_err("malformed address passed: %s:%s\n", opts->traddr, port);
1860                 goto out_free_ctrl;
1861         }
1862
1863         if (opts->mask & NVMF_OPT_HOST_TRADDR) {
1864                 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1865                         opts->host_traddr, NULL, &ctrl->src_addr);
1866                 if (ret) {
1867                         pr_err("malformed src address passed: %s\n",
1868                                opts->host_traddr);
1869                         goto out_free_ctrl;
1870                 }
1871         }
1872
1873         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
1874                                 0 /* no quirks, we're perfect! */);
1875         if (ret)
1876                 goto out_free_ctrl;
1877
1878         INIT_DELAYED_WORK(&ctrl->reconnect_work,
1879                         nvme_rdma_reconnect_ctrl_work);
1880         INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
1881         INIT_WORK(&ctrl->delete_work, nvme_rdma_del_ctrl_work);
1882         INIT_WORK(&ctrl->reset_work, nvme_rdma_reset_ctrl_work);
1883
1884         ctrl->queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
1885         ctrl->ctrl.sqsize = opts->queue_size - 1;
1886         ctrl->ctrl.kato = opts->kato;
1887
1888         ret = -ENOMEM;
1889         ctrl->queues = kcalloc(ctrl->queue_count, sizeof(*ctrl->queues),
1890                                 GFP_KERNEL);
1891         if (!ctrl->queues)
1892                 goto out_uninit_ctrl;
1893
1894         ret = nvme_rdma_configure_admin_queue(ctrl);
1895         if (ret)
1896                 goto out_kfree_queues;
1897
1898         /* sanity check icdoff */
1899         if (ctrl->ctrl.icdoff) {
1900                 dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
1901                 ret = -EINVAL;
1902                 goto out_remove_admin_queue;
1903         }
1904
1905         /* sanity check keyed sgls */
1906         if (!(ctrl->ctrl.sgls & (1 << 20))) {
1907                 dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not support\n");
1908                 ret = -EINVAL;
1909                 goto out_remove_admin_queue;
1910         }
1911
1912         if (opts->queue_size > ctrl->ctrl.maxcmd) {
1913                 /* warn if maxcmd is lower than queue_size */
1914                 dev_warn(ctrl->ctrl.device,
1915                         "queue_size %zu > ctrl maxcmd %u, clamping down\n",
1916                         opts->queue_size, ctrl->ctrl.maxcmd);
1917                 opts->queue_size = ctrl->ctrl.maxcmd;
1918         }
1919
1920         if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
1921                 /* warn if sqsize is lower than queue_size */
1922                 dev_warn(ctrl->ctrl.device,
1923                         "queue_size %zu > ctrl sqsize %u, clamping down\n",
1924                         opts->queue_size, ctrl->ctrl.sqsize + 1);
1925                 opts->queue_size = ctrl->ctrl.sqsize + 1;
1926         }
1927
1928         if (opts->nr_io_queues) {
1929                 ret = nvme_rdma_create_io_queues(ctrl);
1930                 if (ret)
1931                         goto out_remove_admin_queue;
1932         }
1933
1934         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1935         WARN_ON_ONCE(!changed);
1936
1937         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
1938                 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1939
1940         kref_get(&ctrl->ctrl.kref);
1941
1942         mutex_lock(&nvme_rdma_ctrl_mutex);
1943         list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
1944         mutex_unlock(&nvme_rdma_ctrl_mutex);
1945
1946         if (opts->nr_io_queues) {
1947                 nvme_queue_scan(&ctrl->ctrl);
1948                 nvme_queue_async_events(&ctrl->ctrl);
1949         }
1950
1951         return &ctrl->ctrl;
1952
1953 out_remove_admin_queue:
1954         nvme_stop_keep_alive(&ctrl->ctrl);
1955         nvme_rdma_destroy_admin_queue(ctrl);
1956 out_kfree_queues:
1957         kfree(ctrl->queues);
1958 out_uninit_ctrl:
1959         nvme_uninit_ctrl(&ctrl->ctrl);
1960         nvme_put_ctrl(&ctrl->ctrl);
1961         if (ret > 0)
1962                 ret = -EIO;
1963         return ERR_PTR(ret);
1964 out_free_ctrl:
1965         kfree(ctrl);
1966         return ERR_PTR(ret);
1967 }
1968
1969 static struct nvmf_transport_ops nvme_rdma_transport = {
1970         .name           = "rdma",
1971         .required_opts  = NVMF_OPT_TRADDR,
1972         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
1973                           NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO,
1974         .create_ctrl    = nvme_rdma_create_ctrl,
1975 };
1976
1977 static void nvme_rdma_add_one(struct ib_device *ib_device)
1978 {
1979 }
1980
1981 static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
1982 {
1983         struct nvme_rdma_ctrl *ctrl;
1984
1985         /* Delete all controllers using this device */
1986         mutex_lock(&nvme_rdma_ctrl_mutex);
1987         list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
1988                 if (ctrl->device->dev != ib_device)
1989                         continue;
1990                 dev_info(ctrl->ctrl.device,
1991                         "Removing ctrl: NQN \"%s\", addr %pISp\n",
1992                         ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1993                 __nvme_rdma_del_ctrl(ctrl);
1994         }
1995         mutex_unlock(&nvme_rdma_ctrl_mutex);
1996
1997         flush_workqueue(nvme_wq);
1998 }
1999
2000 static struct ib_client nvme_rdma_ib_client = {
2001         .name   = "nvme_rdma",
2002         .add = nvme_rdma_add_one,
2003         .remove = nvme_rdma_remove_one
2004 };
2005
2006 static int __init nvme_rdma_init_module(void)
2007 {
2008         int ret;
2009
2010         ret = ib_register_client(&nvme_rdma_ib_client);
2011         if (ret)
2012                 return ret;
2013
2014         ret = nvmf_register_transport(&nvme_rdma_transport);
2015         if (ret)
2016                 goto err_unreg_client;
2017
2018         return 0;
2019
2020 err_unreg_client:
2021         ib_unregister_client(&nvme_rdma_ib_client);
2022         return ret;
2023 }
2024
2025 static void __exit nvme_rdma_cleanup_module(void)
2026 {
2027         nvmf_unregister_transport(&nvme_rdma_transport);
2028         ib_unregister_client(&nvme_rdma_ib_client);
2029 }
2030
2031 module_init(nvme_rdma_init_module);
2032 module_exit(nvme_rdma_cleanup_module);
2033
2034 MODULE_LICENSE("GPL v2");