drivers/nvme/host/core.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #include <linux/blkdev.h>
  16 #include <linux/blk-mq.h>
  17 #include <linux/delay.h>
  18 #include <linux/errno.h>
  19 #include <linux/hdreg.h>
  20 #include <linux/kernel.h>
  21 #include <linux/module.h>
  22 #include <linux/list_sort.h>
  23 #include <linux/slab.h>
  24 #include <linux/types.h>
  25 #include <linux/pr.h>
  26 #include <linux/ptrace.h>
  27 #include <linux/nvme_ioctl.h>
  28 #include <linux/t10-pi.h>
  29 #include <linux/pm_qos.h>
  30 #include <asm/unaligned.h>
  31
  32 #include "nvme.h"
  33 #include "fabrics.h"
  34
  35 #define NVME_MINORS             (1U << MINORBITS)
  36
  37 unsigned int admin_timeout = 60;
  38 module_param(admin_timeout, uint, 0644);
  39 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  40 EXPORT_SYMBOL_GPL(admin_timeout);
  41
  42 unsigned int nvme_io_timeout = 30;
  43 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  44 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  45 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  46
  47 static unsigned char shutdown_timeout = 5;
  48 module_param(shutdown_timeout, byte, 0644);
  49 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  50
  51 static u8 nvme_max_retries = 5;
  52 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  53 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  54
  55 static unsigned long default_ps_max_latency_us = 100000;
  56 module_param(default_ps_max_latency_us, ulong, 0644);
  57 MODULE_PARM_DESC(default_ps_max_latency_us,
  58                  "max power saving latency for new devices; use PM QOS to change per device");
  59
  60 static bool force_apst;
  61 module_param(force_apst, bool, 0644);
  62 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  63
  64 static bool streams;
  65 module_param(streams, bool, 0644);
  66 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
  67
  68 struct workqueue_struct *nvme_wq;
  69 EXPORT_SYMBOL_GPL(nvme_wq);
  70
  71 static DEFINE_IDA(nvme_instance_ida);
  72 static dev_t nvme_chr_devt;
  73 static struct class *nvme_class;
  74
  75 static void nvme_ns_remove(struct nvme_ns *ns);
  76 static int nvme_revalidate_disk(struct gendisk *disk);
  77
  78 static __le32 nvme_get_log_dw10(u8 lid, size_t size)
  79 {
  80         return cpu_to_le32((((size / 4) - 1) << 16) | lid);
  81 }
  82
  83 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
  84 {
  85         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
  86                 return -EBUSY;
  87         if (!queue_work(nvme_wq, &ctrl->reset_work))
  88                 return -EBUSY;
  89         return 0;
  90 }
  91 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
  92
  93 static int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
  94 {
  95         int ret;
  96
  97         ret = nvme_reset_ctrl(ctrl);
  98         if (!ret)
  99                 flush_work(&ctrl->reset_work);
 100         return ret;
 101 }
 102
 103 static void nvme_delete_ctrl_work(struct work_struct *work)
 104 {
 105         struct nvme_ctrl *ctrl =
 106                 container_of(work, struct nvme_ctrl, delete_work);
 107
 108         flush_work(&ctrl->reset_work);
 109         nvme_stop_ctrl(ctrl);
 110         nvme_remove_namespaces(ctrl);
 111         ctrl->ops->delete_ctrl(ctrl);
 112         nvme_uninit_ctrl(ctrl);
 113         nvme_put_ctrl(ctrl);
 114 }
 115
 116 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 117 {
 118         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 119                 return -EBUSY;
 120         if (!queue_work(nvme_wq, &ctrl->delete_work))
 121                 return -EBUSY;
 122         return 0;
 123 }
 124 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 125
 126 int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 127 {
 128         int ret = 0;
 129
 130         /*
 131          * Keep a reference until the work is flushed since ->delete_ctrl
 132          * can free the controller.
 133          */
 134         nvme_get_ctrl(ctrl);
 135         ret = nvme_delete_ctrl(ctrl);
 136         if (!ret)
 137                 flush_work(&ctrl->delete_work);
 138         nvme_put_ctrl(ctrl);
 139         return ret;
 140 }
 141 EXPORT_SYMBOL_GPL(nvme_delete_ctrl_sync);
 142
 143 static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
 144 {
 145         return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
 146 }
 147
 148 static blk_status_t nvme_error_status(struct request *req)
 149 {
 150         switch (nvme_req(req)->status & 0x7ff) {
 151         case NVME_SC_SUCCESS:
 152                 return BLK_STS_OK;
 153         case NVME_SC_CAP_EXCEEDED:
 154                 return BLK_STS_NOSPC;
 155         case NVME_SC_ONCS_NOT_SUPPORTED:
 156                 return BLK_STS_NOTSUPP;
 157         case NVME_SC_WRITE_FAULT:
 158         case NVME_SC_READ_ERROR:
 159         case NVME_SC_UNWRITTEN_BLOCK:
 160         case NVME_SC_ACCESS_DENIED:
 161         case NVME_SC_READ_ONLY:
 162                 return BLK_STS_MEDIUM;
 163         case NVME_SC_GUARD_CHECK:
 164         case NVME_SC_APPTAG_CHECK:
 165         case NVME_SC_REFTAG_CHECK:
 166         case NVME_SC_INVALID_PI:
 167                 return BLK_STS_PROTECTION;
 168         case NVME_SC_RESERVATION_CONFLICT:
 169                 return BLK_STS_NEXUS;
 170         default:
 171                 return BLK_STS_IOERR;
 172         }
 173 }
 174
 175 static inline bool nvme_req_needs_retry(struct request *req)
 176 {
 177         if (blk_noretry_request(req))
 178                 return false;
 179         if (nvme_req(req)->status & NVME_SC_DNR)
 180                 return false;
 181         if (nvme_req(req)->retries >= nvme_max_retries)
 182                 return false;
 183         if (blk_queue_dying(req->q))
 184                 return false;
 185         return true;
 186 }
 187
 188 void nvme_complete_rq(struct request *req)
 189 {
 190         if (unlikely(nvme_req(req)->status && nvme_req_needs_retry(req))) {
 191                 nvme_req(req)->retries++;
 192                 blk_mq_requeue_request(req, true);
 193                 return;
 194         }
 195
 196         blk_mq_end_request(req, nvme_error_status(req));
 197 }
 198 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 199
 200 void nvme_cancel_request(struct request *req, void *data, bool reserved)
 201 {
 202         if (!blk_mq_request_started(req))
 203                 return;
 204
 205         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 206                                 "Cancelling I/O %d", req->tag);
 207
 208         nvme_req(req)->status = NVME_SC_ABORT_REQ;
 209         blk_mq_complete_request(req);
 210
 211 }
 212 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 213
 214 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 215                 enum nvme_ctrl_state new_state)
 216 {
 217         enum nvme_ctrl_state old_state;
 218         unsigned long flags;
 219         bool changed = false;
 220
 221         spin_lock_irqsave(&ctrl->lock, flags);
 222
 223         old_state = ctrl->state;
 224         switch (new_state) {
 225         case NVME_CTRL_LIVE:
 226                 switch (old_state) {
 227                 case NVME_CTRL_NEW:
 228                 case NVME_CTRL_RESETTING:
 229                 case NVME_CTRL_RECONNECTING:
 230                         changed = true;
 231                         /* FALLTHRU */
 232                 default:
 233                         break;
 234                 }
 235                 break;
 236         case NVME_CTRL_RESETTING:
 237                 switch (old_state) {
 238                 case NVME_CTRL_NEW:
 239                 case NVME_CTRL_LIVE:
 240                         changed = true;
 241                         /* FALLTHRU */
 242                 default:
 243                         break;
 244                 }
 245                 break;
 246         case NVME_CTRL_RECONNECTING:
 247                 switch (old_state) {
 248                 case NVME_CTRL_LIVE:
 249                 case NVME_CTRL_RESETTING:
 250                         changed = true;
 251                         /* FALLTHRU */
 252                 default:
 253                         break;
 254                 }
 255                 break;
 256         case NVME_CTRL_DELETING:
 257                 switch (old_state) {
 258                 case NVME_CTRL_LIVE:
 259                 case NVME_CTRL_RESETTING:
 260                 case NVME_CTRL_RECONNECTING:
 261                         changed = true;
 262                         /* FALLTHRU */
 263                 default:
 264                         break;
 265                 }
 266                 break;
 267         case NVME_CTRL_DEAD:
 268                 switch (old_state) {
 269                 case NVME_CTRL_DELETING:
 270                         changed = true;
 271                         /* FALLTHRU */
 272                 default:
 273                         break;
 274                 }
 275                 break;
 276         default:
 277                 break;
 278         }
 279
 280         if (changed)
 281                 ctrl->state = new_state;
 282
 283         spin_unlock_irqrestore(&ctrl->lock, flags);
 284
 285         return changed;
 286 }
 287 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 288
 289 static void nvme_free_ns(struct kref *kref)
 290 {
 291         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 292
 293         if (ns->ndev)
 294                 nvme_nvm_unregister(ns);
 295
 296         put_disk(ns->disk);
 297         ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
 298         nvme_put_ctrl(ns->ctrl);
 299         kfree(ns);
 300 }
 301
 302 static void nvme_put_ns(struct nvme_ns *ns)
 303 {
 304         kref_put(&ns->kref, nvme_free_ns);
 305 }
 306
 307 struct request *nvme_alloc_request(struct request_queue *q,
 308                 struct nvme_command *cmd, unsigned int flags, int qid)
 309 {
 310         unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
 311         struct request *req;
 312
 313         if (qid == NVME_QID_ANY) {
 314                 req = blk_mq_alloc_request(q, op, flags);
 315         } else {
 316                 req = blk_mq_alloc_request_hctx(q, op, flags,
 317                                 qid ? qid - 1 : 0);
 318         }
 319         if (IS_ERR(req))
 320                 return req;
 321
 322         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 323         nvme_req(req)->cmd = cmd;
 324
 325         return req;
 326 }
 327 EXPORT_SYMBOL_GPL(nvme_alloc_request);
 328
 329 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
 330 {
 331         struct nvme_command c;
 332
 333         memset(&c, 0, sizeof(c));
 334
 335         c.directive.opcode = nvme_admin_directive_send;
 336         c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
 337         c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
 338         c.directive.dtype = NVME_DIR_IDENTIFY;
 339         c.directive.tdtype = NVME_DIR_STREAMS;
 340         c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
 341
 342         return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
 343 }
 344
 345 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
 346 {
 347         return nvme_toggle_streams(ctrl, false);
 348 }
 349
 350 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
 351 {
 352         return nvme_toggle_streams(ctrl, true);
 353 }
 354
 355 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
 356                                   struct streams_directive_params *s, u32 nsid)
 357 {
 358         struct nvme_command c;
 359
 360         memset(&c, 0, sizeof(c));
 361         memset(s, 0, sizeof(*s));
 362
 363         c.directive.opcode = nvme_admin_directive_recv;
 364         c.directive.nsid = cpu_to_le32(nsid);
 365         c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
 366         c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
 367         c.directive.dtype = NVME_DIR_STREAMS;
 368
 369         return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
 370 }
 371
 372 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
 373 {
 374         struct streams_directive_params s;
 375         int ret;
 376
 377         if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
 378                 return 0;
 379         if (!streams)
 380                 return 0;
 381
 382         ret = nvme_enable_streams(ctrl);
 383         if (ret)
 384                 return ret;
 385
 386         ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
 387         if (ret)
 388                 return ret;
 389
 390         ctrl->nssa = le16_to_cpu(s.nssa);
 391         if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
 392                 dev_info(ctrl->device, "too few streams (%u) available\n",
 393                                         ctrl->nssa);
 394                 nvme_disable_streams(ctrl);
 395                 return 0;
 396         }
 397
 398         ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
 399         dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
 400         return 0;
 401 }
 402
 403 /*
 404  * Check if 'req' has a write hint associated with it. If it does, assign
 405  * a valid namespace stream to the write.
 406  */
 407 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
 408                                      struct request *req, u16 *control,
 409                                      u32 *dsmgmt)
 410 {
 411         enum rw_hint streamid = req->write_hint;
 412
 413         if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
 414                 streamid = 0;
 415         else {
 416                 streamid--;
 417                 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
 418                         return;
 419
 420                 *control |= NVME_RW_DTYPE_STREAMS;
 421                 *dsmgmt |= streamid << 16;
 422         }
 423
 424         if (streamid < ARRAY_SIZE(req->q->write_hints))
 425                 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
 426 }
 427
 428 static inline void nvme_setup_flush(struct nvme_ns *ns,
 429                 struct nvme_command *cmnd)
 430 {
 431         memset(cmnd, 0, sizeof(*cmnd));
 432         cmnd->common.opcode = nvme_cmd_flush;
 433         cmnd->common.nsid = cpu_to_le32(ns->ns_id);
 434 }
 435
 436 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 437                 struct nvme_command *cmnd)
 438 {
 439         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 440         struct nvme_dsm_range *range;
 441         struct bio *bio;
 442
 443         range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
 444         if (!range)
 445                 return BLK_STS_RESOURCE;
 446
 447         __rq_for_each_bio(bio, req) {
 448                 u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
 449                 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 450
 451                 range[n].cattr = cpu_to_le32(0);
 452                 range[n].nlb = cpu_to_le32(nlb);
 453                 range[n].slba = cpu_to_le64(slba);
 454                 n++;
 455         }
 456
 457         if (WARN_ON_ONCE(n != segments)) {
 458                 kfree(range);
 459                 return BLK_STS_IOERR;
 460         }
 461
 462         memset(cmnd, 0, sizeof(*cmnd));
 463         cmnd->dsm.opcode = nvme_cmd_dsm;
 464         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
 465         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 466         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 467
 468         req->special_vec.bv_page = virt_to_page(range);
 469         req->special_vec.bv_offset = offset_in_page(range);
 470         req->special_vec.bv_len = sizeof(*range) * segments;
 471         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 472
 473         return BLK_STS_OK;
 474 }
 475
 476 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 477                 struct request *req, struct nvme_command *cmnd)
 478 {
 479         struct nvme_ctrl *ctrl = ns->ctrl;
 480         u16 control = 0;
 481         u32 dsmgmt = 0;
 482
 483         if (req->cmd_flags & REQ_FUA)
 484                 control |= NVME_RW_FUA;
 485         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 486                 control |= NVME_RW_LR;
 487
 488         if (req->cmd_flags & REQ_RAHEAD)
 489                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 490
 491         memset(cmnd, 0, sizeof(*cmnd));
 492         cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
 493         cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
 494         cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
 495         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 496
 497         if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
 498                 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
 499
 500         if (ns->ms) {
 501                 /*
 502                  * If formated with metadata, the block layer always provides a
 503                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 504                  * we enable the PRACT bit for protection information or set the
 505                  * namespace capacity to zero to prevent any I/O.
 506                  */
 507                 if (!blk_integrity_rq(req)) {
 508                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 509                                 return BLK_STS_NOTSUPP;
 510                         control |= NVME_RW_PRINFO_PRACT;
 511                 }
 512
 513                 switch (ns->pi_type) {
 514                 case NVME_NS_DPS_PI_TYPE3:
 515                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 516                         break;
 517                 case NVME_NS_DPS_PI_TYPE1:
 518                 case NVME_NS_DPS_PI_TYPE2:
 519                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 520                                         NVME_RW_PRINFO_PRCHK_REF;
 521                         cmnd->rw.reftag = cpu_to_le32(
 522                                         nvme_block_nr(ns, blk_rq_pos(req)));
 523                         break;
 524                 }
 525         }
 526
 527         cmnd->rw.control = cpu_to_le16(control);
 528         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 529         return 0;
 530 }
 531
 532 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 533                 struct nvme_command *cmd)
 534 {
 535         blk_status_t ret = BLK_STS_OK;
 536
 537         if (!(req->rq_flags & RQF_DONTPREP)) {
 538                 nvme_req(req)->retries = 0;
 539                 nvme_req(req)->flags = 0;
 540                 req->rq_flags |= RQF_DONTPREP;
 541         }
 542
 543         switch (req_op(req)) {
 544         case REQ_OP_DRV_IN:
 545         case REQ_OP_DRV_OUT:
 546                 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
 547                 break;
 548         case REQ_OP_FLUSH:
 549                 nvme_setup_flush(ns, cmd);
 550                 break;
 551         case REQ_OP_WRITE_ZEROES:
 552                 /* currently only aliased to deallocate for a few ctrls: */
 553         case REQ_OP_DISCARD:
 554                 ret = nvme_setup_discard(ns, req, cmd);
 555                 break;
 556         case REQ_OP_READ:
 557         case REQ_OP_WRITE:
 558                 ret = nvme_setup_rw(ns, req, cmd);
 559                 break;
 560         default:
 561                 WARN_ON_ONCE(1);
 562                 return BLK_STS_IOERR;
 563         }
 564
 565         cmd->common.command_id = req->tag;
 566         return ret;
 567 }
 568 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
 569
 570 /*
 571  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 572  * if the result is positive, it's an NVM Express status code
 573  */
 574 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 575                 union nvme_result *result, void *buffer, unsigned bufflen,
 576                 unsigned timeout, int qid, int at_head, int flags)
 577 {
 578         struct request *req;
 579         int ret;
 580
 581         req = nvme_alloc_request(q, cmd, flags, qid);
 582         if (IS_ERR(req))
 583                 return PTR_ERR(req);
 584
 585         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
 586
 587         if (buffer && bufflen) {
 588                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
 589                 if (ret)
 590                         goto out;
 591         }
 592
 593         blk_execute_rq(req->q, NULL, req, at_head);
 594         if (result)
 595                 *result = nvme_req(req)->result;
 596         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 597                 ret = -EINTR;
 598         else
 599                 ret = nvme_req(req)->status;
 600  out:
 601         blk_mq_free_request(req);
 602         return ret;
 603 }
 604 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
 605
 606 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 607                 void *buffer, unsigned bufflen)
 608 {
 609         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
 610                         NVME_QID_ANY, 0, 0);
 611 }
 612 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 613
 614 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
 615                 unsigned len, u32 seed, bool write)
 616 {
 617         struct bio_integrity_payload *bip;
 618         int ret = -ENOMEM;
 619         void *buf;
 620
 621         buf = kmalloc(len, GFP_KERNEL);
 622         if (!buf)
 623                 goto out;
 624
 625         ret = -EFAULT;
 626         if (write && copy_from_user(buf, ubuf, len))
 627                 goto out_free_meta;
 628
 629         bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
 630         if (IS_ERR(bip)) {
 631                 ret = PTR_ERR(bip);
 632                 goto out_free_meta;
 633         }
 634
 635         bip->bip_iter.bi_size = len;
 636         bip->bip_iter.bi_sector = seed;
 637         ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
 638                         offset_in_page(buf));
 639         if (ret == len)
 640                 return buf;
 641         ret = -ENOMEM;
 642 out_free_meta:
 643         kfree(buf);
 644 out:
 645         return ERR_PTR(ret);
 646 }
 647
 648 static int nvme_submit_user_cmd(struct request_queue *q,
 649                 struct nvme_command *cmd, void __user *ubuffer,
 650                 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
 651                 u32 meta_seed, u32 *result, unsigned timeout)
 652 {
 653         bool write = nvme_is_write(cmd);
 654         struct nvme_ns *ns = q->queuedata;
 655         struct gendisk *disk = ns ? ns->disk : NULL;
 656         struct request *req;
 657         struct bio *bio = NULL;
 658         void *meta = NULL;
 659         int ret;
 660
 661         req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
 662         if (IS_ERR(req))
 663                 return PTR_ERR(req);
 664
 665         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
 666
 667         if (ubuffer && bufflen) {
 668                 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
 669                                 GFP_KERNEL);
 670                 if (ret)
 671                         goto out;
 672                 bio = req->bio;
 673                 bio->bi_disk = disk;
 674                 if (disk && meta_buffer && meta_len) {
 675                         meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
 676                                         meta_seed, write);
 677                         if (IS_ERR(meta)) {
 678                                 ret = PTR_ERR(meta);
 679                                 goto out_unmap;
 680                         }
 681                 }
 682         }
 683
 684         blk_execute_rq(req->q, disk, req, 0);
 685         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 686                 ret = -EINTR;
 687         else
 688                 ret = nvme_req(req)->status;
 689         if (result)
 690                 *result = le32_to_cpu(nvme_req(req)->result.u32);
 691         if (meta && !ret && !write) {
 692                 if (copy_to_user(meta_buffer, meta, meta_len))
 693                         ret = -EFAULT;
 694         }
 695         kfree(meta);
 696  out_unmap:
 697         if (bio)
 698                 blk_rq_unmap_user(bio);
 699  out:
 700         blk_mq_free_request(req);
 701         return ret;
 702 }
 703
 704 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
 705 {
 706         struct nvme_ctrl *ctrl = rq->end_io_data;
 707
 708         blk_mq_free_request(rq);
 709
 710         if (status) {
 711                 dev_err(ctrl->device,
 712                         "failed nvme_keep_alive_end_io error=%d\n",
 713                                 status);
 714                 return;
 715         }
 716
 717         schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
 718 }
 719
 720 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
 721 {
 722         struct nvme_command c;
 723         struct request *rq;
 724
 725         memset(&c, 0, sizeof(c));
 726         c.common.opcode = nvme_admin_keep_alive;
 727
 728         rq = nvme_alloc_request(ctrl->admin_q, &c, BLK_MQ_REQ_RESERVED,
 729                         NVME_QID_ANY);
 730         if (IS_ERR(rq))
 731                 return PTR_ERR(rq);
 732
 733         rq->timeout = ctrl->kato * HZ;
 734         rq->end_io_data = ctrl;
 735
 736         blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
 737
 738         return 0;
 739 }
 740
 741 static void nvme_keep_alive_work(struct work_struct *work)
 742 {
 743         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
 744                         struct nvme_ctrl, ka_work);
 745
 746         if (nvme_keep_alive(ctrl)) {
 747                 /* allocation failure, reset the controller */
 748                 dev_err(ctrl->device, "keep-alive failed\n");
 749                 nvme_reset_ctrl(ctrl);
 750                 return;
 751         }
 752 }
 753
 754 void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
 755 {
 756         if (unlikely(ctrl->kato == 0))
 757                 return;
 758
 759         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
 760         schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
 761 }
 762 EXPORT_SYMBOL_GPL(nvme_start_keep_alive);
 763
 764 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
 765 {
 766         if (unlikely(ctrl->kato == 0))
 767                 return;
 768
 769         cancel_delayed_work_sync(&ctrl->ka_work);
 770 }
 771 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
 772
 773 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
 774 {
 775         struct nvme_command c = { };
 776         int error;
 777
 778         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
 779         c.identify.opcode = nvme_admin_identify;
 780         c.identify.cns = NVME_ID_CNS_CTRL;
 781
 782         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
 783         if (!*id)
 784                 return -ENOMEM;
 785
 786         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
 787                         sizeof(struct nvme_id_ctrl));
 788         if (error)
 789                 kfree(*id);
 790         return error;
 791 }
 792
 793 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
 794                 u8 *eui64, u8 *nguid, uuid_t *uuid)
 795 {
 796         struct nvme_command c = { };
 797         int status;
 798         void *data;
 799         int pos;
 800         int len;
 801
 802         c.identify.opcode = nvme_admin_identify;
 803         c.identify.nsid = cpu_to_le32(nsid);
 804         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
 805
 806         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
 807         if (!data)
 808                 return -ENOMEM;
 809
 810         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
 811                                       NVME_IDENTIFY_DATA_SIZE);
 812         if (status)
 813                 goto free_data;
 814
 815         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
 816                 struct nvme_ns_id_desc *cur = data + pos;
 817
 818                 if (cur->nidl == 0)
 819                         break;
 820
 821                 switch (cur->nidt) {
 822                 case NVME_NIDT_EUI64:
 823                         if (cur->nidl != NVME_NIDT_EUI64_LEN) {
 824                                 dev_warn(ctrl->device,
 825                                          "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
 826                                          cur->nidl);
 827                                 goto free_data;
 828                         }
 829                         len = NVME_NIDT_EUI64_LEN;
 830                         memcpy(eui64, data + pos + sizeof(*cur), len);
 831                         break;
 832                 case NVME_NIDT_NGUID:
 833                         if (cur->nidl != NVME_NIDT_NGUID_LEN) {
 834                                 dev_warn(ctrl->device,
 835                                          "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
 836                                          cur->nidl);
 837                                 goto free_data;
 838                         }
 839                         len = NVME_NIDT_NGUID_LEN;
 840                         memcpy(nguid, data + pos + sizeof(*cur), len);
 841                         break;
 842                 case NVME_NIDT_UUID:
 843                         if (cur->nidl != NVME_NIDT_UUID_LEN) {
 844                                 dev_warn(ctrl->device,
 845                                          "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
 846                                          cur->nidl);
 847                                 goto free_data;
 848                         }
 849                         len = NVME_NIDT_UUID_LEN;
 850                         uuid_copy(uuid, data + pos + sizeof(*cur));
 851                         break;
 852                 default:
 853                         /* Skip unnkown types */
 854                         len = cur->nidl;
 855                         break;
 856                 }
 857
 858                 len += sizeof(*cur);
 859         }
 860 free_data:
 861         kfree(data);
 862         return status;
 863 }
 864
 865 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
 866 {
 867         struct nvme_command c = { };
 868
 869         c.identify.opcode = nvme_admin_identify;
 870         c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
 871         c.identify.nsid = cpu_to_le32(nsid);
 872         return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
 873 }
 874
 875 static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
 876                 unsigned nsid)
 877 {
 878         struct nvme_id_ns *id;
 879         struct nvme_command c = { };
 880         int error;
 881
 882         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
 883         c.identify.opcode = nvme_admin_identify;
 884         c.identify.nsid = cpu_to_le32(nsid);
 885         c.identify.cns = NVME_ID_CNS_NS;
 886
 887         id = kmalloc(sizeof(*id), GFP_KERNEL);
 888         if (!id)
 889                 return NULL;
 890
 891         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
 892         if (error) {
 893                 dev_warn(ctrl->device, "Identify namespace failed\n");
 894                 kfree(id);
 895                 return NULL;
 896         }
 897
 898         return id;
 899 }
 900
 901 static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
 902                       void *buffer, size_t buflen, u32 *result)
 903 {
 904         struct nvme_command c;
 905         union nvme_result res;
 906         int ret;
 907
 908         memset(&c, 0, sizeof(c));
 909         c.features.opcode = nvme_admin_set_features;
 910         c.features.fid = cpu_to_le32(fid);
 911         c.features.dword11 = cpu_to_le32(dword11);
 912
 913         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
 914                         buffer, buflen, 0, NVME_QID_ANY, 0, 0);
 915         if (ret >= 0 && result)
 916                 *result = le32_to_cpu(res.u32);
 917         return ret;
 918 }
 919
 920 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
 921 {
 922         u32 q_count = (*count - 1) | ((*count - 1) << 16);
 923         u32 result;
 924         int status, nr_io_queues;
 925
 926         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
 927                         &result);
 928         if (status < 0)
 929                 return status;
 930
 931         /*
 932          * Degraded controllers might return an error when setting the queue
 933          * count.  We still want to be able to bring them online and offer
 934          * access to the admin queue, as that might be only way to fix them up.
 935          */
 936         if (status > 0) {
 937                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
 938                 *count = 0;
 939         } else {
 940                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
 941                 *count = min(*count, nr_io_queues);
 942         }
 943
 944         return 0;
 945 }
 946 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
 947
 948 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
 949 {
 950         struct nvme_user_io io;
 951         struct nvme_command c;
 952         unsigned length, meta_len;
 953         void __user *metadata;
 954
 955         if (copy_from_user(&io, uio, sizeof(io)))
 956                 return -EFAULT;
 957         if (io.flags)
 958                 return -EINVAL;
 959
 960         switch (io.opcode) {
 961         case nvme_cmd_write:
 962         case nvme_cmd_read:
 963         case nvme_cmd_compare:
 964                 break;
 965         default:
 966                 return -EINVAL;
 967         }
 968
 969         length = (io.nblocks + 1) << ns->lba_shift;
 970         meta_len = (io.nblocks + 1) * ns->ms;
 971         metadata = (void __user *)(uintptr_t)io.metadata;
 972
 973         if (ns->ext) {
 974                 length += meta_len;
 975                 meta_len = 0;
 976         } else if (meta_len) {
 977                 if ((io.metadata & 3) || !io.metadata)
 978                         return -EINVAL;
 979         }
 980
 981         memset(&c, 0, sizeof(c));
 982         c.rw.opcode = io.opcode;
 983         c.rw.flags = io.flags;
 984         c.rw.nsid = cpu_to_le32(ns->ns_id);
 985         c.rw.slba = cpu_to_le64(io.slba);
 986         c.rw.length = cpu_to_le16(io.nblocks);
 987         c.rw.control = cpu_to_le16(io.control);
 988         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
 989         c.rw.reftag = cpu_to_le32(io.reftag);
 990         c.rw.apptag = cpu_to_le16(io.apptag);
 991         c.rw.appmask = cpu_to_le16(io.appmask);
 992
 993         return nvme_submit_user_cmd(ns->queue, &c,
 994                         (void __user *)(uintptr_t)io.addr, length,
 995                         metadata, meta_len, io.slba, NULL, 0);
 996 }
 997
 998 static u32 nvme_known_admin_effects(u8 opcode)
 999 {
1000         switch (opcode) {
1001         case nvme_admin_format_nvm:
1002                 return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
1003                                         NVME_CMD_EFFECTS_CSE_MASK;
1004         case nvme_admin_sanitize_nvm:
1005                 return NVME_CMD_EFFECTS_CSE_MASK;
1006         default:
1007                 break;
1008         }
1009         return 0;
1010 }
1011
1012 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1013                                                                 u8 opcode)
1014 {
1015         u32 effects = 0;
1016
1017         if (ns) {
1018                 if (ctrl->effects)
1019                         effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1020                 if (effects & ~NVME_CMD_EFFECTS_CSUPP)
1021                         dev_warn(ctrl->device,
1022                                  "IO command:%02x has unhandled effects:%08x\n",
1023                                  opcode, effects);
1024                 return 0;
1025         }
1026
1027         if (ctrl->effects)
1028                 effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1029         else
1030                 effects = nvme_known_admin_effects(opcode);
1031
1032         /*
1033          * For simplicity, IO to all namespaces is quiesced even if the command
1034          * effects say only one namespace is affected.
1035          */
1036         if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1037                 nvme_start_freeze(ctrl);
1038                 nvme_wait_freeze(ctrl);
1039         }
1040         return effects;
1041 }
1042
1043 static void nvme_update_formats(struct nvme_ctrl *ctrl)
1044 {
1045         struct nvme_ns *ns;
1046
1047         mutex_lock(&ctrl->namespaces_mutex);
1048         list_for_each_entry(ns, &ctrl->namespaces, list) {
1049                 if (ns->disk && nvme_revalidate_disk(ns->disk))
1050                         nvme_ns_remove(ns);
1051         }
1052         mutex_unlock(&ctrl->namespaces_mutex);
1053 }
1054
1055 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1056 {
1057         /*
1058          * Revalidate LBA changes prior to unfreezing. This is necessary to
1059          * prevent memory corruption if a logical block size was changed by
1060          * this command.
1061          */
1062         if (effects & NVME_CMD_EFFECTS_LBCC)
1063                 nvme_update_formats(ctrl);
1064         if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK))
1065                 nvme_unfreeze(ctrl);
1066         if (effects & NVME_CMD_EFFECTS_CCC)
1067                 nvme_init_identify(ctrl);
1068         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))
1069                 nvme_queue_scan(ctrl);
1070 }
1071
1072 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1073                         struct nvme_passthru_cmd __user *ucmd)
1074 {
1075         struct nvme_passthru_cmd cmd;
1076         struct nvme_command c;
1077         unsigned timeout = 0;
1078         u32 effects;
1079         int status;
1080
1081         if (!capable(CAP_SYS_ADMIN))
1082                 return -EACCES;
1083         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1084                 return -EFAULT;
1085         if (cmd.flags)
1086                 return -EINVAL;
1087
1088         memset(&c, 0, sizeof(c));
1089         c.common.opcode = cmd.opcode;
1090         c.common.flags = cmd.flags;
1091         c.common.nsid = cpu_to_le32(cmd.nsid);
1092         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1093         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1094         c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1095         c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1096         c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1097         c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1098         c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1099         c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1100
1101         if (cmd.timeout_ms)
1102                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1103
1104         effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1105         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1106                         (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1107                         (void __user *)(uintptr_t)cmd.metadata, cmd.metadata,
1108                         0, &cmd.result, timeout);
1109         nvme_passthru_end(ctrl, effects);
1110
1111         if (status >= 0) {
1112                 if (put_user(cmd.result, &ucmd->result))
1113                         return -EFAULT;
1114         }
1115
1116         return status;
1117 }
1118
1119 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1120                 unsigned int cmd, unsigned long arg)
1121 {
1122         struct nvme_ns *ns = bdev->bd_disk->private_data;
1123
1124         switch (cmd) {
1125         case NVME_IOCTL_ID:
1126                 force_successful_syscall_return();
1127                 return ns->ns_id;
1128         case NVME_IOCTL_ADMIN_CMD:
1129                 return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
1130         case NVME_IOCTL_IO_CMD:
1131                 return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
1132         case NVME_IOCTL_SUBMIT_IO:
1133                 return nvme_submit_io(ns, (void __user *)arg);
1134         default:
1135 #ifdef CONFIG_NVM
1136                 if (ns->ndev)
1137                         return nvme_nvm_ioctl(ns, cmd, arg);
1138 #endif
1139                 if (is_sed_ioctl(cmd))
1140                         return sed_ioctl(ns->ctrl->opal_dev, cmd,
1141                                          (void __user *) arg);
1142                 return -ENOTTY;
1143         }
1144 }
1145
1146 static int nvme_open(struct block_device *bdev, fmode_t mode)
1147 {
1148         struct nvme_ns *ns = bdev->bd_disk->private_data;
1149
1150         if (!kref_get_unless_zero(&ns->kref))
1151                 return -ENXIO;
1152         return 0;
1153 }
1154
1155 static void nvme_release(struct gendisk *disk, fmode_t mode)
1156 {
1157         nvme_put_ns(disk->private_data);
1158 }
1159
1160 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1161 {
1162         /* some standard values */
1163         geo->heads = 1 << 6;
1164         geo->sectors = 1 << 5;
1165         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1166         return 0;
1167 }
1168
1169 #ifdef CONFIG_BLK_DEV_INTEGRITY
1170 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1171 {
1172         struct blk_integrity integrity;
1173
1174         memset(&integrity, 0, sizeof(integrity));
1175         switch (pi_type) {
1176         case NVME_NS_DPS_PI_TYPE3:
1177                 integrity.profile = &t10_pi_type3_crc;
1178                 integrity.tag_size = sizeof(u16) + sizeof(u32);
1179                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1180                 break;
1181         case NVME_NS_DPS_PI_TYPE1:
1182         case NVME_NS_DPS_PI_TYPE2:
1183                 integrity.profile = &t10_pi_type1_crc;
1184                 integrity.tag_size = sizeof(u16);
1185                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1186                 break;
1187         default:
1188                 integrity.profile = NULL;
1189                 break;
1190         }
1191         integrity.tuple_size = ms;
1192         blk_integrity_register(disk, &integrity);
1193         blk_queue_max_integrity_segments(disk->queue, 1);
1194 }
1195 #else
1196 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1197 {
1198 }
1199 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1200
1201 static void nvme_set_chunk_size(struct nvme_ns *ns)
1202 {
1203         u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9));
1204         blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
1205 }
1206
1207 static void nvme_config_discard(struct nvme_ctrl *ctrl,
1208                 unsigned stream_alignment, struct request_queue *queue)
1209 {
1210         u32 size = queue_logical_block_size(queue);
1211
1212         if (stream_alignment)
1213                 size *= stream_alignment;
1214
1215         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1216                         NVME_DSM_MAX_RANGES);
1217
1218         queue->limits.discard_alignment = size;
1219         queue->limits.discard_granularity = size;
1220
1221         blk_queue_max_discard_sectors(queue, UINT_MAX);
1222         blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1223         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, queue);
1224
1225         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1226                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1227 }
1228
1229 static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1230                 struct nvme_id_ns *id, u8 *eui64, u8 *nguid, uuid_t *uuid)
1231 {
1232         if (ctrl->vs >= NVME_VS(1, 1, 0))
1233                 memcpy(eui64, id->eui64, sizeof(id->eui64));
1234         if (ctrl->vs >= NVME_VS(1, 2, 0))
1235                 memcpy(nguid, id->nguid, sizeof(id->nguid));
1236         if (ctrl->vs >= NVME_VS(1, 3, 0)) {
1237                  /* Don't treat error as fatal we potentially
1238                   * already have a NGUID or EUI-64
1239                   */
1240                 if (nvme_identify_ns_descs(ctrl, nsid, eui64, nguid, uuid))
1241                         dev_warn(ctrl->device,
1242                                  "%s: Identify Descriptors failed\n", __func__);
1243         }
1244 }
1245
1246 static void nvme_update_disk_info(struct gendisk *disk,
1247                 struct nvme_ns *ns, struct nvme_id_ns *id)
1248 {
1249         sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9);
1250         unsigned stream_alignment = 0;
1251
1252         if (ns->ctrl->nr_streams && ns->sws && ns->sgs)
1253                 stream_alignment = ns->sws * ns->sgs;
1254
1255         blk_mq_freeze_queue(disk->queue);
1256         blk_integrity_unregister(disk);
1257
1258         blk_queue_logical_block_size(disk->queue, 1 << ns->lba_shift);
1259         if (ns->ms && !ns->ext &&
1260             (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1261                 nvme_init_integrity(disk, ns->ms, ns->pi_type);
1262         if (ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk))
1263                 capacity = 0;
1264         set_capacity(disk, capacity);
1265
1266         if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
1267                 nvme_config_discard(ns->ctrl, stream_alignment, disk->queue);
1268         blk_mq_unfreeze_queue(disk->queue);
1269 }
1270
1271 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
1272 {
1273         struct nvme_ns *ns = disk->private_data;
1274
1275         /*
1276          * If identify namespace failed, use default 512 byte block size so
1277          * block layer can use before failing read/write for 0 capacity.
1278          */
1279         ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
1280         if (ns->lba_shift == 0)
1281                 ns->lba_shift = 9;
1282         ns->noiob = le16_to_cpu(id->noiob);
1283         ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
1284         ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1285         /* the PI implementation requires metadata equal t10 pi tuple size */
1286         if (ns->ms == sizeof(struct t10_pi_tuple))
1287                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1288         else
1289                 ns->pi_type = 0;
1290
1291         if (ns->noiob)
1292                 nvme_set_chunk_size(ns);
1293         nvme_update_disk_info(disk, ns, id);
1294 }
1295
1296 static int nvme_revalidate_disk(struct gendisk *disk)
1297 {
1298         struct nvme_ns *ns = disk->private_data;
1299         struct nvme_ctrl *ctrl = ns->ctrl;
1300         struct nvme_id_ns *id;
1301         u8 eui64[8] = { 0 }, nguid[16] = { 0 };
1302         uuid_t uuid = uuid_null;
1303         int ret = 0;
1304
1305         if (test_bit(NVME_NS_DEAD, &ns->flags)) {
1306                 set_capacity(disk, 0);
1307                 return -ENODEV;
1308         }
1309
1310         id = nvme_identify_ns(ctrl, ns->ns_id);
1311         if (!id)
1312                 return -ENODEV;
1313
1314         if (id->ncap == 0) {
1315                 ret = -ENODEV;
1316                 goto out;
1317         }
1318
1319         nvme_report_ns_ids(ctrl, ns->ns_id, id, eui64, nguid, &uuid);
1320         if (!uuid_equal(&ns->uuid, &uuid) ||
1321             memcmp(&ns->nguid, &nguid, sizeof(ns->nguid)) ||
1322             memcmp(&ns->eui, &eui64, sizeof(ns->eui))) {
1323                 dev_err(ctrl->device,
1324                         "identifiers changed for nsid %d\n", ns->ns_id);
1325                 ret = -ENODEV;
1326         }
1327
1328 out:
1329         kfree(id);
1330         return ret;
1331 }
1332
1333 static char nvme_pr_type(enum pr_type type)
1334 {
1335         switch (type) {
1336         case PR_WRITE_EXCLUSIVE:
1337                 return 1;
1338         case PR_EXCLUSIVE_ACCESS:
1339                 return 2;
1340         case PR_WRITE_EXCLUSIVE_REG_ONLY:
1341                 return 3;
1342         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
1343                 return 4;
1344         case PR_WRITE_EXCLUSIVE_ALL_REGS:
1345                 return 5;
1346         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
1347                 return 6;
1348         default:
1349                 return 0;
1350         }
1351 };
1352
1353 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
1354                                 u64 key, u64 sa_key, u8 op)
1355 {
1356         struct nvme_ns *ns = bdev->bd_disk->private_data;
1357         struct nvme_command c;
1358         u8 data[16] = { 0, };
1359
1360         put_unaligned_le64(key, &data[0]);
1361         put_unaligned_le64(sa_key, &data[8]);
1362
1363         memset(&c, 0, sizeof(c));
1364         c.common.opcode = op;
1365         c.common.nsid = cpu_to_le32(ns->ns_id);
1366         c.common.cdw10[0] = cpu_to_le32(cdw10);
1367
1368         return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1369 }
1370
1371 static int nvme_pr_register(struct block_device *bdev, u64 old,
1372                 u64 new, unsigned flags)
1373 {
1374         u32 cdw10;
1375
1376         if (flags & ~PR_FL_IGNORE_KEY)
1377                 return -EOPNOTSUPP;
1378
1379         cdw10 = old ? 2 : 0;
1380         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
1381         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
1382         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
1383 }
1384
1385 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
1386                 enum pr_type type, unsigned flags)
1387 {
1388         u32 cdw10;
1389
1390         if (flags & ~PR_FL_IGNORE_KEY)
1391                 return -EOPNOTSUPP;
1392
1393         cdw10 = nvme_pr_type(type) << 8;
1394         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
1395         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
1396 }
1397
1398 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
1399                 enum pr_type type, bool abort)
1400 {
1401         u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
1402         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
1403 }
1404
1405 static int nvme_pr_clear(struct block_device *bdev, u64 key)
1406 {
1407         u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1408         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
1409 }
1410
1411 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
1412 {
1413         u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
1414         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1415 }
1416
1417 static const struct pr_ops nvme_pr_ops = {
1418         .pr_register    = nvme_pr_register,
1419         .pr_reserve     = nvme_pr_reserve,
1420         .pr_release     = nvme_pr_release,
1421         .pr_preempt     = nvme_pr_preempt,
1422         .pr_clear       = nvme_pr_clear,
1423 };
1424
1425 #ifdef CONFIG_BLK_SED_OPAL
1426 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
1427                 bool send)
1428 {
1429         struct nvme_ctrl *ctrl = data;
1430         struct nvme_command cmd;
1431
1432         memset(&cmd, 0, sizeof(cmd));
1433         if (send)
1434                 cmd.common.opcode = nvme_admin_security_send;
1435         else
1436                 cmd.common.opcode = nvme_admin_security_recv;
1437         cmd.common.nsid = 0;
1438         cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
1439         cmd.common.cdw10[1] = cpu_to_le32(len);
1440
1441         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
1442                                       ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
1443 }
1444 EXPORT_SYMBOL_GPL(nvme_sec_submit);
1445 #endif /* CONFIG_BLK_SED_OPAL */
1446
1447 static const struct block_device_operations nvme_fops = {
1448         .owner          = THIS_MODULE,
1449         .ioctl          = nvme_ioctl,
1450         .compat_ioctl   = nvme_ioctl,
1451         .open           = nvme_open,
1452         .release        = nvme_release,
1453         .getgeo         = nvme_getgeo,
1454         .revalidate_disk= nvme_revalidate_disk,
1455         .pr_ops         = &nvme_pr_ops,
1456 };
1457
1458 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
1459 {
1460         unsigned long timeout =
1461                 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1462         u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
1463         int ret;
1464
1465         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
1466                 if (csts == ~0)
1467                         return -ENODEV;
1468                 if ((csts & NVME_CSTS_RDY) == bit)
1469                         break;
1470
1471                 msleep(100);
1472                 if (fatal_signal_pending(current))
1473                         return -EINTR;
1474                 if (time_after(jiffies, timeout)) {
1475                         dev_err(ctrl->device,
1476                                 "Device not ready; aborting %s\n", enabled ?
1477                                                 "initialisation" : "reset");
1478                         return -ENODEV;
1479                 }
1480         }
1481
1482         return ret;
1483 }
1484
1485 /*
1486  * If the device has been passed off to us in an enabled state, just clear
1487  * the enabled bit.  The spec says we should set the 'shutdown notification
1488  * bits', but doing so may cause the device to complete commands to the
1489  * admin queue ... and we don't know what memory that might be pointing at!
1490  */
1491 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
1492 {
1493         int ret;
1494
1495         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
1496         ctrl->ctrl_config &= ~NVME_CC_ENABLE;
1497
1498         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1499         if (ret)
1500                 return ret;
1501
1502         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
1503                 msleep(NVME_QUIRK_DELAY_AMOUNT);
1504
1505         return nvme_wait_ready(ctrl, cap, false);
1506 }
1507 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1508
1509 int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
1510 {
1511         /*
1512          * Default to a 4K page size, with the intention to update this
1513          * path in the future to accomodate architectures with differing
1514          * kernel and IO page sizes.
1515          */
1516         unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
1517         int ret;
1518
1519         if (page_shift < dev_page_min) {
1520                 dev_err(ctrl->device,
1521                         "Minimum device page size %u too large for host (%u)\n",
1522                         1 << dev_page_min, 1 << page_shift);
1523                 return -ENODEV;
1524         }
1525
1526         ctrl->page_size = 1 << page_shift;
1527
1528         ctrl->ctrl_config = NVME_CC_CSS_NVM;
1529         ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
1530         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
1531         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1532         ctrl->ctrl_config |= NVME_CC_ENABLE;
1533
1534         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1535         if (ret)
1536                 return ret;
1537         return nvme_wait_ready(ctrl, cap, true);
1538 }
1539 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1540
1541 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
1542 {
1543         unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
1544         u32 csts;
1545         int ret;
1546
1547         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
1548         ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
1549
1550         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1551         if (ret)
1552                 return ret;
1553
1554         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
1555                 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
1556                         break;
1557
1558                 msleep(100);
1559                 if (fatal_signal_pending(current))
1560                         return -EINTR;
1561                 if (time_after(jiffies, timeout)) {
1562                         dev_err(ctrl->device,
1563                                 "Device shutdown incomplete; abort shutdown\n");
1564                         return -ENODEV;
1565                 }
1566         }
1567
1568         return ret;
1569 }
1570 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1571
1572 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1573                 struct request_queue *q)
1574 {
1575         bool vwc = false;
1576
1577         if (ctrl->max_hw_sectors) {
1578                 u32 max_segments =
1579                         (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
1580
1581                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1582                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1583         }
1584         if (ctrl->quirks & NVME_QUIRK_STRIPE_SIZE)
1585                 blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1586         blk_queue_virt_boundary(q, ctrl->page_size - 1);
1587         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
1588                 vwc = true;
1589         blk_queue_write_cache(q, vwc, vwc);
1590 }
1591
1592 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
1593 {
1594         __le64 ts;
1595         int ret;
1596
1597         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
1598                 return 0;
1599
1600         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
1601         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
1602                         NULL);
1603         if (ret)
1604                 dev_warn_once(ctrl->device,
1605                         "could not set timestamp (%d)\n", ret);
1606         return ret;
1607 }
1608
1609 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
1610 {
1611         /*
1612          * APST (Autonomous Power State Transition) lets us program a
1613          * table of power state transitions that the controller will
1614          * perform automatically.  We configure it with a simple
1615          * heuristic: we are willing to spend at most 2% of the time
1616          * transitioning between power states.  Therefore, when running
1617          * in any given state, we will enter the next lower-power
1618          * non-operational state after waiting 50 * (enlat + exlat)
1619          * microseconds, as long as that state's exit latency is under
1620          * the requested maximum latency.
1621          *
1622          * We will not autonomously enter any non-operational state for
1623          * which the total latency exceeds ps_max_latency_us.  Users
1624          * can set ps_max_latency_us to zero to turn off APST.
1625          */
1626
1627         unsigned apste;
1628         struct nvme_feat_auto_pst *table;
1629         u64 max_lat_us = 0;
1630         int max_ps = -1;
1631         int ret;
1632
1633         /*
1634          * If APST isn't supported or if we haven't been initialized yet,
1635          * then don't do anything.
1636          */
1637         if (!ctrl->apsta)
1638                 return 0;
1639
1640         if (ctrl->npss > 31) {
1641                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
1642                 return 0;
1643         }
1644
1645         table = kzalloc(sizeof(*table), GFP_KERNEL);
1646         if (!table)
1647                 return 0;
1648
1649         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
1650                 /* Turn off APST. */
1651                 apste = 0;
1652                 dev_dbg(ctrl->device, "APST disabled\n");
1653         } else {
1654                 __le64 target = cpu_to_le64(0);
1655                 int state;
1656
1657                 /*
1658                  * Walk through all states from lowest- to highest-power.
1659                  * According to the spec, lower-numbered states use more
1660                  * power.  NPSS, despite the name, is the index of the
1661                  * lowest-power state, not the number of states.
1662                  */
1663                 for (state = (int)ctrl->npss; state >= 0; state--) {
1664                         u64 total_latency_us, exit_latency_us, transition_ms;
1665
1666                         if (target)
1667                                 table->entries[state] = target;
1668
1669                         /*
1670                          * Don't allow transitions to the deepest state
1671                          * if it's quirked off.
1672                          */
1673                         if (state == ctrl->npss &&
1674                             (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
1675                                 continue;
1676
1677                         /*
1678                          * Is this state a useful non-operational state for
1679                          * higher-power states to autonomously transition to?
1680                          */
1681                         if (!(ctrl->psd[state].flags &
1682                               NVME_PS_FLAGS_NON_OP_STATE))
1683                                 continue;
1684
1685                         exit_latency_us =
1686                                 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
1687                         if (exit_latency_us > ctrl->ps_max_latency_us)
1688                                 continue;
1689
1690                         total_latency_us =
1691                                 exit_latency_us +
1692                                 le32_to_cpu(ctrl->psd[state].entry_lat);
1693
1694                         /*
1695                          * This state is good.  Use it as the APST idle
1696                          * target for higher power states.
1697                          */
1698                         transition_ms = total_latency_us + 19;
1699                         do_div(transition_ms, 20);
1700                         if (transition_ms > (1 << 24) - 1)
1701                                 transition_ms = (1 << 24) - 1;
1702
1703                         target = cpu_to_le64((state << 3) |
1704                                              (transition_ms << 8));
1705
1706                         if (max_ps == -1)
1707                                 max_ps = state;
1708
1709                         if (total_latency_us > max_lat_us)
1710                                 max_lat_us = total_latency_us;
1711                 }
1712
1713                 apste = 1;
1714
1715                 if (max_ps == -1) {
1716                         dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
1717                 } else {
1718                         dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
1719                                 max_ps, max_lat_us, (int)sizeof(*table), table);
1720                 }
1721         }
1722
1723         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
1724                                 table, sizeof(*table), NULL);
1725         if (ret)
1726                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
1727
1728         kfree(table);
1729         return ret;
1730 }
1731
1732 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
1733 {
1734         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1735         u64 latency;
1736
1737         switch (val) {
1738         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
1739         case PM_QOS_LATENCY_ANY:
1740                 latency = U64_MAX;
1741                 break;
1742
1743         default:
1744                 latency = val;
1745         }
1746
1747         if (ctrl->ps_max_latency_us != latency) {
1748                 ctrl->ps_max_latency_us = latency;
1749                 nvme_configure_apst(ctrl);
1750         }
1751 }
1752
1753 struct nvme_core_quirk_entry {
1754         /*
1755          * NVMe model and firmware strings are padded with spaces.  For
1756          * simplicity, strings in the quirk table are padded with NULLs
1757          * instead.
1758          */
1759         u16 vid;
1760         const char *mn;
1761         const char *fr;
1762         unsigned long quirks;
1763 };
1764
1765 static const struct nvme_core_quirk_entry core_quirks[] = {
1766         {
1767                 /*
1768                  * This Toshiba device seems to die using any APST states.  See:
1769                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
1770                  */
1771                 .vid = 0x1179,
1772                 .mn = "THNSF5256GPUK TOSHIBA",
1773                 .quirks = NVME_QUIRK_NO_APST,
1774         }
1775 };
1776
1777 /* match is null-terminated but idstr is space-padded. */
1778 static bool string_matches(const char *idstr, const char *match, size_t len)
1779 {
1780         size_t matchlen;
1781
1782         if (!match)
1783                 return true;
1784
1785         matchlen = strlen(match);
1786         WARN_ON_ONCE(matchlen > len);
1787
1788         if (memcmp(idstr, match, matchlen))
1789                 return false;
1790
1791         for (; matchlen < len; matchlen++)
1792                 if (idstr[matchlen] != ' ')
1793                         return false;
1794
1795         return true;
1796 }
1797
1798 static bool quirk_matches(const struct nvme_id_ctrl *id,
1799                           const struct nvme_core_quirk_entry *q)
1800 {
1801         return q->vid == le16_to_cpu(id->vid) &&
1802                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
1803                 string_matches(id->fr, q->fr, sizeof(id->fr));
1804 }
1805
1806 static void nvme_init_subnqn(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
1807 {
1808         size_t nqnlen;
1809         int off;
1810
1811         nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
1812         if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
1813                 strcpy(ctrl->subnqn, id->subnqn);
1814                 return;
1815         }
1816
1817         if (ctrl->vs >= NVME_VS(1, 2, 1))
1818                 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
1819
1820         /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
1821         off = snprintf(ctrl->subnqn, NVMF_NQN_SIZE,
1822                         "nqn.2014.08.org.nvmexpress:%4x%4x",
1823                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
1824         memcpy(ctrl->subnqn + off, id->sn, sizeof(id->sn));
1825         off += sizeof(id->sn);
1826         memcpy(ctrl->subnqn + off, id->mn, sizeof(id->mn));
1827         off += sizeof(id->mn);
1828         memset(ctrl->subnqn + off, 0, sizeof(ctrl->subnqn) - off);
1829 }
1830
1831 static int nvme_get_log(struct nvme_ctrl *ctrl, u8 log_page, void *log,
1832                         size_t size)
1833 {
1834         struct nvme_command c = { };
1835
1836         c.common.opcode = nvme_admin_get_log_page;
1837         c.common.nsid = cpu_to_le32(NVME_NSID_ALL);
1838         c.common.cdw10[0] = nvme_get_log_dw10(log_page, size);
1839
1840         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
1841 }
1842
1843 static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
1844 {
1845         int ret;
1846
1847         if (!ctrl->effects)
1848                 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
1849
1850         if (!ctrl->effects)
1851                 return 0;
1852
1853         ret = nvme_get_log(ctrl, NVME_LOG_CMD_EFFECTS, ctrl->effects,
1854                                         sizeof(*ctrl->effects));
1855         if (ret) {
1856                 kfree(ctrl->effects);
1857                 ctrl->effects = NULL;
1858         }
1859         return ret;
1860 }
1861
1862 /*
1863  * Initialize the cached copies of the Identify data and various controller
1864  * register in our nvme_ctrl structure.  This should be called as soon as
1865  * the admin queue is fully up and running.
1866  */
1867 int nvme_init_identify(struct nvme_ctrl *ctrl)
1868 {
1869         struct nvme_id_ctrl *id;
1870         u64 cap;
1871         int ret, page_shift;
1872         u32 max_hw_sectors;
1873         bool prev_apst_enabled;
1874
1875         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
1876         if (ret) {
1877                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
1878                 return ret;
1879         }
1880
1881         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
1882         if (ret) {
1883                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
1884                 return ret;
1885         }
1886         page_shift = NVME_CAP_MPSMIN(cap) + 12;
1887
1888         if (ctrl->vs >= NVME_VS(1, 1, 0))
1889                 ctrl->subsystem = NVME_CAP_NSSRC(cap);
1890
1891         ret = nvme_identify_ctrl(ctrl, &id);
1892         if (ret) {
1893                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
1894                 return -EIO;
1895         }
1896
1897         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
1898                 ret = nvme_get_effects_log(ctrl);
1899                 if (ret < 0)
1900                         return ret;
1901         }
1902
1903         nvme_init_subnqn(ctrl, id);
1904
1905         if (!ctrl->identified) {
1906                 /*
1907                  * Check for quirks.  Quirk can depend on firmware version,
1908                  * so, in principle, the set of quirks present can change
1909                  * across a reset.  As a possible future enhancement, we
1910                  * could re-scan for quirks every time we reinitialize
1911                  * the device, but we'd have to make sure that the driver
1912                  * behaves intelligently if the quirks change.
1913                  */
1914
1915                 int i;
1916
1917                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
1918                         if (quirk_matches(id, &core_quirks[i]))
1919                                 ctrl->quirks |= core_quirks[i].quirks;
1920                 }
1921         }
1922
1923         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
1924                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
1925                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
1926         }
1927
1928         ctrl->oacs = le16_to_cpu(id->oacs);
1929         ctrl->vid = le16_to_cpu(id->vid);
1930         ctrl->oncs = le16_to_cpup(&id->oncs);
1931         atomic_set(&ctrl->abort_limit, id->acl + 1);
1932         ctrl->vwc = id->vwc;
1933         ctrl->cntlid = le16_to_cpup(&id->cntlid);
1934         memcpy(ctrl->serial, id->sn, sizeof(id->sn));
1935         memcpy(ctrl->model, id->mn, sizeof(id->mn));
1936         memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
1937         if (id->mdts)
1938                 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
1939         else
1940                 max_hw_sectors = UINT_MAX;
1941         ctrl->max_hw_sectors =
1942                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
1943
1944         nvme_set_queue_limits(ctrl, ctrl->admin_q);
1945         ctrl->sgls = le32_to_cpu(id->sgls);
1946         ctrl->kas = le16_to_cpu(id->kas);
1947
1948         if (id->rtd3e) {
1949                 /* us -> s */
1950                 u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
1951
1952                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
1953                                                  shutdown_timeout, 60);
1954
1955                 if (ctrl->shutdown_timeout != shutdown_timeout)
1956                         dev_warn(ctrl->device,
1957                                  "Shutdown timeout set to %u seconds\n",
1958                                  ctrl->shutdown_timeout);
1959         } else
1960                 ctrl->shutdown_timeout = shutdown_timeout;
1961
1962         ctrl->npss = id->npss;
1963         ctrl->apsta = id->apsta;
1964         prev_apst_enabled = ctrl->apst_enabled;
1965         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
1966                 if (force_apst && id->apsta) {
1967                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
1968                         ctrl->apst_enabled = true;
1969                 } else {
1970                         ctrl->apst_enabled = false;
1971                 }
1972         } else {
1973                 ctrl->apst_enabled = id->apsta;
1974         }
1975         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
1976
1977         if (ctrl->ops->flags & NVME_F_FABRICS) {
1978                 ctrl->icdoff = le16_to_cpu(id->icdoff);
1979                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
1980                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
1981                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
1982
1983                 /*
1984                  * In fabrics we need to verify the cntlid matches the
1985                  * admin connect
1986                  */
1987                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
1988                         ret = -EINVAL;
1989                         goto out_free;
1990                 }
1991
1992                 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
1993                         dev_err(ctrl->device,
1994                                 "keep-alive support is mandatory for fabrics\n");
1995                         ret = -EINVAL;
1996                         goto out_free;
1997                 }
1998         } else {
1999                 ctrl->cntlid = le16_to_cpu(id->cntlid);
2000                 ctrl->hmpre = le32_to_cpu(id->hmpre);
2001                 ctrl->hmmin = le32_to_cpu(id->hmmin);
2002                 ctrl->hmminds = le32_to_cpu(id->hmminds);
2003                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
2004         }
2005
2006         kfree(id);
2007
2008         if (ctrl->apst_enabled && !prev_apst_enabled)
2009                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
2010         else if (!ctrl->apst_enabled && prev_apst_enabled)
2011                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
2012
2013         ret = nvme_configure_apst(ctrl);
2014         if (ret < 0)
2015                 return ret;
2016
2017         ret = nvme_configure_timestamp(ctrl);
2018         if (ret < 0)
2019                 return ret;
2020
2021         ret = nvme_configure_directives(ctrl);
2022         if (ret < 0)
2023                 return ret;
2024
2025         ctrl->identified = true;
2026
2027         return 0;
2028
2029 out_free:
2030         kfree(id);
2031         return ret;
2032 }
2033 EXPORT_SYMBOL_GPL(nvme_init_identify);
2034
2035 static int nvme_dev_open(struct inode *inode, struct file *file)
2036 {
2037         struct nvme_ctrl *ctrl =
2038                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
2039
2040         if (ctrl->state != NVME_CTRL_LIVE)
2041                 return -EWOULDBLOCK;
2042         file->private_data = ctrl;
2043         return 0;
2044 }
2045
2046 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
2047 {
2048         struct nvme_ns *ns;
2049         int ret;
2050
2051         mutex_lock(&ctrl->namespaces_mutex);
2052         if (list_empty(&ctrl->namespaces)) {
2053                 ret = -ENOTTY;
2054                 goto out_unlock;
2055         }
2056
2057         ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
2058         if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
2059                 dev_warn(ctrl->device,
2060                         "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
2061                 ret = -EINVAL;
2062                 goto out_unlock;
2063         }
2064
2065         dev_warn(ctrl->device,
2066                 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
2067         kref_get(&ns->kref);
2068         mutex_unlock(&ctrl->namespaces_mutex);
2069
2070         ret = nvme_user_cmd(ctrl, ns, argp);
2071         nvme_put_ns(ns);
2072         return ret;
2073
2074 out_unlock:
2075         mutex_unlock(&ctrl->namespaces_mutex);
2076         return ret;
2077 }
2078
2079 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
2080                 unsigned long arg)
2081 {
2082         struct nvme_ctrl *ctrl = file->private_data;
2083         void __user *argp = (void __user *)arg;
2084
2085         switch (cmd) {
2086         case NVME_IOCTL_ADMIN_CMD:
2087                 return nvme_user_cmd(ctrl, NULL, argp);
2088         case NVME_IOCTL_IO_CMD:
2089                 return nvme_dev_user_cmd(ctrl, argp);
2090         case NVME_IOCTL_RESET:
2091                 dev_warn(ctrl->device, "resetting controller\n");
2092                 return nvme_reset_ctrl_sync(ctrl);
2093         case NVME_IOCTL_SUBSYS_RESET:
2094                 return nvme_reset_subsystem(ctrl);
2095         case NVME_IOCTL_RESCAN:
2096                 nvme_queue_scan(ctrl);
2097                 return 0;
2098         default:
2099                 return -ENOTTY;
2100         }
2101 }
2102
2103 static const struct file_operations nvme_dev_fops = {
2104         .owner          = THIS_MODULE,
2105         .open           = nvme_dev_open,
2106         .unlocked_ioctl = nvme_dev_ioctl,
2107         .compat_ioctl   = nvme_dev_ioctl,
2108 };
2109
2110 static ssize_t nvme_sysfs_reset(struct device *dev,
2111                                 struct device_attribute *attr, const char *buf,
2112                                 size_t count)
2113 {
2114         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2115         int ret;
2116
2117         ret = nvme_reset_ctrl_sync(ctrl);
2118         if (ret < 0)
2119                 return ret;
2120         return count;
2121 }
2122 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
2123
2124 static ssize_t nvme_sysfs_rescan(struct device *dev,
2125                                 struct device_attribute *attr, const char *buf,
2126                                 size_t count)
2127 {
2128         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2129
2130         nvme_queue_scan(ctrl);
2131         return count;
2132 }
2133 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
2134
2135 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
2136                                                                 char *buf)
2137 {
2138         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2139         struct nvme_ctrl *ctrl = ns->ctrl;
2140         int serial_len = sizeof(ctrl->serial);
2141         int model_len = sizeof(ctrl->model);
2142
2143         if (!uuid_is_null(&ns->uuid))
2144                 return sprintf(buf, "uuid.%pU\n", &ns->uuid);
2145
2146         if (memchr_inv(ns->nguid, 0, sizeof(ns->nguid)))
2147                 return sprintf(buf, "eui.%16phN\n", ns->nguid);
2148
2149         if (memchr_inv(ns->eui, 0, sizeof(ns->eui)))
2150                 return sprintf(buf, "eui.%8phN\n", ns->eui);
2151
2152         while (serial_len > 0 && (ctrl->serial[serial_len - 1] == ' ' ||
2153                                   ctrl->serial[serial_len - 1] == '\0'))
2154                 serial_len--;
2155         while (model_len > 0 && (ctrl->model[model_len - 1] == ' ' ||
2156                                  ctrl->model[model_len - 1] == '\0'))
2157                 model_len--;
2158
2159         return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", ctrl->vid,
2160                 serial_len, ctrl->serial, model_len, ctrl->model, ns->ns_id);
2161 }
2162 static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);
2163
2164 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
2165                           char *buf)
2166 {
2167         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2168         return sprintf(buf, "%pU\n", ns->nguid);
2169 }
2170 static DEVICE_ATTR(nguid, S_IRUGO, nguid_show, NULL);
2171
2172 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
2173                                                                 char *buf)
2174 {
2175         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2176
2177         /* For backward compatibility expose the NGUID to userspace if
2178          * we have no UUID set
2179          */
2180         if (uuid_is_null(&ns->uuid)) {
2181                 printk_ratelimited(KERN_WARNING
2182                                    "No UUID available providing old NGUID\n");
2183                 return sprintf(buf, "%pU\n", ns->nguid);
2184         }
2185         return sprintf(buf, "%pU\n", &ns->uuid);
2186 }
2187 static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
2188
2189 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
2190                                                                 char *buf)
2191 {
2192         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2193         return sprintf(buf, "%8phd\n", ns->eui);
2194 }
2195 static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
2196
2197 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
2198                                                                 char *buf)
2199 {
2200         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2201         return sprintf(buf, "%d\n", ns->ns_id);
2202 }
2203 static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
2204
2205 static struct attribute *nvme_ns_attrs[] = {
2206         &dev_attr_wwid.attr,
2207         &dev_attr_uuid.attr,
2208         &dev_attr_nguid.attr,
2209         &dev_attr_eui.attr,
2210         &dev_attr_nsid.attr,
2211         NULL,
2212 };
2213
2214 static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
2215                 struct attribute *a, int n)
2216 {
2217         struct device *dev = container_of(kobj, struct device, kobj);
2218         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
2219
2220         if (a == &dev_attr_uuid.attr) {
2221                 if (uuid_is_null(&ns->uuid) ||
2222                     !memchr_inv(ns->nguid, 0, sizeof(ns->nguid)))
2223                         return 0;
2224         }
2225         if (a == &dev_attr_nguid.attr) {
2226                 if (!memchr_inv(ns->nguid, 0, sizeof(ns->nguid)))
2227                         return 0;
2228         }
2229         if (a == &dev_attr_eui.attr) {
2230                 if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
2231                         return 0;
2232         }
2233         return a->mode;
2234 }
2235
2236 static const struct attribute_group nvme_ns_attr_group = {
2237         .attrs          = nvme_ns_attrs,
2238         .is_visible     = nvme_ns_attrs_are_visible,
2239 };
2240
2241 #define nvme_show_str_function(field)                                           \
2242 static ssize_t  field##_show(struct device *dev,                                \
2243                             struct device_attribute *attr, char *buf)           \
2244 {                                                                               \
2245         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
2246         return sprintf(buf, "%.*s\n", (int)sizeof(ctrl->field), ctrl->field);   \
2247 }                                                                               \
2248 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
2249
2250 #define nvme_show_int_function(field)                                           \
2251 static ssize_t  field##_show(struct device *dev,                                \
2252                             struct device_attribute *attr, char *buf)           \
2253 {                                                                               \
2254         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
2255         return sprintf(buf, "%d\n", ctrl->field);       \
2256 }                                                                               \
2257 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
2258
2259 nvme_show_str_function(model);
2260 nvme_show_str_function(serial);
2261 nvme_show_str_function(firmware_rev);
2262 nvme_show_int_function(cntlid);
2263
2264 static ssize_t nvme_sysfs_delete(struct device *dev,
2265                                 struct device_attribute *attr, const char *buf,
2266                                 size_t count)
2267 {
2268         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2269
2270         if (device_remove_file_self(dev, attr))
2271                 nvme_delete_ctrl_sync(ctrl);
2272         return count;
2273 }
2274 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
2275
2276 static ssize_t nvme_sysfs_show_transport(struct device *dev,
2277                                          struct device_attribute *attr,
2278                                          char *buf)
2279 {
2280         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2281
2282         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
2283 }
2284 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
2285
2286 static ssize_t nvme_sysfs_show_state(struct device *dev,
2287                                      struct device_attribute *attr,
2288                                      char *buf)
2289 {
2290         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2291         static const char *const state_name[] = {
2292                 [NVME_CTRL_NEW]         = "new",
2293                 [NVME_CTRL_LIVE]        = "live",
2294                 [NVME_CTRL_RESETTING]   = "resetting",
2295                 [NVME_CTRL_RECONNECTING]= "reconnecting",
2296                 [NVME_CTRL_DELETING]    = "deleting",
2297                 [NVME_CTRL_DEAD]        = "dead",
2298         };
2299
2300         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
2301             state_name[ctrl->state])
2302                 return sprintf(buf, "%s\n", state_name[ctrl->state]);
2303
2304         return sprintf(buf, "unknown state\n");
2305 }
2306
2307 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
2308
2309 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
2310                                          struct device_attribute *attr,
2311                                          char *buf)
2312 {
2313         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2314
2315         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subnqn);
2316 }
2317 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
2318
2319 static ssize_t nvme_sysfs_show_address(struct device *dev,
2320                                          struct device_attribute *attr,
2321                                          char *buf)
2322 {
2323         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2324
2325         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
2326 }
2327 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
2328
2329 static struct attribute *nvme_dev_attrs[] = {
2330         &dev_attr_reset_controller.attr,
2331         &dev_attr_rescan_controller.attr,
2332         &dev_attr_model.attr,
2333         &dev_attr_serial.attr,
2334         &dev_attr_firmware_rev.attr,
2335         &dev_attr_cntlid.attr,
2336         &dev_attr_delete_controller.attr,
2337         &dev_attr_transport.attr,
2338         &dev_attr_subsysnqn.attr,
2339         &dev_attr_address.attr,
2340         &dev_attr_state.attr,
2341         NULL
2342 };
2343
2344 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
2345                 struct attribute *a, int n)
2346 {
2347         struct device *dev = container_of(kobj, struct device, kobj);
2348         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2349
2350         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
2351                 return 0;
2352         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
2353                 return 0;
2354
2355         return a->mode;
2356 }
2357
2358 static struct attribute_group nvme_dev_attrs_group = {
2359         .attrs          = nvme_dev_attrs,
2360         .is_visible     = nvme_dev_attrs_are_visible,
2361 };
2362
2363 static const struct attribute_group *nvme_dev_attr_groups[] = {
2364         &nvme_dev_attrs_group,
2365         NULL,
2366 };
2367
2368 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
2369 {
2370         struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
2371         struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
2372
2373         return nsa->ns_id - nsb->ns_id;
2374 }
2375
2376 static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2377 {
2378         struct nvme_ns *ns, *ret = NULL;
2379
2380         mutex_lock(&ctrl->namespaces_mutex);
2381         list_for_each_entry(ns, &ctrl->namespaces, list) {
2382                 if (ns->ns_id == nsid) {
2383                         if (!kref_get_unless_zero(&ns->kref))
2384                                 continue;
2385                         ret = ns;
2386                         break;
2387                 }
2388                 if (ns->ns_id > nsid)
2389                         break;
2390         }
2391         mutex_unlock(&ctrl->namespaces_mutex);
2392         return ret;
2393 }
2394
2395 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
2396 {
2397         struct streams_directive_params s;
2398         int ret;
2399
2400         if (!ctrl->nr_streams)
2401                 return 0;
2402
2403         ret = nvme_get_stream_params(ctrl, &s, ns->ns_id);
2404         if (ret)
2405                 return ret;
2406
2407         ns->sws = le32_to_cpu(s.sws);
2408         ns->sgs = le16_to_cpu(s.sgs);
2409
2410         if (ns->sws) {
2411                 unsigned int bs = 1 << ns->lba_shift;
2412
2413                 blk_queue_io_min(ns->queue, bs * ns->sws);
2414                 if (ns->sgs)
2415                         blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
2416         }
2417
2418         return 0;
2419 }
2420
2421 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2422 {
2423         struct nvme_ns *ns;
2424         struct gendisk *disk;
2425         struct nvme_id_ns *id;
2426         char disk_name[DISK_NAME_LEN];
2427         int node = dev_to_node(ctrl->dev);
2428
2429         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
2430         if (!ns)
2431                 return;
2432
2433         ns->instance = ida_simple_get(&ctrl->ns_ida, 1, 0, GFP_KERNEL);
2434         if (ns->instance < 0)
2435                 goto out_free_ns;
2436
2437         ns->queue = blk_mq_init_queue(ctrl->tagset);
2438         if (IS_ERR(ns->queue))
2439                 goto out_release_instance;
2440         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
2441         ns->queue->queuedata = ns;
2442         ns->ctrl = ctrl;
2443
2444         kref_init(&ns->kref);
2445         ns->ns_id = nsid;
2446         ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
2447
2448         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
2449         nvme_set_queue_limits(ctrl, ns->queue);
2450         nvme_setup_streams_ns(ctrl, ns);
2451
2452         sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
2453
2454         id = nvme_identify_ns(ctrl, nsid);
2455         if (!id)
2456                 goto out_free_queue;
2457
2458         if (id->ncap == 0)
2459                 goto out_free_id;
2460
2461         nvme_report_ns_ids(ctrl, ns->ns_id, id, ns->eui, ns->nguid, &ns->uuid);
2462
2463         if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
2464                 if (nvme_nvm_register(ns, disk_name, node)) {
2465                         dev_warn(ctrl->device, "LightNVM init failure\n");
2466                         goto out_free_id;
2467                 }
2468         }
2469
2470         disk = alloc_disk_node(0, node);
2471         if (!disk)
2472                 goto out_free_id;
2473
2474         disk->fops = &nvme_fops;
2475         disk->private_data = ns;
2476         disk->queue = ns->queue;
2477         disk->flags = GENHD_FL_EXT_DEVT;
2478         memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
2479         ns->disk = disk;
2480
2481         __nvme_revalidate_disk(disk, id);
2482
2483         mutex_lock(&ctrl->namespaces_mutex);
2484         list_add_tail(&ns->list, &ctrl->namespaces);
2485         mutex_unlock(&ctrl->namespaces_mutex);
2486
2487         nvme_get_ctrl(ctrl);
2488
2489         kfree(id);
2490
2491         device_add_disk(ctrl->device, ns->disk);
2492         if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
2493                                         &nvme_ns_attr_group))
2494                 pr_warn("%s: failed to create sysfs group for identification\n",
2495                         ns->disk->disk_name);
2496         if (ns->ndev && nvme_nvm_register_sysfs(ns))
2497                 pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
2498                         ns->disk->disk_name);
2499         return;
2500  out_free_id:
2501         kfree(id);
2502  out_free_queue:
2503         blk_cleanup_queue(ns->queue);
2504  out_release_instance:
2505         ida_simple_remove(&ctrl->ns_ida, ns->instance);
2506  out_free_ns:
2507         kfree(ns);
2508 }
2509
2510 static void nvme_ns_remove(struct nvme_ns *ns)
2511 {
2512         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
2513                 return;
2514
2515         if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
2516                 if (blk_get_integrity(ns->disk))
2517                         blk_integrity_unregister(ns->disk);
2518                 sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
2519                                         &nvme_ns_attr_group);
2520                 if (ns->ndev)
2521                         nvme_nvm_unregister_sysfs(ns);
2522                 del_gendisk(ns->disk);
2523                 blk_cleanup_queue(ns->queue);
2524         }
2525
2526         mutex_lock(&ns->ctrl->namespaces_mutex);
2527         list_del_init(&ns->list);
2528         mutex_unlock(&ns->ctrl->namespaces_mutex);
2529
2530         nvme_put_ns(ns);
2531 }
2532
2533 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2534 {
2535         struct nvme_ns *ns;
2536
2537         ns = nvme_find_get_ns(ctrl, nsid);
2538         if (ns) {
2539                 if (ns->disk && revalidate_disk(ns->disk))
2540                         nvme_ns_remove(ns);
2541                 nvme_put_ns(ns);
2542         } else
2543                 nvme_alloc_ns(ctrl, nsid);
2544 }
2545
2546 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
2547                                         unsigned nsid)
2548 {
2549         struct nvme_ns *ns, *next;
2550
2551         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
2552                 if (ns->ns_id > nsid)
2553                         nvme_ns_remove(ns);
2554         }
2555 }
2556
2557 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
2558 {
2559         struct nvme_ns *ns;
2560         __le32 *ns_list;
2561         unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
2562         int ret = 0;
2563
2564         ns_list = kzalloc(0x1000, GFP_KERNEL);
2565         if (!ns_list)
2566                 return -ENOMEM;
2567
2568         for (i = 0; i < num_lists; i++) {
2569                 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
2570                 if (ret)
2571                         goto free;
2572
2573                 for (j = 0; j < min(nn, 1024U); j++) {
2574                         nsid = le32_to_cpu(ns_list[j]);
2575                         if (!nsid)
2576                                 goto out;
2577
2578                         nvme_validate_ns(ctrl, nsid);
2579
2580                         while (++prev < nsid) {
2581                                 ns = nvme_find_get_ns(ctrl, prev);
2582                                 if (ns) {
2583                                         nvme_ns_remove(ns);
2584                                         nvme_put_ns(ns);
2585                                 }
2586                         }
2587                 }
2588                 nn -= j;
2589         }
2590  out:
2591         nvme_remove_invalid_namespaces(ctrl, prev);
2592  free:
2593         kfree(ns_list);
2594         return ret;
2595 }
2596
2597 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
2598 {
2599         unsigned i;
2600
2601         for (i = 1; i <= nn; i++)
2602                 nvme_validate_ns(ctrl, i);
2603
2604         nvme_remove_invalid_namespaces(ctrl, nn);
2605 }
2606
2607 static void nvme_scan_work(struct work_struct *work)
2608 {
2609         struct nvme_ctrl *ctrl =
2610                 container_of(work, struct nvme_ctrl, scan_work);
2611         struct nvme_id_ctrl *id;
2612         unsigned nn;
2613
2614         if (ctrl->state != NVME_CTRL_LIVE)
2615                 return;
2616
2617         if (nvme_identify_ctrl(ctrl, &id))
2618                 return;
2619
2620         nn = le32_to_cpu(id->nn);
2621         if (ctrl->vs >= NVME_VS(1, 1, 0) &&
2622             !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
2623                 if (!nvme_scan_ns_list(ctrl, nn))
2624                         goto done;
2625         }
2626         nvme_scan_ns_sequential(ctrl, nn);
2627  done:
2628         mutex_lock(&ctrl->namespaces_mutex);
2629         list_sort(NULL, &ctrl->namespaces, ns_cmp);
2630         mutex_unlock(&ctrl->namespaces_mutex);
2631         kfree(id);
2632 }
2633
2634 void nvme_queue_scan(struct nvme_ctrl *ctrl)
2635 {
2636         /*
2637          * Do not queue new scan work when a controller is reset during
2638          * removal.
2639          */
2640         if (ctrl->state == NVME_CTRL_LIVE)
2641                 queue_work(nvme_wq, &ctrl->scan_work);
2642 }
2643 EXPORT_SYMBOL_GPL(nvme_queue_scan);
2644
2645 /*
2646  * This function iterates the namespace list unlocked to allow recovery from
2647  * controller failure. It is up to the caller to ensure the namespace list is
2648  * not modified by scan work while this function is executing.
2649  */
2650 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
2651 {
2652         struct nvme_ns *ns, *next;
2653
2654         /*
2655          * The dead states indicates the controller was not gracefully
2656          * disconnected. In that case, we won't be able to flush any data while
2657          * removing the namespaces' disks; fail all the queues now to avoid
2658          * potentially having to clean up the failed sync later.
2659          */
2660         if (ctrl->state == NVME_CTRL_DEAD)
2661                 nvme_kill_queues(ctrl);
2662
2663         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
2664                 nvme_ns_remove(ns);
2665 }
2666 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
2667
2668 static void nvme_async_event_work(struct work_struct *work)
2669 {
2670         struct nvme_ctrl *ctrl =
2671                 container_of(work, struct nvme_ctrl, async_event_work);
2672
2673         spin_lock_irq(&ctrl->lock);
2674         while (ctrl->state == NVME_CTRL_LIVE && ctrl->event_limit > 0) {
2675                 int aer_idx = --ctrl->event_limit;
2676
2677                 spin_unlock_irq(&ctrl->lock);
2678                 ctrl->ops->submit_async_event(ctrl, aer_idx);
2679                 spin_lock_irq(&ctrl->lock);
2680         }
2681         spin_unlock_irq(&ctrl->lock);
2682 }
2683
2684 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
2685 {
2686
2687         u32 csts;
2688
2689         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
2690                 return false;
2691
2692         if (csts == ~0)
2693                 return false;
2694
2695         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
2696 }
2697
2698 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
2699 {
2700         struct nvme_fw_slot_info_log *log;
2701
2702         log = kmalloc(sizeof(*log), GFP_KERNEL);
2703         if (!log)
2704                 return;
2705
2706         if (nvme_get_log(ctrl, NVME_LOG_FW_SLOT, log, sizeof(*log)))
2707                 dev_warn(ctrl->device,
2708                                 "Get FW SLOT INFO log error\n");
2709         kfree(log);
2710 }
2711
2712 static void nvme_fw_act_work(struct work_struct *work)
2713 {
2714         struct nvme_ctrl *ctrl = container_of(work,
2715                                 struct nvme_ctrl, fw_act_work);
2716         unsigned long fw_act_timeout;
2717
2718         if (ctrl->mtfa)
2719                 fw_act_timeout = jiffies +
2720                                 msecs_to_jiffies(ctrl->mtfa * 100);
2721         else
2722                 fw_act_timeout = jiffies +
2723                                 msecs_to_jiffies(admin_timeout * 1000);
2724
2725         nvme_stop_queues(ctrl);
2726         while (nvme_ctrl_pp_status(ctrl)) {
2727                 if (time_after(jiffies, fw_act_timeout)) {
2728                         dev_warn(ctrl->device,
2729                                 "Fw activation timeout, reset controller\n");
2730                         nvme_reset_ctrl(ctrl);
2731                         break;
2732                 }
2733                 msleep(100);
2734         }
2735
2736         if (ctrl->state != NVME_CTRL_LIVE)
2737                 return;
2738
2739         nvme_start_queues(ctrl);
2740         /* read FW slot information to clear the AER */
2741         nvme_get_fw_slot_info(ctrl);
2742 }
2743
2744 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
2745                 union nvme_result *res)
2746 {
2747         u32 result = le32_to_cpu(res->u32);
2748         bool done = true;
2749
2750         switch (le16_to_cpu(status) >> 1) {
2751         case NVME_SC_SUCCESS:
2752                 done = false;
2753                 /*FALLTHRU*/
2754         case NVME_SC_ABORT_REQ:
2755                 ++ctrl->event_limit;
2756                 if (ctrl->state == NVME_CTRL_LIVE)
2757                         queue_work(nvme_wq, &ctrl->async_event_work);
2758                 break;
2759         default:
2760                 break;
2761         }
2762
2763         if (done)
2764                 return;
2765
2766         switch (result & 0xff07) {
2767         case NVME_AER_NOTICE_NS_CHANGED:
2768                 dev_info(ctrl->device, "rescanning\n");
2769                 nvme_queue_scan(ctrl);
2770                 break;
2771         case NVME_AER_NOTICE_FW_ACT_STARTING:
2772                 queue_work(nvme_wq, &ctrl->fw_act_work);
2773                 break;
2774         default:
2775                 dev_warn(ctrl->device, "async event result %08x\n", result);
2776         }
2777 }
2778 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
2779
2780 void nvme_queue_async_events(struct nvme_ctrl *ctrl)
2781 {
2782         ctrl->event_limit = NVME_NR_AERS;
2783         queue_work(nvme_wq, &ctrl->async_event_work);
2784 }
2785 EXPORT_SYMBOL_GPL(nvme_queue_async_events);
2786
2787 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
2788 {
2789         nvme_stop_keep_alive(ctrl);
2790         flush_work(&ctrl->async_event_work);
2791         flush_work(&ctrl->scan_work);
2792         cancel_work_sync(&ctrl->fw_act_work);
2793 }
2794 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
2795
2796 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
2797 {
2798         if (ctrl->kato)
2799                 nvme_start_keep_alive(ctrl);
2800
2801         if (ctrl->queue_count > 1) {
2802                 nvme_queue_scan(ctrl);
2803                 nvme_queue_async_events(ctrl);
2804                 nvme_start_queues(ctrl);
2805         }
2806 }
2807 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
2808
2809 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
2810 {
2811         cdev_device_del(&ctrl->cdev, ctrl->device);
2812 }
2813 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
2814
2815 static void nvme_free_ctrl(struct device *dev)
2816 {
2817         struct nvme_ctrl *ctrl =
2818                 container_of(dev, struct nvme_ctrl, ctrl_device);
2819
2820         ida_simple_remove(&nvme_instance_ida, ctrl->instance);
2821         ida_destroy(&ctrl->ns_ida);
2822         kfree(ctrl->effects);
2823
2824         ctrl->ops->free_ctrl(ctrl);
2825 }
2826
2827 /*
2828  * Initialize a NVMe controller structures.  This needs to be called during
2829  * earliest initialization so that we have the initialized structured around
2830  * during probing.
2831  */
2832 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
2833                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
2834 {
2835         int ret;
2836
2837         ctrl->state = NVME_CTRL_NEW;
2838         spin_lock_init(&ctrl->lock);
2839         INIT_LIST_HEAD(&ctrl->namespaces);
2840         mutex_init(&ctrl->namespaces_mutex);
2841         ctrl->dev = dev;
2842         ctrl->ops = ops;
2843         ctrl->quirks = quirks;
2844         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
2845         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
2846         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
2847         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
2848
2849         ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
2850         if (ret < 0)
2851                 goto out;
2852         ctrl->instance = ret;
2853
2854         device_initialize(&ctrl->ctrl_device);
2855         ctrl->device = &ctrl->ctrl_device;
2856         ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
2857         ctrl->device->class = nvme_class;
2858         ctrl->device->parent = ctrl->dev;
2859         ctrl->device->groups = nvme_dev_attr_groups;
2860         ctrl->device->release = nvme_free_ctrl;
2861         dev_set_drvdata(ctrl->device, ctrl);
2862         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
2863         if (ret)
2864                 goto out_release_instance;
2865
2866         cdev_init(&ctrl->cdev, &nvme_dev_fops);
2867         ctrl->cdev.owner = ops->module;
2868         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
2869         if (ret)
2870                 goto out_free_name;
2871
2872         ida_init(&ctrl->ns_ida);
2873
2874         /*
2875          * Initialize latency tolerance controls.  The sysfs files won't
2876          * be visible to userspace unless the device actually supports APST.
2877          */
2878         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
2879         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
2880                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
2881
2882         return 0;
2883 out_free_name:
2884         kfree_const(dev->kobj.name);
2885 out_release_instance:
2886         ida_simple_remove(&nvme_instance_ida, ctrl->instance);
2887 out:
2888         return ret;
2889 }
2890 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
2891
2892 /**
2893  * nvme_kill_queues(): Ends all namespace queues
2894  * @ctrl: the dead controller that needs to end
2895  *
2896  * Call this function when the driver determines it is unable to get the
2897  * controller in a state capable of servicing IO.
2898  */
2899 void nvme_kill_queues(struct nvme_ctrl *ctrl)
2900 {
2901         struct nvme_ns *ns;
2902
2903         mutex_lock(&ctrl->namespaces_mutex);
2904
2905         /* Forcibly unquiesce queues to avoid blocking dispatch */
2906         if (ctrl->admin_q)
2907                 blk_mq_unquiesce_queue(ctrl->admin_q);
2908
2909         list_for_each_entry(ns, &ctrl->namespaces, list) {
2910                 /*
2911                  * Revalidating a dead namespace sets capacity to 0. This will
2912                  * end buffered writers dirtying pages that can't be synced.
2913                  */
2914                 if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
2915                         continue;
2916                 revalidate_disk(ns->disk);
2917                 blk_set_queue_dying(ns->queue);
2918
2919                 /* Forcibly unquiesce queues to avoid blocking dispatch */
2920                 blk_mq_unquiesce_queue(ns->queue);
2921         }
2922         mutex_unlock(&ctrl->namespaces_mutex);
2923 }
2924 EXPORT_SYMBOL_GPL(nvme_kill_queues);
2925
2926 void nvme_unfreeze(struct nvme_ctrl *ctrl)
2927 {
2928         struct nvme_ns *ns;
2929
2930         mutex_lock(&ctrl->namespaces_mutex);
2931         list_for_each_entry(ns, &ctrl->namespaces, list)
2932                 blk_mq_unfreeze_queue(ns->queue);
2933         mutex_unlock(&ctrl->namespaces_mutex);
2934 }
2935 EXPORT_SYMBOL_GPL(nvme_unfreeze);
2936
2937 void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
2938 {
2939         struct nvme_ns *ns;
2940
2941         mutex_lock(&ctrl->namespaces_mutex);
2942         list_for_each_entry(ns, &ctrl->namespaces, list) {
2943                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
2944                 if (timeout <= 0)
2945                         break;
2946         }
2947         mutex_unlock(&ctrl->namespaces_mutex);
2948 }
2949 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
2950
2951 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
2952 {
2953         struct nvme_ns *ns;
2954
2955         mutex_lock(&ctrl->namespaces_mutex);
2956         list_for_each_entry(ns, &ctrl->namespaces, list)
2957                 blk_mq_freeze_queue_wait(ns->queue);
2958         mutex_unlock(&ctrl->namespaces_mutex);
2959 }
2960 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
2961
2962 void nvme_start_freeze(struct nvme_ctrl *ctrl)
2963 {
2964         struct nvme_ns *ns;
2965
2966         mutex_lock(&ctrl->namespaces_mutex);
2967         list_for_each_entry(ns, &ctrl->namespaces, list)
2968                 blk_freeze_queue_start(ns->queue);
2969         mutex_unlock(&ctrl->namespaces_mutex);
2970 }
2971 EXPORT_SYMBOL_GPL(nvme_start_freeze);
2972
2973 void nvme_stop_queues(struct nvme_ctrl *ctrl)
2974 {
2975         struct nvme_ns *ns;
2976
2977         mutex_lock(&ctrl->namespaces_mutex);
2978         list_for_each_entry(ns, &ctrl->namespaces, list)
2979                 blk_mq_quiesce_queue(ns->queue);
2980         mutex_unlock(&ctrl->namespaces_mutex);
2981 }
2982 EXPORT_SYMBOL_GPL(nvme_stop_queues);
2983
2984 void nvme_start_queues(struct nvme_ctrl *ctrl)
2985 {
2986         struct nvme_ns *ns;
2987
2988         mutex_lock(&ctrl->namespaces_mutex);
2989         list_for_each_entry(ns, &ctrl->namespaces, list)
2990                 blk_mq_unquiesce_queue(ns->queue);
2991         mutex_unlock(&ctrl->namespaces_mutex);
2992 }
2993 EXPORT_SYMBOL_GPL(nvme_start_queues);
2994
2995 int nvme_reinit_tagset(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set)
2996 {
2997         if (!ctrl->ops->reinit_request)
2998                 return 0;
2999
3000         return blk_mq_tagset_iter(set, set->driver_data,
3001                         ctrl->ops->reinit_request);
3002 }
3003 EXPORT_SYMBOL_GPL(nvme_reinit_tagset);
3004
3005 int __init nvme_core_init(void)
3006 {
3007         int result;
3008
3009         nvme_wq = alloc_workqueue("nvme-wq",
3010                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
3011         if (!nvme_wq)
3012                 return -ENOMEM;
3013
3014         result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
3015         if (result < 0)
3016                 goto destroy_wq;
3017
3018         nvme_class = class_create(THIS_MODULE, "nvme");
3019         if (IS_ERR(nvme_class)) {
3020                 result = PTR_ERR(nvme_class);
3021                 goto unregister_chrdev;
3022         }
3023
3024         return 0;
3025
3026 unregister_chrdev:
3027         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3028 destroy_wq:
3029         destroy_workqueue(nvme_wq);
3030         return result;
3031 }
3032
3033 void nvme_core_exit(void)
3034 {
3035         class_destroy(nvme_class);
3036         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3037         destroy_workqueue(nvme_wq);
3038 }
3039
3040 MODULE_LICENSE("GPL");
3041 MODULE_VERSION("1.0");
3042 module_init(nvme_core_init);
3043 module_exit(nvme_core_exit);