1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVM Express device driver 4 * Copyright (c) 2011-2014, Intel Corporation. 5 */ 6 7 #include <linux/acpi.h> 8 #include <linux/async.h> 9 #include <linux/blkdev.h> 10 #include <linux/blk-mq-dma.h> 11 #include <linux/blk-integrity.h> 12 #include <linux/dmi.h> 13 #include <linux/init.h> 14 #include <linux/interrupt.h> 15 #include <linux/io.h> 16 #include <linux/kstrtox.h> 17 #include <linux/memremap.h> 18 #include <linux/mm.h> 19 #include <linux/module.h> 20 #include <linux/mutex.h> 21 #include <linux/nodemask.h> 22 #include <linux/once.h> 23 #include <linux/pci.h> 24 #include <linux/suspend.h> 25 #include <linux/t10-pi.h> 26 #include <linux/types.h> 27 #include <linux/io-64-nonatomic-lo-hi.h> 28 #include <linux/io-64-nonatomic-hi-lo.h> 29 #include <linux/sed-opal.h> 30 31 #include "trace.h" 32 #include "nvme.h" 33 #include "pci-dmabuf.h" 34 35 #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes) 36 #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion)) 37 38 /* Optimisation for I/Os between 4k and 128k */ 39 #define NVME_SMALL_POOL_SIZE 256 40 41 /* 42 * Arbitrary upper bound. 43 */ 44 #define NVME_MAX_BYTES SZ_8M 45 #define NVME_MAX_NR_DESCRIPTORS 5 46 47 /* 48 * For data SGLs we support a single descriptors worth of SGL entries. 49 * For PRPs, segments don't matter at all. 50 */ 51 #define NVME_MAX_SEGS \ 52 (NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc)) 53 54 /* 55 * For metadata SGLs, only the small descriptor is supported, and the first 56 * entry is the segment descriptor, which for the data pointer sits in the SQE. 57 */ 58 #define NVME_MAX_META_SEGS \ 59 ((NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc)) - 1) 60 61 /* 62 * The last entry is used to link to the next descriptor. 63 */ 64 #define PRPS_PER_PAGE \ 65 (((NVME_CTRL_PAGE_SIZE / sizeof(__le64))) - 1) 66 67 /* 68 * I/O could be non-aligned both at the beginning and end. 69 */ 70 #define MAX_PRP_RANGE \ 71 (NVME_MAX_BYTES + 2 * (NVME_CTRL_PAGE_SIZE - 1)) 72 73 static_assert(MAX_PRP_RANGE / NVME_CTRL_PAGE_SIZE <= 74 (1 /* prp1 */ + NVME_MAX_NR_DESCRIPTORS * PRPS_PER_PAGE)); 75 76 struct quirk_entry { 77 u16 vendor_id; 78 u16 dev_id; 79 u32 enabled_quirks; 80 u32 disabled_quirks; 81 }; 82 83 static int use_threaded_interrupts; 84 module_param(use_threaded_interrupts, int, 0444); 85 86 static bool use_cmb_sqes = true; 87 module_param(use_cmb_sqes, bool, 0444); 88 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes"); 89 90 static unsigned int max_host_mem_size_mb = 128; 91 module_param(max_host_mem_size_mb, uint, 0444); 92 MODULE_PARM_DESC(max_host_mem_size_mb, 93 "Maximum Host Memory Buffer (HMB) size per controller (in MiB)"); 94 95 static unsigned int sgl_threshold = SZ_32K; 96 module_param(sgl_threshold, uint, 0644); 97 MODULE_PARM_DESC(sgl_threshold, 98 "Use SGLs when average request segment size is larger or equal to " 99 "this size. Use 0 to disable SGLs."); 100 101 #define NVME_PCI_MIN_QUEUE_SIZE 2 102 #define NVME_PCI_MAX_QUEUE_SIZE 4095 103 static int io_queue_depth_set(const char *val, const struct kernel_param *kp); 104 static const struct kernel_param_ops io_queue_depth_ops = { 105 .set = io_queue_depth_set, 106 .get = param_get_uint, 107 }; 108 109 static unsigned int io_queue_depth = 1024; 110 module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); 111 MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2 and < 4096"); 112 113 static struct quirk_entry *nvme_pci_quirk_list; 114 static unsigned int nvme_pci_quirk_count; 115 116 /* Helper to parse individual quirk names */ 117 static int nvme_parse_quirk_names(char *quirk_str, struct quirk_entry *entry) 118 { 119 int i; 120 size_t field_len; 121 bool disabled, found; 122 char *p = quirk_str, *field; 123 124 while ((field = strsep(&p, ",")) && *field) { 125 disabled = false; 126 found = false; 127 128 if (*field == '^') { 129 /* Skip the '^' character */ 130 disabled = true; 131 field++; 132 } 133 134 field_len = strlen(field); 135 for (i = 0; i < 32; i++) { 136 unsigned int bit = 1U << i; 137 char *q_name = nvme_quirk_name(bit); 138 size_t q_len = strlen(q_name); 139 140 if (!strcmp(q_name, "unknown")) 141 break; 142 143 if (!strcmp(q_name, field) && 144 q_len == field_len) { 145 if (disabled) 146 entry->disabled_quirks |= bit; 147 else 148 entry->enabled_quirks |= bit; 149 found = true; 150 break; 151 } 152 } 153 154 if (!found) { 155 pr_err("nvme: unrecognized quirk %s\n", field); 156 return -EINVAL; 157 } 158 } 159 return 0; 160 } 161 162 /* Helper to parse a single VID:DID:quirk_names entry */ 163 static int nvme_parse_quirk_entry(char *s, struct quirk_entry *entry) 164 { 165 char *field; 166 167 field = strsep(&s, ":"); 168 if (!field || kstrtou16(field, 16, &entry->vendor_id)) 169 return -EINVAL; 170 171 field = strsep(&s, ":"); 172 if (!field || kstrtou16(field, 16, &entry->dev_id)) 173 return -EINVAL; 174 175 field = strsep(&s, ":"); 176 if (!field) 177 return -EINVAL; 178 179 return nvme_parse_quirk_names(field, entry); 180 } 181 182 static int quirks_param_set(const char *value, const struct kernel_param *kp) 183 { 184 int count, err, i; 185 struct quirk_entry *qlist; 186 char *field, *val, *sep_ptr; 187 188 err = param_set_copystring(value, kp); 189 if (err) 190 return err; 191 192 val = kstrdup(value, GFP_KERNEL); 193 if (!val) 194 return -ENOMEM; 195 196 if (!*val) 197 goto out_free_val; 198 199 count = 1; 200 for (i = 0; val[i]; i++) { 201 if (val[i] == '-') 202 count++; 203 } 204 205 qlist = kzalloc_objs(*qlist, count); 206 if (!qlist) { 207 err = -ENOMEM; 208 goto out_free_val; 209 } 210 211 i = 0; 212 sep_ptr = val; 213 while ((field = strsep(&sep_ptr, "-"))) { 214 if (nvme_parse_quirk_entry(field, &qlist[i])) { 215 pr_err("nvme: failed to parse quirk string %s\n", 216 value); 217 err = -EINVAL; 218 goto out_free_qlist; 219 } 220 221 i++; 222 } 223 224 kfree(nvme_pci_quirk_list); 225 nvme_pci_quirk_count = count; 226 nvme_pci_quirk_list = qlist; 227 goto out_free_val; 228 229 out_free_qlist: 230 kfree(qlist); 231 out_free_val: 232 kfree(val); 233 return err; 234 } 235 236 static char quirks_param[256]; 237 static const struct kernel_param_ops quirks_param_ops = { 238 .set = quirks_param_set, 239 .get = param_get_string, 240 }; 241 242 static struct kparam_string quirks_param_string = { 243 .maxlen = sizeof(quirks_param), 244 .string = quirks_param, 245 }; 246 247 device_param_cb(quirks, &quirks_param_ops, &quirks_param_string, 0444); 248 MODULE_PARM_DESC(quirks, "Enable/disable NVMe quirks by specifying " 249 "quirks=VID:DID:quirk_names"); 250 251 static int io_queue_count_set(const char *val, const struct kernel_param *kp) 252 { 253 unsigned int n; 254 int ret; 255 256 ret = kstrtouint(val, 10, &n); 257 if (ret != 0 || n > blk_mq_num_possible_queues(0)) 258 return -EINVAL; 259 return param_set_uint(val, kp); 260 } 261 262 static const struct kernel_param_ops io_queue_count_ops = { 263 .set = io_queue_count_set, 264 .get = param_get_uint, 265 }; 266 267 static unsigned int write_queues; 268 module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644); 269 MODULE_PARM_DESC(write_queues, 270 "Number of queues to use for writes. If not set, reads and writes " 271 "will share a queue set."); 272 273 static unsigned int poll_queues; 274 module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644); 275 MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO."); 276 277 static bool noacpi; 278 module_param(noacpi, bool, 0444); 279 MODULE_PARM_DESC(noacpi, "disable acpi bios quirks"); 280 281 struct nvme_dev; 282 struct nvme_queue; 283 284 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown); 285 static void nvme_delete_io_queues(struct nvme_dev *dev); 286 static void nvme_update_attrs(struct nvme_dev *dev); 287 288 struct nvme_descriptor_pools { 289 struct dma_pool *large; 290 struct dma_pool *small; 291 }; 292 293 /* 294 * Represents an NVM Express device. Each nvme_dev is a PCI function. 295 */ 296 struct nvme_dev { 297 struct nvme_queue *queues; 298 struct blk_mq_tag_set tagset; 299 struct blk_mq_tag_set admin_tagset; 300 u32 __iomem *dbs; 301 struct device *dev; 302 unsigned online_queues; 303 unsigned max_qid; 304 unsigned io_queues[HCTX_MAX_TYPES]; 305 unsigned int num_vecs; 306 u32 q_depth; 307 int io_sqes; 308 u32 db_stride; 309 void __iomem *bar; 310 unsigned long bar_mapped_size; 311 struct mutex shutdown_lock; 312 bool subsystem; 313 u64 cmb_size; 314 bool cmb_use_sqes; 315 u32 cmbsz; 316 u32 cmbloc; 317 struct nvme_ctrl ctrl; 318 u32 last_ps; 319 bool hmb; 320 struct sg_table *hmb_sgt; 321 mempool_t *dmavec_mempool; 322 struct nvme_dma_buf_list dmabufs; 323 324 /* shadow doorbell buffer support: */ 325 __le32 *dbbuf_dbs; 326 dma_addr_t dbbuf_dbs_dma_addr; 327 __le32 *dbbuf_eis; 328 dma_addr_t dbbuf_eis_dma_addr; 329 330 /* host memory buffer support: */ 331 u64 host_mem_size; 332 u32 nr_host_mem_descs; 333 u32 host_mem_descs_size; 334 dma_addr_t host_mem_descs_dma; 335 struct nvme_host_mem_buf_desc *host_mem_descs; 336 void **host_mem_desc_bufs; 337 unsigned int nr_allocated_queues; 338 unsigned int nr_write_queues; 339 unsigned int nr_poll_queues; 340 struct nvme_descriptor_pools descriptor_pools[]; 341 }; 342 343 static int io_queue_depth_set(const char *val, const struct kernel_param *kp) 344 { 345 return param_set_uint_minmax(val, kp, NVME_PCI_MIN_QUEUE_SIZE, 346 NVME_PCI_MAX_QUEUE_SIZE); 347 } 348 349 static inline unsigned int sq_idx(unsigned int qid, u32 stride) 350 { 351 return qid * 2 * stride; 352 } 353 354 static inline unsigned int cq_idx(unsigned int qid, u32 stride) 355 { 356 return (qid * 2 + 1) * stride; 357 } 358 359 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) 360 { 361 return container_of(ctrl, struct nvme_dev, ctrl); 362 } 363 364 /* 365 * An NVM Express queue. Each device has at least two (one for admin 366 * commands and one for I/O commands). 367 */ 368 struct nvme_queue { 369 struct nvme_dev *dev; 370 struct nvme_descriptor_pools descriptor_pools; 371 spinlock_t sq_lock; 372 void *sq_cmds 373 __guarded_by(&sq_lock); 374 /* only used for poll queues: */ 375 spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; 376 struct nvme_completion *cqes; 377 dma_addr_t sq_dma_addr; 378 dma_addr_t cq_dma_addr; 379 u32 __iomem *q_db; 380 u32 q_depth; 381 u16 cq_vector; 382 u16 cq_head; 383 u16 sq_tail 384 __guarded_by(&sq_lock); 385 u16 last_sq_tail 386 __guarded_by(&sq_lock); 387 u16 qid; 388 u8 cq_phase; 389 u8 sqes; 390 unsigned long flags; 391 #define NVMEQ_ENABLED 0 392 #define NVMEQ_SQ_CMB 1 393 #define NVMEQ_POLLED 3 394 __le32 *dbbuf_sq_db; 395 __le32 *dbbuf_cq_db; 396 __le32 *dbbuf_sq_ei; 397 __le32 *dbbuf_cq_ei; 398 struct completion delete_done; 399 }; 400 401 /* bits for iod->flags */ 402 enum nvme_iod_flags { 403 /* this command has been aborted by the timeout handler */ 404 IOD_ABORTED = 1U << 0, 405 406 /* uses the small descriptor pool */ 407 IOD_SMALL_DESCRIPTOR = 1U << 1, 408 409 /* single segment dma mapping */ 410 IOD_SINGLE_SEGMENT = 1U << 2, 411 412 /* Data payload contains p2p memory */ 413 IOD_DATA_P2P = 1U << 3, 414 415 /* Metadata contains p2p memory */ 416 IOD_META_P2P = 1U << 4, 417 418 /* Data payload contains MMIO memory */ 419 IOD_DATA_MMIO = 1U << 5, 420 421 /* Metadata contains MMIO memory */ 422 IOD_META_MMIO = 1U << 6, 423 424 /* Metadata using non-coalesced MPTR */ 425 IOD_SINGLE_META_SEGMENT = 1U << 7, 426 }; 427 428 struct nvme_dma_vec { 429 dma_addr_t addr; 430 unsigned int len; 431 }; 432 433 /* 434 * The nvme_iod describes the data in an I/O. 435 */ 436 struct nvme_iod { 437 struct nvme_request req; 438 struct nvme_command cmd; 439 u8 flags; 440 u8 nr_descriptors; 441 442 size_t total_len; 443 struct dma_iova_state dma_state; 444 void *descriptors[NVME_MAX_NR_DESCRIPTORS]; 445 struct nvme_dma_vec *dma_vecs; 446 unsigned int nr_dma_vecs; 447 448 dma_addr_t meta_dma; 449 size_t meta_total_len; 450 struct dma_iova_state meta_dma_state; 451 struct nvme_sgl_desc *meta_descriptor; 452 }; 453 454 static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) 455 { 456 return dev->nr_allocated_queues * 8 * dev->db_stride; 457 } 458 459 static void nvme_dbbuf_dma_alloc(struct nvme_dev *dev) 460 { 461 unsigned int mem_size = nvme_dbbuf_size(dev); 462 463 if (!(dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP)) 464 return; 465 466 if (dev->dbbuf_dbs) { 467 /* 468 * Clear the dbbuf memory so the driver doesn't observe stale 469 * values from the previous instantiation. 470 */ 471 memset(dev->dbbuf_dbs, 0, mem_size); 472 memset(dev->dbbuf_eis, 0, mem_size); 473 return; 474 } 475 476 dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size, 477 &dev->dbbuf_dbs_dma_addr, 478 GFP_KERNEL); 479 if (!dev->dbbuf_dbs) 480 goto fail; 481 dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size, 482 &dev->dbbuf_eis_dma_addr, 483 GFP_KERNEL); 484 if (!dev->dbbuf_eis) 485 goto fail_free_dbbuf_dbs; 486 return; 487 488 fail_free_dbbuf_dbs: 489 dma_free_coherent(dev->dev, mem_size, dev->dbbuf_dbs, 490 dev->dbbuf_dbs_dma_addr); 491 dev->dbbuf_dbs = NULL; 492 fail: 493 dev_warn(dev->dev, "unable to allocate dma for dbbuf\n"); 494 } 495 496 static void nvme_dbbuf_dma_free(struct nvme_dev *dev) 497 { 498 unsigned int mem_size = nvme_dbbuf_size(dev); 499 500 if (dev->dbbuf_dbs) { 501 dma_free_coherent(dev->dev, mem_size, 502 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr); 503 dev->dbbuf_dbs = NULL; 504 } 505 if (dev->dbbuf_eis) { 506 dma_free_coherent(dev->dev, mem_size, 507 dev->dbbuf_eis, dev->dbbuf_eis_dma_addr); 508 dev->dbbuf_eis = NULL; 509 } 510 } 511 512 static void nvme_dbbuf_init(struct nvme_dev *dev, 513 struct nvme_queue *nvmeq, int qid) 514 { 515 if (!dev->dbbuf_dbs || !qid) 516 return; 517 518 nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)]; 519 nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)]; 520 nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)]; 521 nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)]; 522 } 523 524 static void nvme_dbbuf_free(struct nvme_queue *nvmeq) 525 { 526 if (!nvmeq->qid) 527 return; 528 529 nvmeq->dbbuf_sq_db = NULL; 530 nvmeq->dbbuf_cq_db = NULL; 531 nvmeq->dbbuf_sq_ei = NULL; 532 nvmeq->dbbuf_cq_ei = NULL; 533 } 534 535 static void nvme_dbbuf_set(struct nvme_dev *dev) 536 { 537 struct nvme_command c = { }; 538 unsigned int i; 539 540 if (!dev->dbbuf_dbs) 541 return; 542 543 c.dbbuf.opcode = nvme_admin_dbbuf; 544 c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr); 545 c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr); 546 547 if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) { 548 dev_warn(dev->ctrl.device, "unable to set dbbuf\n"); 549 /* Free memory and continue on */ 550 nvme_dbbuf_dma_free(dev); 551 552 for (i = 1; i < dev->online_queues; i++) 553 nvme_dbbuf_free(&dev->queues[i]); 554 } 555 } 556 557 static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old) 558 { 559 return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old); 560 } 561 562 /* Update dbbuf and return true if an MMIO is required */ 563 static bool nvme_dbbuf_update_and_check_event(u16 value, __le32 *dbbuf_db, 564 volatile __le32 *dbbuf_ei) 565 { 566 if (dbbuf_db) { 567 u16 old_value, event_idx; 568 569 /* 570 * Ensure that the queue is written before updating 571 * the doorbell in memory 572 */ 573 wmb(); 574 575 old_value = le32_to_cpu(*dbbuf_db); 576 *dbbuf_db = cpu_to_le32(value); 577 578 /* 579 * Ensure that the doorbell is updated before reading the event 580 * index from memory. The controller needs to provide similar 581 * ordering to ensure the event index is updated before reading 582 * the doorbell. 583 */ 584 mb(); 585 586 event_idx = le32_to_cpu(*dbbuf_ei); 587 if (!nvme_dbbuf_need_event(event_idx, value, old_value)) 588 return false; 589 } 590 591 return true; 592 } 593 594 static struct nvme_descriptor_pools * 595 nvme_setup_descriptor_pools(struct nvme_dev *dev, int numa_node) 596 { 597 struct nvme_descriptor_pools *pools; 598 size_t small_align = NVME_SMALL_POOL_SIZE; 599 600 if (numa_node == NUMA_NO_NODE) 601 numa_node = 0; 602 603 pools = &dev->descriptor_pools[numa_node]; 604 605 if (pools->small) 606 return pools; /* already initialized */ 607 608 pools->large = dma_pool_create_node("nvme descriptor page", dev->dev, 609 NVME_CTRL_PAGE_SIZE, NVME_CTRL_PAGE_SIZE, 0, numa_node); 610 if (!pools->large) 611 return ERR_PTR(-ENOMEM); 612 613 if (dev->ctrl.quirks & NVME_QUIRK_DMAPOOL_ALIGN_512) 614 small_align = 512; 615 616 pools->small = dma_pool_create_node("nvme descriptor small", dev->dev, 617 NVME_SMALL_POOL_SIZE, small_align, 0, numa_node); 618 if (!pools->small) { 619 dma_pool_destroy(pools->large); 620 pools->large = NULL; 621 return ERR_PTR(-ENOMEM); 622 } 623 624 return pools; 625 } 626 627 static void nvme_release_descriptor_pools(struct nvme_dev *dev) 628 { 629 unsigned i; 630 631 for (i = 0; i < nr_node_ids; i++) { 632 struct nvme_descriptor_pools *pools = &dev->descriptor_pools[i]; 633 634 dma_pool_destroy(pools->large); 635 dma_pool_destroy(pools->small); 636 } 637 } 638 639 static int nvme_init_hctx_common(struct blk_mq_hw_ctx *hctx, void *data, 640 unsigned qid) 641 { 642 struct nvme_dev *dev = to_nvme_dev(data); 643 struct nvme_queue *nvmeq = &dev->queues[qid]; 644 struct nvme_descriptor_pools *pools; 645 struct blk_mq_tags *tags; 646 647 tags = qid ? dev->tagset.tags[qid - 1] : dev->admin_tagset.tags[0]; 648 WARN_ON(tags != hctx->tags); 649 pools = nvme_setup_descriptor_pools(dev, hctx->numa_node); 650 if (IS_ERR(pools)) 651 return PTR_ERR(pools); 652 653 nvmeq->descriptor_pools = *pools; 654 hctx->driver_data = nvmeq; 655 return 0; 656 } 657 658 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 659 unsigned int hctx_idx) 660 { 661 WARN_ON(hctx_idx != 0); 662 return nvme_init_hctx_common(hctx, data, 0); 663 } 664 665 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 666 unsigned int hctx_idx) 667 { 668 return nvme_init_hctx_common(hctx, data, hctx_idx + 1); 669 } 670 671 static int nvme_pci_init_request(struct blk_mq_tag_set *set, 672 struct request *req, unsigned int hctx_idx, 673 int numa_node) 674 { 675 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 676 677 nvme_req(req)->ctrl = set->driver_data; 678 nvme_req(req)->cmd = &iod->cmd; 679 return 0; 680 } 681 682 static int queue_irq_offset(struct nvme_dev *dev) 683 { 684 /* if we have more than 1 vec, admin queue offsets us by 1 */ 685 if (dev->num_vecs > 1) 686 return 1; 687 688 return 0; 689 } 690 691 static void nvme_pci_map_queues(struct blk_mq_tag_set *set) 692 { 693 struct nvme_dev *dev = to_nvme_dev(set->driver_data); 694 int i, qoff, offset; 695 696 offset = queue_irq_offset(dev); 697 for (i = 0, qoff = 0; i < set->nr_maps; i++) { 698 struct blk_mq_queue_map *map = &set->map[i]; 699 700 map->nr_queues = dev->io_queues[i]; 701 if (!map->nr_queues) { 702 BUG_ON(i == HCTX_TYPE_DEFAULT); 703 continue; 704 } 705 706 /* 707 * The poll queue(s) doesn't have an IRQ (and hence IRQ 708 * affinity), so use the regular blk-mq cpu mapping 709 */ 710 map->queue_offset = qoff; 711 if (i != HCTX_TYPE_POLL && offset) 712 blk_mq_map_hw_queues(map, dev->dev, offset); 713 else 714 blk_mq_map_queues(map); 715 qoff += map->nr_queues; 716 offset += map->nr_queues; 717 } 718 } 719 720 /* 721 * Write sq tail if we are asked to, or if the next command would wrap. 722 */ 723 static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) 724 __must_hold(&nvmeq->sq_lock) 725 { 726 if (!write_sq) { 727 u16 next_tail = nvmeq->sq_tail + 1; 728 729 if (next_tail == nvmeq->q_depth) 730 next_tail = 0; 731 if (next_tail != nvmeq->last_sq_tail) 732 return; 733 } 734 735 if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail, 736 nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei)) 737 writel(nvmeq->sq_tail, nvmeq->q_db); 738 nvmeq->last_sq_tail = nvmeq->sq_tail; 739 } 740 741 static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq, 742 struct nvme_command *cmd) 743 __must_hold(&nvmeq->sq_lock) 744 { 745 memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes), 746 absolute_pointer(cmd), sizeof(*cmd)); 747 if (++nvmeq->sq_tail == nvmeq->q_depth) 748 nvmeq->sq_tail = 0; 749 } 750 751 static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx) 752 { 753 struct nvme_queue *nvmeq = hctx->driver_data; 754 755 spin_lock(&nvmeq->sq_lock); 756 if (nvmeq->sq_tail != nvmeq->last_sq_tail) 757 nvme_write_sq_db(nvmeq, true); 758 spin_unlock(&nvmeq->sq_lock); 759 } 760 761 enum nvme_use_sgl { 762 SGL_UNSUPPORTED, 763 SGL_SUPPORTED, 764 SGL_FORCED, 765 }; 766 767 static inline bool nvme_pci_metadata_use_sgls(struct request *req) 768 { 769 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 770 struct nvme_dev *dev = nvmeq->dev; 771 772 if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl)) 773 return false; 774 return req->nr_integrity_segments > 1 || 775 nvme_req(req)->flags & NVME_REQ_USERCMD; 776 } 777 778 static inline enum nvme_use_sgl nvme_pci_use_sgls(struct nvme_dev *dev, 779 struct request *req) 780 { 781 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 782 783 if (nvmeq->qid && nvme_ctrl_sgl_supported(&dev->ctrl)) { 784 /* 785 * When the controller is capable of using SGL, there are 786 * several conditions that we force to use it: 787 * 788 * 1. A request containing page gaps within the controller's 789 * mask can not use the PRP format. 790 * 791 * 2. User commands use SGL because that lets the device 792 * validate the requested transfer lengths. 793 * 794 * 3. Multiple integrity segments must use SGL as that's the 795 * only way to describe such a command in NVMe. 796 */ 797 if (req_phys_gap_mask(req) & (NVME_CTRL_PAGE_SIZE - 1) || 798 nvme_req(req)->flags & NVME_REQ_USERCMD || 799 req->nr_integrity_segments > 1) 800 return SGL_FORCED; 801 return SGL_SUPPORTED; 802 } 803 804 return SGL_UNSUPPORTED; 805 } 806 807 static unsigned int nvme_pci_avg_seg_size(struct request *req) 808 { 809 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 810 unsigned int nseg; 811 812 if (blk_rq_dma_map_coalesce(&iod->dma_state)) 813 nseg = 1; 814 else 815 nseg = blk_rq_nr_phys_segments(req); 816 return DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg); 817 } 818 819 static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq, 820 struct nvme_iod *iod) 821 { 822 if (iod->flags & IOD_SMALL_DESCRIPTOR) 823 return nvmeq->descriptor_pools.small; 824 return nvmeq->descriptor_pools.large; 825 } 826 827 static inline bool nvme_pci_cmd_use_meta_sgl(struct nvme_command *cmd) 828 { 829 return (cmd->common.flags & NVME_CMD_SGL_ALL) == NVME_CMD_SGL_METASEG; 830 } 831 832 static inline bool nvme_pci_cmd_use_sgl(struct nvme_command *cmd) 833 { 834 return cmd->common.flags & 835 (NVME_CMD_SGL_METABUF | NVME_CMD_SGL_METASEG); 836 } 837 838 static inline dma_addr_t nvme_pci_first_desc_dma_addr(struct nvme_command *cmd) 839 { 840 if (nvme_pci_cmd_use_sgl(cmd)) 841 return le64_to_cpu(cmd->common.dptr.sgl.addr); 842 return le64_to_cpu(cmd->common.dptr.prp2); 843 } 844 845 static void nvme_free_descriptors(struct request *req) 846 { 847 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 848 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; 849 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 850 dma_addr_t dma_addr = nvme_pci_first_desc_dma_addr(&iod->cmd); 851 int i; 852 853 if (iod->nr_descriptors == 1) { 854 dma_pool_free(nvme_dma_pool(nvmeq, iod), iod->descriptors[0], 855 dma_addr); 856 return; 857 } 858 859 for (i = 0; i < iod->nr_descriptors; i++) { 860 __le64 *prp_list = iod->descriptors[i]; 861 dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]); 862 863 dma_pool_free(nvmeq->descriptor_pools.large, prp_list, 864 dma_addr); 865 dma_addr = next_dma_addr; 866 } 867 } 868 869 static blk_status_t nvme_rq_setup_dmabuf_map(struct request *req, 870 struct nvme_queue *nvmeq) 871 { 872 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 873 struct nvme_dmabuf_map *map = nvme_req_to_map(req); 874 unsigned bvec_done = req->bio->bi_iter.bi_offset; 875 unsigned map_idx = bvec_done / NVME_CTRL_PAGE_SIZE; 876 unsigned offset = bvec_done & (NVME_CTRL_PAGE_SIZE - 1); 877 int length = blk_rq_payload_bytes(req) - (NVME_CTRL_PAGE_SIZE - offset); 878 dma_addr_t *dma_list = map->dma_list; 879 u64 prp1_dma = dma_list[map_idx++] + offset; 880 u64 dma_addr, prp2_dma; 881 dma_addr_t prp_dma; 882 __le64 *prp_list; 883 unsigned i; 884 885 if (length <= 0) { 886 prp2_dma = 0; 887 goto done; 888 } 889 890 if (length <= NVME_CTRL_PAGE_SIZE) { 891 prp2_dma = dma_list[map_idx]; 892 goto done; 893 } 894 895 if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <= 896 NVME_SMALL_POOL_SIZE / sizeof(__le64)) 897 iod->flags |= IOD_SMALL_DESCRIPTOR; 898 899 prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, 900 &prp_dma); 901 if (!prp_list) 902 return BLK_STS_RESOURCE; 903 904 iod->descriptors[iod->nr_descriptors++] = prp_list; 905 prp2_dma = prp_dma; 906 i = 0; 907 for (;;) { 908 if (i == NVME_CTRL_PAGE_SIZE >> 3) { 909 __le64 *old_prp_list = prp_list; 910 911 prp_list = dma_pool_alloc(nvmeq->descriptor_pools.large, 912 GFP_ATOMIC, &prp_dma); 913 if (!prp_list) 914 goto free_prps; 915 iod->descriptors[iod->nr_descriptors++] = prp_list; 916 prp_list[0] = old_prp_list[i - 1]; 917 old_prp_list[i - 1] = cpu_to_le64(prp_dma); 918 i = 1; 919 } 920 921 dma_addr = dma_list[map_idx++]; 922 prp_list[i++] = cpu_to_le64(dma_addr); 923 924 length -= NVME_CTRL_PAGE_SIZE; 925 if (length <= 0) 926 break; 927 } 928 done: 929 iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); 930 iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); 931 return BLK_STS_OK; 932 free_prps: 933 iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); 934 iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); 935 nvme_free_descriptors(req); 936 return BLK_STS_RESOURCE; 937 } 938 939 static void nvme_free_prps(struct request *req, unsigned int attrs) 940 { 941 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 942 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 943 unsigned int i; 944 945 for (i = 0; i < iod->nr_dma_vecs; i++) 946 dma_unmap_phys(nvmeq->dev->dev, iod->dma_vecs[i].addr, 947 iod->dma_vecs[i].len, rq_dma_dir(req), attrs); 948 mempool_free(iod->dma_vecs, nvmeq->dev->dmavec_mempool); 949 } 950 951 static void nvme_free_sgls(struct request *req, struct nvme_sgl_desc *sge, 952 struct nvme_sgl_desc *sg_list, unsigned int attrs) 953 { 954 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 955 enum dma_data_direction dir = rq_dma_dir(req); 956 unsigned int len = le32_to_cpu(sge->length); 957 struct device *dma_dev = nvmeq->dev->dev; 958 unsigned int i; 959 960 if (sge->type == (NVME_SGL_FMT_DATA_DESC << 4)) { 961 dma_unmap_phys(dma_dev, le64_to_cpu(sge->addr), len, dir, 962 attrs); 963 return; 964 } 965 966 for (i = 0; i < len / sizeof(*sg_list); i++) 967 dma_unmap_phys(dma_dev, le64_to_cpu(sg_list[i].addr), 968 le32_to_cpu(sg_list[i].length), dir, attrs); 969 } 970 971 static void nvme_unmap_metadata(struct request *req) 972 { 973 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 974 enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE; 975 enum dma_data_direction dir = rq_dma_dir(req); 976 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 977 struct device *dma_dev = nvmeq->dev->dev; 978 struct nvme_sgl_desc *sge = iod->meta_descriptor; 979 unsigned int attrs = 0; 980 981 if (iod->flags & IOD_SINGLE_META_SEGMENT) { 982 dma_unmap_page(dma_dev, iod->meta_dma, 983 rq_integrity_vec(req).bv_len, 984 rq_dma_dir(req)); 985 return; 986 } 987 988 if (iod->flags & IOD_META_P2P) 989 map = PCI_P2PDMA_MAP_BUS_ADDR; 990 else if (iod->flags & IOD_META_MMIO) { 991 map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; 992 attrs |= DMA_ATTR_MMIO; 993 } 994 995 if (!blk_rq_dma_unmap(req, dma_dev, &iod->meta_dma_state, 996 iod->meta_total_len, map)) { 997 if (nvme_pci_cmd_use_meta_sgl(&iod->cmd)) 998 nvme_free_sgls(req, sge, &sge[1], attrs); 999 else 1000 dma_unmap_phys(dma_dev, iod->meta_dma, 1001 iod->meta_total_len, dir, attrs); 1002 } 1003 1004 if (iod->meta_descriptor) 1005 dma_pool_free(nvmeq->descriptor_pools.small, 1006 iod->meta_descriptor, iod->meta_dma); 1007 } 1008 1009 static void nvme_unmap_data(struct request *req) 1010 { 1011 enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE; 1012 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1013 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1014 struct device *dma_dev = nvmeq->dev->dev; 1015 unsigned int attrs = 0; 1016 1017 if (blk_mq_rq_is_dmabuf(req)) 1018 goto out_free_desc; 1019 1020 if (iod->flags & IOD_SINGLE_SEGMENT) { 1021 static_assert(offsetof(union nvme_data_ptr, prp1) == 1022 offsetof(union nvme_data_ptr, sgl.addr)); 1023 dma_unmap_page(dma_dev, le64_to_cpu(iod->cmd.common.dptr.prp1), 1024 iod->total_len, rq_dma_dir(req)); 1025 return; 1026 } 1027 1028 if (iod->flags & IOD_DATA_P2P) 1029 map = PCI_P2PDMA_MAP_BUS_ADDR; 1030 else if (iod->flags & IOD_DATA_MMIO) { 1031 map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; 1032 attrs |= DMA_ATTR_MMIO; 1033 } 1034 1035 if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len, 1036 map)) { 1037 if (nvme_pci_cmd_use_sgl(&iod->cmd)) 1038 nvme_free_sgls(req, &iod->cmd.common.dptr.sgl, 1039 iod->descriptors[0], attrs); 1040 else 1041 nvme_free_prps(req, attrs); 1042 } 1043 1044 out_free_desc: 1045 if (iod->nr_descriptors) 1046 nvme_free_descriptors(req); 1047 } 1048 1049 static bool nvme_pci_prp_save_mapping(struct request *req, 1050 struct device *dma_dev, 1051 struct blk_dma_iter *iter) 1052 { 1053 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1054 1055 if (dma_use_iova(&iod->dma_state) || !dma_need_unmap(dma_dev) || 1056 (iod->flags & IOD_DATA_P2P)) 1057 return true; 1058 1059 if (!iod->nr_dma_vecs) { 1060 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1061 1062 iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool, 1063 GFP_ATOMIC); 1064 if (!iod->dma_vecs) { 1065 iter->status = BLK_STS_RESOURCE; 1066 return false; 1067 } 1068 } 1069 1070 iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr; 1071 iod->dma_vecs[iod->nr_dma_vecs].len = iter->len; 1072 iod->nr_dma_vecs++; 1073 return true; 1074 } 1075 1076 static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev, 1077 struct blk_dma_iter *iter) 1078 { 1079 if (iter->len) 1080 return true; 1081 if (!blk_rq_dma_map_iter_next(req, dma_dev, iter)) 1082 return false; 1083 return nvme_pci_prp_save_mapping(req, dma_dev, iter); 1084 } 1085 1086 static void nvme_unmap_iter(struct request *req, struct blk_dma_iter *iter, 1087 struct dma_iova_state *state) 1088 { 1089 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1090 struct device *dev = nvmeq->dev->dev; 1091 1092 if (!blk_rq_dma_unmap(req, dev, state, iter->len, iter->p2pdma.map)) { 1093 unsigned int attrs = 0; 1094 1095 if (iter->p2pdma.map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) 1096 attrs |= DMA_ATTR_MMIO; 1097 1098 dma_unmap_phys(dev, iter->addr, iter->len, rq_dma_dir(req), 1099 attrs); 1100 } 1101 } 1102 1103 static blk_status_t nvme_pci_setup_data_prp(struct request *req, 1104 struct blk_dma_iter *iter) 1105 { 1106 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1107 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1108 unsigned int length = blk_rq_payload_bytes(req); 1109 dma_addr_t prp1_dma, prp2_dma = 0; 1110 unsigned int prp_len, i; 1111 __le64 *prp_list; 1112 1113 if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter)) { 1114 nvme_unmap_iter(req, iter, &iod->dma_state); 1115 return iter->status; 1116 } 1117 1118 /* 1119 * PRP1 always points to the start of the DMA transfers. 1120 * 1121 * This is the only PRP (except for the list entries) that could be 1122 * non-aligned. 1123 */ 1124 prp1_dma = iter->addr; 1125 prp_len = min(length, NVME_CTRL_PAGE_SIZE - 1126 (iter->addr & (NVME_CTRL_PAGE_SIZE - 1))); 1127 iod->total_len += prp_len; 1128 iter->addr += prp_len; 1129 iter->len -= prp_len; 1130 length -= prp_len; 1131 if (!length) 1132 goto done; 1133 1134 if (!nvme_pci_prp_iter_next(req, nvmeq->dev->dev, iter)) { 1135 if (WARN_ON_ONCE(!iter->status)) 1136 goto bad_sgl; 1137 goto done; 1138 } 1139 1140 /* 1141 * PRP2 is usually a list, but can point to data if all data to be 1142 * transferred fits into PRP1 + PRP2: 1143 */ 1144 if (length <= NVME_CTRL_PAGE_SIZE) { 1145 prp2_dma = iter->addr; 1146 iod->total_len += length; 1147 goto done; 1148 } 1149 1150 if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <= 1151 NVME_SMALL_POOL_SIZE / sizeof(__le64)) 1152 iod->flags |= IOD_SMALL_DESCRIPTOR; 1153 1154 prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, 1155 &prp2_dma); 1156 if (!prp_list) { 1157 iter->status = BLK_STS_RESOURCE; 1158 goto done; 1159 } 1160 iod->descriptors[iod->nr_descriptors++] = prp_list; 1161 1162 i = 0; 1163 for (;;) { 1164 prp_list[i++] = cpu_to_le64(iter->addr); 1165 prp_len = min(length, NVME_CTRL_PAGE_SIZE); 1166 if (WARN_ON_ONCE(iter->len < prp_len)) 1167 goto bad_sgl; 1168 1169 iod->total_len += prp_len; 1170 iter->addr += prp_len; 1171 iter->len -= prp_len; 1172 length -= prp_len; 1173 if (!length) 1174 break; 1175 1176 if (!nvme_pci_prp_iter_next(req, nvmeq->dev->dev, iter)) { 1177 if (WARN_ON_ONCE(!iter->status)) 1178 goto bad_sgl; 1179 goto done; 1180 } 1181 1182 /* 1183 * If we've filled the entire descriptor, allocate a new that is 1184 * pointed to be the last entry in the previous PRP list. To 1185 * accommodate for that move the last actual entry to the new 1186 * descriptor. 1187 */ 1188 if (i == NVME_CTRL_PAGE_SIZE >> 3) { 1189 __le64 *old_prp_list = prp_list; 1190 dma_addr_t prp_list_dma; 1191 1192 prp_list = dma_pool_alloc(nvmeq->descriptor_pools.large, 1193 GFP_ATOMIC, &prp_list_dma); 1194 if (!prp_list) { 1195 iter->status = BLK_STS_RESOURCE; 1196 goto done; 1197 } 1198 iod->descriptors[iod->nr_descriptors++] = prp_list; 1199 1200 prp_list[0] = old_prp_list[i - 1]; 1201 old_prp_list[i - 1] = cpu_to_le64(prp_list_dma); 1202 i = 1; 1203 } 1204 } 1205 1206 done: 1207 /* 1208 * nvme_unmap_data uses the DPT field in the SQE to tear down the 1209 * mapping, so initialize it even for failures. 1210 */ 1211 iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); 1212 iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); 1213 if (unlikely(iter->status)) 1214 nvme_unmap_data(req); 1215 return iter->status; 1216 1217 bad_sgl: 1218 dev_err_once(nvmeq->dev->dev, 1219 "Incorrectly formed request for payload:%d nents:%d\n", 1220 blk_rq_payload_bytes(req), blk_rq_nr_phys_segments(req)); 1221 nvme_unmap_data(req); 1222 return BLK_STS_IOERR; 1223 } 1224 1225 static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge, 1226 dma_addr_t addr, u32 len) 1227 { 1228 sge->addr = cpu_to_le64(addr); 1229 sge->length = cpu_to_le32(len); 1230 sge->type = NVME_SGL_FMT_DATA_DESC << 4; 1231 } 1232 1233 static void nvme_pci_dma_iter_set_sgl(struct nvme_sgl_desc *sge, 1234 struct blk_dma_iter *iter) 1235 { 1236 nvme_pci_sgl_set_data(sge, iter->addr, iter->len); 1237 } 1238 1239 static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge, 1240 dma_addr_t dma_addr, int entries) 1241 { 1242 sge->addr = cpu_to_le64(dma_addr); 1243 sge->length = cpu_to_le32(entries * sizeof(*sge)); 1244 sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4; 1245 } 1246 1247 static unsigned int nvme_pci_dmabuf_sgl_nents(struct request *req) 1248 { 1249 struct bio *bio = req->bio; 1250 struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map); 1251 struct scatterlist *sg; 1252 unsigned long tmp; 1253 size_t offset = bio->bi_iter.bi_offset; 1254 size_t remaining = blk_rq_payload_bytes(req); 1255 unsigned int nents = 0; 1256 1257 for_each_sgtable_dma_sg(map->sgt, sg, tmp) { 1258 size_t sg_len = sg_dma_len(sg); 1259 1260 if (!remaining) 1261 break; 1262 if (offset >= sg_len) { 1263 offset -= sg_len; 1264 continue; 1265 } 1266 1267 sg_len -= offset; 1268 offset = 0; 1269 1270 do { 1271 size_t chunk = min(remaining, sg_len); 1272 1273 nents++; 1274 sg_len -= chunk; 1275 remaining -= chunk; 1276 } while (sg_len && remaining); 1277 } 1278 1279 if (unlikely(remaining)) 1280 return 0; 1281 1282 return nents; 1283 } 1284 1285 static blk_status_t nvme_rq_setup_dmabuf_sgl(struct request *req, 1286 struct nvme_queue *nvmeq, unsigned int entries) 1287 { 1288 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1289 struct nvme_dmabuf_map *map = nvme_req_to_map(req); 1290 size_t length = blk_rq_payload_bytes(req); 1291 struct nvme_sgl_desc *sg_list = &iod->cmd.common.dptr.sgl; 1292 size_t offset = req->bio->bi_iter.bi_offset; 1293 bool pooled = entries > 1; 1294 dma_addr_t sgl_dma = 0; 1295 unsigned int mapped = 0; 1296 unsigned long tmp; 1297 struct scatterlist *sg; 1298 size_t remaining = length; 1299 1300 if (!entries) 1301 return BLK_STS_IOERR; 1302 1303 iod->cmd.common.flags = NVME_CMD_SGL_METABUF; 1304 iod->total_len = length; 1305 1306 /* entries == 1 fits in the inline descriptor; more needs a pool. */ 1307 if (pooled) { 1308 if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list)) 1309 iod->flags |= IOD_SMALL_DESCRIPTOR; 1310 1311 sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, 1312 &sgl_dma); 1313 if (!sg_list) 1314 return BLK_STS_RESOURCE; 1315 iod->descriptors[iod->nr_descriptors++] = sg_list; 1316 } 1317 1318 for_each_sgtable_dma_sg(map->sgt, sg, tmp) { 1319 size_t sg_len = sg_dma_len(sg); 1320 dma_addr_t addr = sg_dma_address(sg); 1321 1322 if (!remaining) 1323 break; 1324 if (offset >= sg_len) { 1325 offset -= sg_len; 1326 continue; 1327 } 1328 1329 addr += offset; 1330 sg_len -= offset; 1331 offset = 0; 1332 1333 do { 1334 u32 chunk = min_t(size_t, remaining, sg_len); 1335 1336 if (WARN_ON_ONCE(mapped == entries)) 1337 goto err_free; 1338 nvme_pci_sgl_set_data(&sg_list[mapped++], addr, chunk); 1339 1340 addr += chunk; 1341 sg_len -= chunk; 1342 remaining -= chunk; 1343 } while (sg_len && remaining); 1344 } 1345 1346 if (unlikely(remaining)) 1347 goto err_free; 1348 1349 if (pooled) 1350 nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, 1351 mapped); 1352 return BLK_STS_OK; 1353 err_free: 1354 if (pooled) { 1355 iod->nr_descriptors--; 1356 dma_pool_free(nvme_dma_pool(nvmeq, iod), sg_list, sgl_dma); 1357 } 1358 return BLK_STS_IOERR; 1359 } 1360 1361 static blk_status_t nvme_rq_setup_dmabuf(struct request *req, 1362 struct nvme_queue *nvmeq, enum nvme_use_sgl use_sgl) 1363 { 1364 unsigned int entries; 1365 size_t avg_seg; 1366 1367 if (use_sgl == SGL_UNSUPPORTED) 1368 return nvme_rq_setup_dmabuf_map(req, nvmeq); 1369 1370 entries = nvme_pci_dmabuf_sgl_nents(req); 1371 1372 if (use_sgl == SGL_FORCED) 1373 return nvme_rq_setup_dmabuf_sgl(req, nvmeq, entries); 1374 1375 avg_seg = entries ? 1376 DIV_ROUND_UP(blk_rq_payload_bytes(req), entries) : 0; 1377 if (sgl_threshold && avg_seg >= sgl_threshold) 1378 return nvme_rq_setup_dmabuf_sgl(req, nvmeq, entries); 1379 1380 return nvme_rq_setup_dmabuf_map(req, nvmeq); 1381 } 1382 1383 static blk_status_t nvme_pci_setup_data_sgl(struct request *req, 1384 struct blk_dma_iter *iter) 1385 { 1386 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1387 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1388 unsigned int entries = blk_rq_nr_phys_segments(req); 1389 struct nvme_sgl_desc *sg_list; 1390 dma_addr_t sgl_dma; 1391 unsigned int mapped = 0; 1392 1393 /* set the transfer type as SGL */ 1394 iod->cmd.common.flags = NVME_CMD_SGL_METABUF; 1395 1396 if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { 1397 nvme_pci_dma_iter_set_sgl(&iod->cmd.common.dptr.sgl, iter); 1398 iod->total_len += iter->len; 1399 return BLK_STS_OK; 1400 } 1401 1402 if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list)) 1403 iod->flags |= IOD_SMALL_DESCRIPTOR; 1404 1405 sg_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC, 1406 &sgl_dma); 1407 if (!sg_list) { 1408 nvme_unmap_iter(req, iter, &iod->dma_state); 1409 return BLK_STS_RESOURCE; 1410 } 1411 1412 iod->descriptors[iod->nr_descriptors++] = sg_list; 1413 1414 do { 1415 if (WARN_ON_ONCE(mapped == entries)) { 1416 iter->status = BLK_STS_IOERR; 1417 break; 1418 } 1419 nvme_pci_dma_iter_set_sgl(&sg_list[mapped++], iter); 1420 iod->total_len += iter->len; 1421 } while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter)); 1422 1423 nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, mapped); 1424 if (unlikely(iter->status)) 1425 nvme_unmap_data(req); 1426 return iter->status; 1427 } 1428 1429 static blk_status_t nvme_pci_setup_data_simple(struct request *req, 1430 enum nvme_use_sgl use_sgl) 1431 { 1432 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1433 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1434 struct bio_vec bv = req_bvec(req); 1435 unsigned int prp1_offset = bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1); 1436 bool prp_possible = prp1_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2; 1437 dma_addr_t dma_addr; 1438 1439 if (!use_sgl && !prp_possible) 1440 return BLK_STS_AGAIN; 1441 if (is_pci_p2pdma_page(bv.bv_page)) 1442 return BLK_STS_AGAIN; 1443 1444 dma_addr = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0); 1445 if (dma_mapping_error(nvmeq->dev->dev, dma_addr)) 1446 return BLK_STS_RESOURCE; 1447 iod->total_len = bv.bv_len; 1448 iod->flags |= IOD_SINGLE_SEGMENT; 1449 1450 if (use_sgl == SGL_FORCED || !prp_possible) { 1451 iod->cmd.common.flags = NVME_CMD_SGL_METABUF; 1452 iod->cmd.common.dptr.sgl.addr = cpu_to_le64(dma_addr); 1453 iod->cmd.common.dptr.sgl.length = cpu_to_le32(bv.bv_len); 1454 iod->cmd.common.dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; 1455 } else { 1456 unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - prp1_offset; 1457 1458 iod->cmd.common.dptr.prp1 = cpu_to_le64(dma_addr); 1459 iod->cmd.common.dptr.prp2 = 0; 1460 if (bv.bv_len > first_prp_len) 1461 iod->cmd.common.dptr.prp2 = 1462 cpu_to_le64(dma_addr + first_prp_len); 1463 } 1464 1465 return BLK_STS_OK; 1466 } 1467 1468 static blk_status_t nvme_map_data(struct request *req) 1469 { 1470 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1471 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1472 struct nvme_dev *dev = nvmeq->dev; 1473 enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req); 1474 struct blk_dma_iter iter; 1475 blk_status_t ret; 1476 1477 if (blk_mq_rq_is_dmabuf(req)) 1478 return nvme_rq_setup_dmabuf(req, nvmeq, use_sgl); 1479 1480 /* 1481 * Try to skip the DMA iterator for single segment requests, as that 1482 * significantly improves performances for small I/O sizes. 1483 */ 1484 if (blk_rq_nr_phys_segments(req) == 1) { 1485 ret = nvme_pci_setup_data_simple(req, use_sgl); 1486 if (ret != BLK_STS_AGAIN) 1487 return ret; 1488 } 1489 1490 if (!blk_rq_dma_map_iter_start(req, dev->dev, &iod->dma_state, &iter)) 1491 return iter.status; 1492 1493 switch (iter.p2pdma.map) { 1494 case PCI_P2PDMA_MAP_BUS_ADDR: 1495 iod->flags |= IOD_DATA_P2P; 1496 break; 1497 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 1498 iod->flags |= IOD_DATA_MMIO; 1499 break; 1500 case PCI_P2PDMA_MAP_NONE: 1501 break; 1502 default: 1503 return BLK_STS_RESOURCE; 1504 } 1505 1506 if (use_sgl == SGL_FORCED || 1507 (use_sgl == SGL_SUPPORTED && 1508 (sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold))) 1509 return nvme_pci_setup_data_sgl(req, &iter); 1510 return nvme_pci_setup_data_prp(req, &iter); 1511 } 1512 1513 static blk_status_t nvme_pci_setup_meta_iter(struct request *req) 1514 { 1515 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1516 unsigned int entries = req->nr_integrity_segments; 1517 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1518 struct nvme_dev *dev = nvmeq->dev; 1519 struct nvme_sgl_desc *sg_list; 1520 struct blk_dma_iter iter; 1521 dma_addr_t sgl_dma; 1522 int i = 0; 1523 1524 if (!blk_rq_integrity_dma_map_iter_start(req, dev->dev, 1525 &iod->meta_dma_state, &iter)) 1526 return iter.status; 1527 1528 switch (iter.p2pdma.map) { 1529 case PCI_P2PDMA_MAP_BUS_ADDR: 1530 iod->flags |= IOD_META_P2P; 1531 break; 1532 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 1533 iod->flags |= IOD_META_MMIO; 1534 break; 1535 case PCI_P2PDMA_MAP_NONE: 1536 break; 1537 default: 1538 return BLK_STS_RESOURCE; 1539 } 1540 1541 if (blk_rq_dma_map_coalesce(&iod->meta_dma_state)) 1542 entries = 1; 1543 1544 /* 1545 * The NVMe MPTR descriptor has an implicit length that the host and 1546 * device must agree on to avoid data/memory corruption. We trust the 1547 * kernel allocated correctly based on the format's parameters, so use 1548 * the more efficient MPTR to avoid extra dma pool allocations for the 1549 * SGL indirection. 1550 * 1551 * But for user commands, we don't necessarily know what they do, so 1552 * the driver can't validate the metadata buffer size. The SGL 1553 * descriptor provides an explicit length, so we're relying on that 1554 * mechanism to catch any misunderstandings between the application and 1555 * device. 1556 * 1557 * P2P DMA also needs to use the blk_dma_iter method, so mptr setup 1558 * leverages this routine when that happens. 1559 */ 1560 if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl) || 1561 (entries == 1 && !(nvme_req(req)->flags & NVME_REQ_USERCMD))) { 1562 iod->cmd.common.metadata = cpu_to_le64(iter.addr); 1563 iod->meta_total_len = iter.len; 1564 iod->meta_dma = iter.addr; 1565 iod->meta_descriptor = NULL; 1566 return BLK_STS_OK; 1567 } 1568 1569 sg_list = dma_pool_alloc(nvmeq->descriptor_pools.small, GFP_ATOMIC, 1570 &sgl_dma); 1571 if (!sg_list) { 1572 nvme_unmap_iter(req, &iter, &iod->meta_dma_state); 1573 return BLK_STS_RESOURCE; 1574 } 1575 1576 iod->meta_descriptor = sg_list; 1577 iod->meta_dma = sgl_dma; 1578 iod->cmd.common.flags = NVME_CMD_SGL_METASEG; 1579 iod->cmd.common.metadata = cpu_to_le64(sgl_dma); 1580 if (entries == 1) { 1581 iod->meta_total_len = iter.len; 1582 nvme_pci_dma_iter_set_sgl(sg_list, &iter); 1583 return BLK_STS_OK; 1584 } 1585 1586 sgl_dma += sizeof(*sg_list); 1587 do { 1588 nvme_pci_dma_iter_set_sgl(&sg_list[++i], &iter); 1589 iod->meta_total_len += iter.len; 1590 } while (blk_rq_integrity_dma_map_iter_next(req, dev->dev, &iter)); 1591 1592 nvme_pci_sgl_set_seg(sg_list, sgl_dma, i); 1593 if (unlikely(iter.status)) 1594 nvme_unmap_metadata(req); 1595 return iter.status; 1596 } 1597 1598 static blk_status_t nvme_pci_setup_meta_mptr(struct request *req) 1599 { 1600 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1601 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 1602 struct bio_vec bv = rq_integrity_vec(req); 1603 1604 if (is_pci_p2pdma_page(bv.bv_page)) 1605 return nvme_pci_setup_meta_iter(req); 1606 1607 iod->meta_dma = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0); 1608 if (dma_mapping_error(nvmeq->dev->dev, iod->meta_dma)) 1609 return BLK_STS_IOERR; 1610 iod->cmd.common.metadata = cpu_to_le64(iod->meta_dma); 1611 iod->flags |= IOD_SINGLE_META_SEGMENT; 1612 return BLK_STS_OK; 1613 } 1614 1615 static blk_status_t nvme_map_metadata(struct request *req) 1616 { 1617 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1618 1619 if ((iod->cmd.common.flags & NVME_CMD_SGL_METABUF) && 1620 nvme_pci_metadata_use_sgls(req)) 1621 return nvme_pci_setup_meta_iter(req); 1622 return nvme_pci_setup_meta_mptr(req); 1623 } 1624 1625 static blk_status_t nvme_prep_rq(struct request *req) 1626 { 1627 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1628 blk_status_t ret; 1629 1630 iod->flags = 0; 1631 iod->nr_descriptors = 0; 1632 iod->total_len = 0; 1633 iod->meta_total_len = 0; 1634 iod->nr_dma_vecs = 0; 1635 1636 ret = nvme_setup_cmd(req->q->queuedata, req); 1637 if (ret) 1638 return ret; 1639 1640 if (blk_rq_nr_phys_segments(req)) { 1641 ret = nvme_map_data(req); 1642 if (ret) 1643 goto out_free_cmd; 1644 } 1645 1646 if (blk_integrity_rq(req)) { 1647 ret = nvme_map_metadata(req); 1648 if (ret) 1649 goto out_unmap_data; 1650 } 1651 1652 nvme_start_request(req); 1653 return BLK_STS_OK; 1654 out_unmap_data: 1655 if (blk_rq_nr_phys_segments(req)) 1656 nvme_unmap_data(req); 1657 out_free_cmd: 1658 nvme_cleanup_cmd(req); 1659 return ret; 1660 } 1661 1662 static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, 1663 const struct blk_mq_queue_data *bd) 1664 { 1665 struct nvme_queue *nvmeq = hctx->driver_data; 1666 struct nvme_dev *dev = nvmeq->dev; 1667 struct request *req = bd->rq; 1668 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1669 blk_status_t ret; 1670 1671 /* 1672 * We should not need to do this, but we're still using this to 1673 * ensure we can drain requests on a dying queue. 1674 */ 1675 if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags))) 1676 return BLK_STS_IOERR; 1677 1678 if (unlikely(!nvme_check_ready(&dev->ctrl, req, true))) 1679 return nvme_fail_nonready_command(&dev->ctrl, req); 1680 1681 ret = nvme_prep_rq(req); 1682 if (unlikely(ret)) 1683 return ret; 1684 spin_lock(&nvmeq->sq_lock); 1685 nvme_sq_copy_cmd(nvmeq, &iod->cmd); 1686 nvme_write_sq_db(nvmeq, bd->last); 1687 spin_unlock(&nvmeq->sq_lock); 1688 return BLK_STS_OK; 1689 } 1690 1691 static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct rq_list *rqlist) 1692 { 1693 struct request *req; 1694 1695 if (rq_list_empty(rqlist)) 1696 return; 1697 1698 spin_lock(&nvmeq->sq_lock); 1699 while ((req = rq_list_pop(rqlist))) { 1700 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 1701 1702 nvme_sq_copy_cmd(nvmeq, &iod->cmd); 1703 } 1704 nvme_write_sq_db(nvmeq, true); 1705 spin_unlock(&nvmeq->sq_lock); 1706 } 1707 1708 static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req) 1709 { 1710 /* 1711 * We should not need to do this, but we're still using this to 1712 * ensure we can drain requests on a dying queue. 1713 */ 1714 if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags))) 1715 return false; 1716 if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true))) 1717 return false; 1718 1719 return nvme_prep_rq(req) == BLK_STS_OK; 1720 } 1721 1722 static void nvme_queue_rqs(struct rq_list *rqlist) 1723 { 1724 struct rq_list submit_list = { }; 1725 struct rq_list requeue_list = { }; 1726 struct nvme_queue *nvmeq = NULL; 1727 struct request *req; 1728 1729 while ((req = rq_list_pop(rqlist))) { 1730 if (nvmeq && nvmeq != req->mq_hctx->driver_data) 1731 nvme_submit_cmds(nvmeq, &submit_list); 1732 nvmeq = req->mq_hctx->driver_data; 1733 1734 if (nvme_prep_rq_batch(nvmeq, req)) 1735 rq_list_add_tail(&submit_list, req); 1736 else 1737 rq_list_add_tail(&requeue_list, req); 1738 } 1739 1740 if (nvmeq) 1741 nvme_submit_cmds(nvmeq, &submit_list); 1742 *rqlist = requeue_list; 1743 } 1744 1745 static __always_inline void nvme_pci_unmap_rq(struct request *req) 1746 { 1747 if (blk_integrity_rq(req)) 1748 nvme_unmap_metadata(req); 1749 if (blk_rq_nr_phys_segments(req)) 1750 nvme_unmap_data(req); 1751 } 1752 1753 static void nvme_pci_complete_rq(struct request *req) 1754 { 1755 nvme_pci_unmap_rq(req); 1756 nvme_complete_rq(req); 1757 } 1758 1759 static void nvme_pci_complete_batch(struct io_comp_batch *iob) 1760 { 1761 nvme_complete_batch(iob, nvme_pci_unmap_rq); 1762 } 1763 1764 /* We read the CQE phase first to check if the rest of the entry is valid */ 1765 static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq) 1766 { 1767 struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head]; 1768 1769 return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase; 1770 } 1771 1772 static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) 1773 { 1774 u16 head = nvmeq->cq_head; 1775 1776 if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db, 1777 nvmeq->dbbuf_cq_ei)) 1778 writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 1779 } 1780 1781 static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq) 1782 { 1783 if (!nvmeq->qid) 1784 return nvmeq->dev->admin_tagset.tags[0]; 1785 return nvmeq->dev->tagset.tags[nvmeq->qid - 1]; 1786 } 1787 1788 static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, 1789 struct io_comp_batch *iob, u16 idx) 1790 { 1791 struct nvme_completion *cqe = &nvmeq->cqes[idx]; 1792 __u16 command_id = READ_ONCE(cqe->command_id); 1793 struct request *req; 1794 1795 /* 1796 * AEN requests are special as they don't time out and can 1797 * survive any kind of queue freeze and often don't respond to 1798 * aborts. We don't even bother to allocate a struct request 1799 * for them but rather special case them here. 1800 */ 1801 if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) { 1802 nvme_complete_async_event(&nvmeq->dev->ctrl, 1803 cqe->status, &cqe->result); 1804 return; 1805 } 1806 1807 req = nvme_find_rq(nvme_queue_tagset(nvmeq), command_id); 1808 if (unlikely(!req)) { 1809 dev_warn_ratelimited(nvmeq->dev->ctrl.device, 1810 "invalid id %d completed on queue %d\n", 1811 command_id, le16_to_cpu(cqe->sq_id)); 1812 return; 1813 } 1814 1815 /* 1816 * Tracing only; annotate a lockless snapshot of nvmeq->sq_tail using 1817 * data_race(). This would also help suppress context analysis warning 1818 * while accessing nvmeq->sq_tail without acquiring ->sq_lock. 1819 */ 1820 trace_nvme_sq(req, cqe->sq_head, data_race(nvmeq->sq_tail)); 1821 if (!nvme_try_complete_req(req, cqe->status, cqe->result) && 1822 !blk_mq_add_to_batch(req, iob, 1823 nvme_req(req)->status != NVME_SC_SUCCESS, 1824 nvme_pci_complete_batch)) 1825 nvme_pci_complete_rq(req); 1826 } 1827 1828 static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) 1829 { 1830 u32 tmp = nvmeq->cq_head + 1; 1831 1832 if (tmp == nvmeq->q_depth) { 1833 nvmeq->cq_head = 0; 1834 nvmeq->cq_phase ^= 1; 1835 } else { 1836 nvmeq->cq_head = tmp; 1837 } 1838 } 1839 1840 static inline bool nvme_poll_cq(struct nvme_queue *nvmeq, 1841 struct io_comp_batch *iob) 1842 { 1843 bool found = false; 1844 1845 while (nvme_cqe_pending(nvmeq)) { 1846 found = true; 1847 /* 1848 * load-load control dependency between phase and the rest of 1849 * the cqe requires a full read memory barrier 1850 */ 1851 dma_rmb(); 1852 nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head); 1853 nvme_update_cq_head(nvmeq); 1854 } 1855 1856 if (found) 1857 nvme_ring_cq_doorbell(nvmeq); 1858 return found; 1859 } 1860 1861 static irqreturn_t nvme_irq(int irq, void *data) 1862 { 1863 struct nvme_queue *nvmeq = data; 1864 DEFINE_IO_COMP_BATCH(iob); 1865 1866 if (nvme_poll_cq(nvmeq, &iob)) { 1867 if (!rq_list_empty(&iob.req_list)) 1868 nvme_pci_complete_batch(&iob); 1869 return IRQ_HANDLED; 1870 } 1871 return IRQ_NONE; 1872 } 1873 1874 static irqreturn_t nvme_irq_check(int irq, void *data) 1875 { 1876 struct nvme_queue *nvmeq = data; 1877 1878 if (nvme_cqe_pending(nvmeq)) 1879 return IRQ_WAKE_THREAD; 1880 return IRQ_NONE; 1881 } 1882 1883 /* 1884 * Poll for completions for any interrupt driven queue 1885 * Can be called from any context. 1886 */ 1887 static void nvme_poll_irqdisable(struct nvme_queue *nvmeq) 1888 { 1889 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); 1890 int irq; 1891 1892 WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags)); 1893 1894 irq = pci_irq_vector(pdev, nvmeq->cq_vector); 1895 disable_irq(irq); 1896 spin_lock(&nvmeq->cq_poll_lock); 1897 nvme_poll_cq(nvmeq, NULL); 1898 spin_unlock(&nvmeq->cq_poll_lock); 1899 enable_irq(irq); 1900 } 1901 1902 static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 1903 { 1904 struct nvme_queue *nvmeq = hctx->driver_data; 1905 bool found; 1906 1907 if (!test_bit(NVMEQ_POLLED, &nvmeq->flags) || 1908 !nvme_cqe_pending(nvmeq)) 1909 return 0; 1910 1911 spin_lock(&nvmeq->cq_poll_lock); 1912 found = nvme_poll_cq(nvmeq, iob); 1913 spin_unlock(&nvmeq->cq_poll_lock); 1914 1915 return found; 1916 } 1917 1918 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl) 1919 { 1920 struct nvme_dev *dev = to_nvme_dev(ctrl); 1921 struct nvme_queue *nvmeq = &dev->queues[0]; 1922 struct nvme_command c = { }; 1923 1924 c.common.opcode = nvme_admin_async_event; 1925 c.common.command_id = NVME_AQ_BLK_MQ_DEPTH; 1926 1927 spin_lock(&nvmeq->sq_lock); 1928 nvme_sq_copy_cmd(nvmeq, &c); 1929 nvme_write_sq_db(nvmeq, true); 1930 spin_unlock(&nvmeq->sq_lock); 1931 } 1932 1933 static int nvme_pci_subsystem_reset(struct nvme_ctrl *ctrl) 1934 { 1935 struct nvme_dev *dev = to_nvme_dev(ctrl); 1936 int ret = 0; 1937 1938 /* 1939 * Taking the shutdown_lock ensures the BAR mapping is not being 1940 * altered by reset_work. Holding this lock before the RESETTING state 1941 * change, if successful, also ensures nvme_remove won't be able to 1942 * proceed to iounmap until we're done. 1943 */ 1944 mutex_lock(&dev->shutdown_lock); 1945 if (!dev->bar_mapped_size) { 1946 ret = -ENODEV; 1947 goto unlock; 1948 } 1949 1950 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) { 1951 ret = -EBUSY; 1952 goto unlock; 1953 } 1954 1955 writel(NVME_SUBSYS_RESET, dev->bar + NVME_REG_NSSR); 1956 1957 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING) || 1958 !nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) 1959 goto unlock; 1960 1961 /* 1962 * Read controller status to flush the previous write and trigger a 1963 * pcie read error. 1964 */ 1965 readl(dev->bar + NVME_REG_CSTS); 1966 unlock: 1967 mutex_unlock(&dev->shutdown_lock); 1968 return ret; 1969 } 1970 1971 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 1972 { 1973 struct nvme_command c = { }; 1974 1975 c.delete_queue.opcode = opcode; 1976 c.delete_queue.qid = cpu_to_le16(id); 1977 1978 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 1979 } 1980 1981 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 1982 struct nvme_queue *nvmeq, s16 vector) 1983 { 1984 struct nvme_command c = { }; 1985 int flags = NVME_QUEUE_PHYS_CONTIG; 1986 1987 if (!test_bit(NVMEQ_POLLED, &nvmeq->flags)) 1988 flags |= NVME_CQ_IRQ_ENABLED; 1989 1990 /* 1991 * Note: we (ab)use the fact that the prp fields survive if no data 1992 * is attached to the request. 1993 */ 1994 c.create_cq.opcode = nvme_admin_create_cq; 1995 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 1996 c.create_cq.cqid = cpu_to_le16(qid); 1997 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 1998 c.create_cq.cq_flags = cpu_to_le16(flags); 1999 c.create_cq.irq_vector = cpu_to_le16(vector); 2000 2001 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 2002 } 2003 2004 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 2005 struct nvme_queue *nvmeq) 2006 { 2007 struct nvme_ctrl *ctrl = &dev->ctrl; 2008 struct nvme_command c = { }; 2009 int flags = NVME_QUEUE_PHYS_CONTIG; 2010 2011 /* 2012 * Some drives have a bug that auto-enables WRRU if MEDIUM isn't 2013 * set. Since URGENT priority is zeroes, it makes all queues 2014 * URGENT. 2015 */ 2016 if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ) 2017 flags |= NVME_SQ_PRIO_MEDIUM; 2018 2019 /* 2020 * Note: we (ab)use the fact that the prp fields survive if no data 2021 * is attached to the request. 2022 */ 2023 c.create_sq.opcode = nvme_admin_create_sq; 2024 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 2025 c.create_sq.sqid = cpu_to_le16(qid); 2026 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 2027 c.create_sq.sq_flags = cpu_to_le16(flags); 2028 c.create_sq.cqid = cpu_to_le16(qid); 2029 2030 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 2031 } 2032 2033 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 2034 { 2035 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 2036 } 2037 2038 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 2039 { 2040 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 2041 } 2042 2043 static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error, 2044 const struct io_comp_batch *iob) 2045 { 2046 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 2047 2048 dev_warn(nvmeq->dev->ctrl.device, 2049 "Abort status: 0x%x", nvme_req(req)->status); 2050 atomic_inc(&nvmeq->dev->ctrl.abort_limit); 2051 blk_mq_free_request(req); 2052 return RQ_END_IO_NONE; 2053 } 2054 2055 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) 2056 { 2057 /* If true, indicates loss of adapter communication, possibly by a 2058 * NVMe Subsystem reset. 2059 */ 2060 bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO); 2061 2062 /* If there is a reset/reinit ongoing, we shouldn't reset again. */ 2063 switch (nvme_ctrl_state(&dev->ctrl)) { 2064 case NVME_CTRL_RESETTING: 2065 case NVME_CTRL_CONNECTING: 2066 return false; 2067 default: 2068 break; 2069 } 2070 2071 /* We shouldn't reset unless the controller is on fatal error state 2072 * _or_ if we lost the communication with it. 2073 */ 2074 if (!(csts & NVME_CSTS_CFS) && !nssro) 2075 return false; 2076 2077 return true; 2078 } 2079 2080 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) 2081 { 2082 /* Read a config register to help see what died. */ 2083 u16 pci_status; 2084 int result; 2085 2086 result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS, 2087 &pci_status); 2088 if (result == PCIBIOS_SUCCESSFUL) 2089 dev_warn(dev->ctrl.device, 2090 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n", 2091 csts, pci_status); 2092 else 2093 dev_warn(dev->ctrl.device, 2094 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n", 2095 csts, result); 2096 2097 if (csts != ~0) 2098 return; 2099 2100 dev_warn(dev->ctrl.device, 2101 "Does your device have a faulty power saving mode enabled?\n"); 2102 dev_warn(dev->ctrl.device, 2103 "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n"); 2104 } 2105 2106 static enum blk_eh_timer_return nvme_timeout(struct request *req) 2107 { 2108 struct nvme_iod *iod = blk_mq_rq_to_pdu(req); 2109 struct nvme_queue *nvmeq = req->mq_hctx->driver_data; 2110 struct nvme_dev *dev = nvmeq->dev; 2111 struct request *abort_req; 2112 struct nvme_command cmd = { }; 2113 struct pci_dev *pdev = to_pci_dev(dev->dev); 2114 u32 csts = readl(dev->bar + NVME_REG_CSTS); 2115 u8 opcode; 2116 2117 /* 2118 * Shutdown the device immediately if we see it is disconnected. This 2119 * unblocks PCIe error handling if the nvme driver is waiting in 2120 * error_resume for a device that has been removed. We can't unbind the 2121 * driver while the driver's error callback is waiting to complete, so 2122 * we're relying on a timeout to break that deadlock if a removal 2123 * occurs while reset work is running. 2124 */ 2125 if (pci_dev_is_disconnected(pdev)) 2126 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); 2127 if (nvme_state_terminal(&dev->ctrl)) 2128 goto disable; 2129 2130 /* If PCI error recovery process is happening, we cannot reset or 2131 * the recovery mechanism will surely fail. 2132 */ 2133 mb(); 2134 if (pci_channel_offline(pdev)) 2135 return BLK_EH_RESET_TIMER; 2136 2137 /* 2138 * Reset immediately if the controller is failed 2139 */ 2140 if (nvme_should_reset(dev, csts)) { 2141 nvme_warn_reset(dev, csts); 2142 goto disable; 2143 } 2144 2145 /* 2146 * Did we miss an interrupt? 2147 */ 2148 if (test_bit(NVMEQ_POLLED, &nvmeq->flags)) 2149 nvme_poll(req->mq_hctx, NULL); 2150 else 2151 nvme_poll_irqdisable(nvmeq); 2152 2153 if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT) { 2154 dev_warn(dev->ctrl.device, 2155 "I/O tag %d (%04x) QID %d timeout, completion polled\n", 2156 req->tag, nvme_cid(req), nvmeq->qid); 2157 return BLK_EH_DONE; 2158 } 2159 2160 /* 2161 * Shutdown immediately if controller times out while starting. The 2162 * reset work will see the pci device disabled when it gets the forced 2163 * cancellation error. All outstanding requests are completed on 2164 * shutdown, so we return BLK_EH_DONE. 2165 */ 2166 switch (nvme_ctrl_state(&dev->ctrl)) { 2167 case NVME_CTRL_CONNECTING: 2168 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); 2169 fallthrough; 2170 case NVME_CTRL_DELETING: 2171 dev_warn_ratelimited(dev->ctrl.device, 2172 "I/O tag %d (%04x) QID %d timeout, disable controller\n", 2173 req->tag, nvme_cid(req), nvmeq->qid); 2174 nvme_req(req)->flags |= NVME_REQ_CANCELLED; 2175 nvme_dev_disable(dev, true); 2176 return BLK_EH_DONE; 2177 case NVME_CTRL_RESETTING: 2178 return BLK_EH_RESET_TIMER; 2179 default: 2180 break; 2181 } 2182 2183 /* 2184 * Shutdown the controller immediately and schedule a reset if the 2185 * command was already aborted once before and still hasn't been 2186 * returned to the driver, or if this is the admin queue. 2187 */ 2188 opcode = nvme_req(req)->cmd->common.opcode; 2189 if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) { 2190 dev_warn(dev->ctrl.device, 2191 "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n", 2192 req->tag, nvme_cid(req), opcode, 2193 nvme_opcode_str(nvmeq->qid, opcode), nvmeq->qid); 2194 nvme_req(req)->flags |= NVME_REQ_CANCELLED; 2195 goto disable; 2196 } 2197 2198 if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) { 2199 atomic_inc(&dev->ctrl.abort_limit); 2200 return BLK_EH_RESET_TIMER; 2201 } 2202 iod->flags |= IOD_ABORTED; 2203 2204 cmd.abort.opcode = nvme_admin_abort_cmd; 2205 cmd.abort.cid = nvme_cid(req); 2206 cmd.abort.sqid = cpu_to_le16(nvmeq->qid); 2207 2208 dev_warn(nvmeq->dev->ctrl.device, 2209 "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, aborting req_op:%s(%u) size:%u\n", 2210 req->tag, nvme_cid(req), opcode, nvme_get_opcode_str(opcode), 2211 nvmeq->qid, blk_op_str(req_op(req)), req_op(req), 2212 blk_rq_bytes(req)); 2213 2214 abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd), 2215 BLK_MQ_REQ_NOWAIT); 2216 if (IS_ERR(abort_req)) { 2217 atomic_inc(&dev->ctrl.abort_limit); 2218 return BLK_EH_RESET_TIMER; 2219 } 2220 nvme_init_request(abort_req, &cmd); 2221 2222 abort_req->end_io = abort_endio; 2223 abort_req->end_io_data = NULL; 2224 blk_execute_rq_nowait(abort_req, false); 2225 2226 /* 2227 * The aborted req will be completed on receiving the abort req. 2228 * We enable the timer again. If hit twice, it'll cause a device reset, 2229 * as the device then is in a faulty state. 2230 */ 2231 return BLK_EH_RESET_TIMER; 2232 2233 disable: 2234 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) { 2235 if (nvme_state_terminal(&dev->ctrl)) 2236 nvme_dev_disable(dev, true); 2237 return BLK_EH_DONE; 2238 } 2239 2240 nvme_dev_disable(dev, false); 2241 if (nvme_try_sched_reset(&dev->ctrl)) 2242 nvme_unquiesce_io_queues(&dev->ctrl); 2243 return BLK_EH_DONE; 2244 } 2245 2246 static void nvme_free_queue(struct nvme_queue *nvmeq) 2247 __context_unsafe(/* frees queue which is no longer in use */) 2248 { 2249 dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq), 2250 (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 2251 if (!nvmeq->sq_cmds) 2252 return; 2253 2254 if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) { 2255 pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev), 2256 nvmeq->sq_cmds, SQ_SIZE(nvmeq)); 2257 } else { 2258 dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq), 2259 nvmeq->sq_cmds, nvmeq->sq_dma_addr); 2260 } 2261 } 2262 2263 static void nvme_free_queues(struct nvme_dev *dev, int lowest) 2264 { 2265 int i; 2266 2267 for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) { 2268 dev->ctrl.queue_count--; 2269 nvme_free_queue(&dev->queues[i]); 2270 } 2271 } 2272 2273 static void nvme_suspend_queue(struct nvme_dev *dev, unsigned int qid) 2274 { 2275 struct nvme_queue *nvmeq = &dev->queues[qid]; 2276 2277 if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags)) 2278 return; 2279 2280 /* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */ 2281 mb(); 2282 2283 nvmeq->dev->online_queues--; 2284 if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q) 2285 nvme_quiesce_admin_queue(&nvmeq->dev->ctrl); 2286 if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags)) 2287 pci_free_irq(to_pci_dev(dev->dev), nvmeq->cq_vector, nvmeq); 2288 } 2289 2290 static void nvme_suspend_io_queues(struct nvme_dev *dev) 2291 { 2292 int i; 2293 2294 for (i = dev->ctrl.queue_count - 1; i > 0; i--) 2295 nvme_suspend_queue(dev, i); 2296 } 2297 2298 /* 2299 * Called only on a device that has been disabled and after all other threads 2300 * that can check this device's completion queues have synced, except 2301 * nvme_poll(). This is the last chance for the driver to see a natural 2302 * completion before nvme_cancel_request() terminates all incomplete requests. 2303 */ 2304 static void nvme_reap_pending_cqes(struct nvme_dev *dev) 2305 { 2306 int i; 2307 2308 for (i = dev->ctrl.queue_count - 1; i > 0; i--) { 2309 spin_lock(&dev->queues[i].cq_poll_lock); 2310 nvme_poll_cq(&dev->queues[i], NULL); 2311 spin_unlock(&dev->queues[i].cq_poll_lock); 2312 } 2313 } 2314 2315 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, 2316 int entry_size) 2317 { 2318 int q_depth = dev->q_depth; 2319 unsigned q_size_aligned = roundup(q_depth * entry_size, 2320 NVME_CTRL_PAGE_SIZE); 2321 2322 if (q_size_aligned * nr_io_queues > dev->cmb_size) { 2323 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); 2324 2325 mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE); 2326 q_depth = div_u64(mem_per_q, entry_size); 2327 2328 /* 2329 * Ensure the reduced q_depth is above some threshold where it 2330 * would be better to map queues in system memory with the 2331 * original depth 2332 */ 2333 if (q_depth < 64) 2334 return -ENOMEM; 2335 } 2336 2337 return q_depth; 2338 } 2339 2340 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, 2341 int qid) 2342 __context_unsafe(/* safe to allocate sq_cmds without any protection */) 2343 { 2344 struct pci_dev *pdev = to_pci_dev(dev->dev); 2345 2346 if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) { 2347 nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq)); 2348 if (nvmeq->sq_cmds) { 2349 nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev, 2350 nvmeq->sq_cmds); 2351 if (nvmeq->sq_dma_addr) { 2352 set_bit(NVMEQ_SQ_CMB, &nvmeq->flags); 2353 return 0; 2354 } 2355 2356 pci_free_p2pmem(pdev, nvmeq->sq_cmds, SQ_SIZE(nvmeq)); 2357 } 2358 } 2359 2360 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq), 2361 &nvmeq->sq_dma_addr, GFP_KERNEL); 2362 if (!nvmeq->sq_cmds) 2363 return -ENOMEM; 2364 return 0; 2365 } 2366 2367 static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) 2368 { 2369 struct nvme_queue *nvmeq = &dev->queues[qid]; 2370 2371 if (dev->ctrl.queue_count > qid) 2372 return 0; 2373 2374 nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES; 2375 nvmeq->q_depth = depth; 2376 nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq), 2377 &nvmeq->cq_dma_addr, GFP_KERNEL); 2378 if (!nvmeq->cqes) 2379 goto free_nvmeq; 2380 2381 if (nvme_alloc_sq_cmds(dev, nvmeq, qid)) 2382 goto free_cqdma; 2383 2384 nvmeq->dev = dev; 2385 spin_lock_init(&nvmeq->sq_lock); 2386 spin_lock_init(&nvmeq->cq_poll_lock); 2387 nvmeq->cq_head = 0; 2388 nvmeq->cq_phase = 1; 2389 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 2390 nvmeq->qid = qid; 2391 dev->ctrl.queue_count++; 2392 2393 return 0; 2394 2395 free_cqdma: 2396 dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes, 2397 nvmeq->cq_dma_addr); 2398 free_nvmeq: 2399 return -ENOMEM; 2400 } 2401 2402 static int queue_request_irq(struct nvme_queue *nvmeq) 2403 { 2404 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev); 2405 int nr = nvmeq->dev->ctrl.instance; 2406 2407 if (use_threaded_interrupts) { 2408 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check, 2409 nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid); 2410 } else { 2411 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq, 2412 NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid); 2413 } 2414 } 2415 2416 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 2417 __context_unsafe(/* initialize unpublished/lock-guarded variables */) 2418 { 2419 struct nvme_dev *dev = nvmeq->dev; 2420 2421 nvmeq->sq_tail = 0; 2422 nvmeq->last_sq_tail = 0; 2423 nvmeq->cq_head = 0; 2424 nvmeq->cq_phase = 1; 2425 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 2426 memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq)); 2427 nvme_dbbuf_init(dev, nvmeq, qid); 2428 dev->online_queues++; 2429 wmb(); /* ensure the first interrupt sees the initialization */ 2430 } 2431 2432 /* 2433 * Try getting shutdown_lock while setting up IO queues. 2434 */ 2435 static int nvme_setup_io_queues_trylock(struct nvme_dev *dev) 2436 __cond_acquires(0, &dev->shutdown_lock) 2437 { 2438 /* 2439 * Give up if the lock is being held by nvme_dev_disable. 2440 */ 2441 if (!mutex_trylock(&dev->shutdown_lock)) 2442 return -ENODEV; 2443 2444 /* 2445 * Controller is in wrong state, fail early. 2446 */ 2447 if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_CONNECTING) { 2448 mutex_unlock(&dev->shutdown_lock); 2449 return -ENODEV; 2450 } 2451 2452 return 0; 2453 } 2454 2455 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) 2456 { 2457 struct nvme_dev *dev = nvmeq->dev; 2458 int result; 2459 u16 vector = 0; 2460 2461 /* 2462 * A queue's vector matches the queue identifier unless the controller 2463 * has only one vector available. 2464 */ 2465 if (!polled) 2466 vector = dev->num_vecs == 1 ? 0 : qid; 2467 else 2468 set_bit(NVMEQ_POLLED, &nvmeq->flags); 2469 2470 result = adapter_alloc_cq(dev, qid, nvmeq, vector); 2471 if (result) 2472 return result; 2473 2474 result = adapter_alloc_sq(dev, qid, nvmeq); 2475 if (result < 0) 2476 return result; 2477 if (result) 2478 goto release_cq; 2479 2480 nvmeq->cq_vector = vector; 2481 2482 result = nvme_setup_io_queues_trylock(dev); 2483 if (result) 2484 return result; 2485 nvme_init_queue(nvmeq, qid); 2486 if (!polled) { 2487 result = queue_request_irq(nvmeq); 2488 if (result < 0) 2489 goto release_sq; 2490 } 2491 2492 set_bit(NVMEQ_ENABLED, &nvmeq->flags); 2493 mutex_unlock(&dev->shutdown_lock); 2494 return result; 2495 2496 release_sq: 2497 dev->online_queues--; 2498 mutex_unlock(&dev->shutdown_lock); 2499 adapter_delete_sq(dev, qid); 2500 release_cq: 2501 adapter_delete_cq(dev, qid); 2502 return result; 2503 } 2504 2505 static const struct blk_mq_ops nvme_mq_admin_ops = { 2506 .queue_rq = nvme_queue_rq, 2507 .complete = nvme_pci_complete_rq, 2508 .commit_rqs = nvme_commit_rqs, 2509 .init_hctx = nvme_admin_init_hctx, 2510 .init_request = nvme_pci_init_request, 2511 .timeout = nvme_timeout, 2512 }; 2513 2514 static const struct blk_mq_ops nvme_mq_ops = { 2515 .queue_rq = nvme_queue_rq, 2516 .queue_rqs = nvme_queue_rqs, 2517 .complete = nvme_pci_complete_rq, 2518 .commit_rqs = nvme_commit_rqs, 2519 .init_hctx = nvme_init_hctx, 2520 .init_request = nvme_pci_init_request, 2521 .map_queues = nvme_pci_map_queues, 2522 .timeout = nvme_timeout, 2523 .poll = nvme_poll, 2524 }; 2525 2526 static void nvme_dev_remove_admin(struct nvme_dev *dev) 2527 { 2528 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) { 2529 /* 2530 * If the controller was reset during removal, it's possible 2531 * user requests may be waiting on a stopped queue. Start the 2532 * queue to flush these to completion. 2533 */ 2534 nvme_unquiesce_admin_queue(&dev->ctrl); 2535 nvme_remove_admin_tag_set(&dev->ctrl); 2536 } 2537 } 2538 2539 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) 2540 { 2541 return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride); 2542 } 2543 2544 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size) 2545 { 2546 struct pci_dev *pdev = to_pci_dev(dev->dev); 2547 2548 if (size <= dev->bar_mapped_size) 2549 return 0; 2550 if (size > pci_resource_len(pdev, 0)) 2551 return -ENOMEM; 2552 if (dev->bar) 2553 iounmap(dev->bar); 2554 dev->bar = ioremap(pci_resource_start(pdev, 0), size); 2555 if (!dev->bar) { 2556 dev->bar_mapped_size = 0; 2557 return -ENOMEM; 2558 } 2559 dev->bar_mapped_size = size; 2560 dev->dbs = dev->bar + NVME_REG_DBS; 2561 2562 return 0; 2563 } 2564 2565 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) 2566 { 2567 int result; 2568 u32 aqa; 2569 struct nvme_queue *nvmeq; 2570 2571 result = nvme_remap_bar(dev, db_bar_size(dev, 0)); 2572 if (result < 0) 2573 return result; 2574 2575 dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ? 2576 NVME_CAP_NSSRC(dev->ctrl.cap) : 0; 2577 2578 if (dev->subsystem && 2579 (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO)) 2580 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS); 2581 2582 /* 2583 * If the device has been passed off to us in an enabled state, just 2584 * clear the enabled bit. The spec says we should set the 'shutdown 2585 * notification bits', but doing so may cause the device to complete 2586 * commands to the admin queue ... and we don't know what memory that 2587 * might be pointing at! 2588 */ 2589 result = nvme_disable_ctrl(&dev->ctrl, false); 2590 if (result < 0) { 2591 struct pci_dev *pdev = to_pci_dev(dev->dev); 2592 2593 /* 2594 * The NVMe Controller Reset method did not get an expected 2595 * CSTS.RDY transition, so something with the device appears to 2596 * be stuck. Use the lower level and bigger hammer PCIe 2597 * Function Level Reset to attempt restoring the device to its 2598 * initial state, and try again. 2599 */ 2600 result = pcie_reset_flr(pdev, false); 2601 if (result < 0) 2602 return result; 2603 2604 pci_restore_state(pdev); 2605 result = nvme_disable_ctrl(&dev->ctrl, false); 2606 if (result < 0) 2607 return result; 2608 2609 dev_info(dev->ctrl.device, 2610 "controller reset completed after pcie flr\n"); 2611 } 2612 2613 result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH); 2614 if (result) 2615 return result; 2616 2617 dev->ctrl.numa_node = dev_to_node(dev->dev); 2618 2619 nvmeq = &dev->queues[0]; 2620 aqa = nvmeq->q_depth - 1; 2621 aqa |= aqa << 16; 2622 2623 writel(aqa, dev->bar + NVME_REG_AQA); 2624 lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ); 2625 lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ); 2626 2627 result = nvme_enable_ctrl(&dev->ctrl); 2628 if (result) 2629 return result; 2630 2631 nvmeq->cq_vector = 0; 2632 nvme_init_queue(nvmeq, 0); 2633 result = queue_request_irq(nvmeq); 2634 if (result) { 2635 dev->online_queues--; 2636 nvme_disable_ctrl(&dev->ctrl, false); 2637 return result; 2638 } 2639 2640 set_bit(NVMEQ_ENABLED, &nvmeq->flags); 2641 return result; 2642 } 2643 2644 static int nvme_create_io_queues(struct nvme_dev *dev) 2645 { 2646 unsigned i, max, rw_queues; 2647 int ret = 0; 2648 2649 for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) { 2650 if (nvme_alloc_queue(dev, i, dev->q_depth)) { 2651 ret = -ENOMEM; 2652 break; 2653 } 2654 } 2655 2656 max = min(dev->max_qid, dev->ctrl.queue_count - 1); 2657 if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) { 2658 rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] + 2659 dev->io_queues[HCTX_TYPE_READ]; 2660 } else { 2661 rw_queues = max; 2662 } 2663 2664 for (i = dev->online_queues; i <= max; i++) { 2665 bool polled = i > rw_queues; 2666 2667 ret = nvme_create_queue(&dev->queues[i], i, polled); 2668 if (ret) 2669 break; 2670 } 2671 2672 /* 2673 * Ignore failing Create SQ/CQ commands, we can continue with less 2674 * than the desired amount of queues, and even a controller without 2675 * I/O queues can still be used to issue admin commands. This might 2676 * be useful to upgrade a buggy firmware for example. 2677 */ 2678 return ret >= 0 ? 0 : ret; 2679 } 2680 2681 static u64 nvme_cmb_size_unit(struct nvme_dev *dev) 2682 { 2683 u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK; 2684 2685 return 1ULL << (12 + 4 * szu); 2686 } 2687 2688 static u32 nvme_cmb_size(struct nvme_dev *dev) 2689 { 2690 return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK; 2691 } 2692 2693 static void nvme_map_cmb(struct nvme_dev *dev) 2694 { 2695 u64 size, offset; 2696 resource_size_t bar_size; 2697 struct pci_dev *pdev = to_pci_dev(dev->dev); 2698 int bar; 2699 2700 if (dev->cmb_size) 2701 return; 2702 2703 if (NVME_CAP_CMBS(dev->ctrl.cap)) 2704 writel(NVME_CMBMSC_CRE, dev->bar + NVME_REG_CMBMSC); 2705 2706 dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ); 2707 if (!dev->cmbsz) 2708 return; 2709 dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC); 2710 2711 size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev); 2712 offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc); 2713 bar = NVME_CMB_BIR(dev->cmbloc); 2714 bar_size = pci_resource_len(pdev, bar); 2715 2716 if (offset > bar_size) 2717 return; 2718 2719 /* 2720 * Controllers may support a CMB size larger than their BAR, for 2721 * example, due to being behind a bridge. Reduce the CMB to the 2722 * reported size of the BAR 2723 */ 2724 size = min(size, bar_size - offset); 2725 2726 if (!IS_ALIGNED(size, memremap_compat_align()) || 2727 !IS_ALIGNED(pci_resource_start(pdev, bar), 2728 memremap_compat_align())) 2729 return; 2730 2731 /* 2732 * Tell the controller about the host side address mapping the CMB, 2733 * and enable CMB decoding for the NVMe 1.4+ scheme: 2734 */ 2735 if (NVME_CAP_CMBS(dev->ctrl.cap)) { 2736 hi_lo_writeq(NVME_CMBMSC_CRE | NVME_CMBMSC_CMSE | 2737 (pci_bus_address(pdev, bar) + offset), 2738 dev->bar + NVME_REG_CMBMSC); 2739 } 2740 2741 if (pci_p2pdma_add_resource(pdev, bar, size, offset)) { 2742 dev_warn(dev->ctrl.device, 2743 "failed to register the CMB\n"); 2744 hi_lo_writeq(0, dev->bar + NVME_REG_CMBMSC); 2745 return; 2746 } 2747 2748 dev->cmb_size = size; 2749 dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS); 2750 2751 if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) == 2752 (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) 2753 pci_p2pmem_publish(pdev, true); 2754 } 2755 2756 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits) 2757 { 2758 u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT; 2759 u64 dma_addr = dev->host_mem_descs_dma; 2760 struct nvme_command c = { }; 2761 int ret; 2762 2763 c.features.opcode = nvme_admin_set_features; 2764 c.features.fid = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF); 2765 c.features.dword11 = cpu_to_le32(bits); 2766 c.features.dword12 = cpu_to_le32(host_mem_size); 2767 c.features.dword13 = cpu_to_le32(lower_32_bits(dma_addr)); 2768 c.features.dword14 = cpu_to_le32(upper_32_bits(dma_addr)); 2769 c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs); 2770 2771 ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0); 2772 if (ret) { 2773 dev_warn(dev->ctrl.device, 2774 "failed to set host mem (err %d, flags %#x).\n", 2775 ret, bits); 2776 } else 2777 dev->hmb = bits & NVME_HOST_MEM_ENABLE; 2778 2779 return ret; 2780 } 2781 2782 static void nvme_free_host_mem_multi(struct nvme_dev *dev) 2783 { 2784 int i; 2785 2786 for (i = 0; i < dev->nr_host_mem_descs; i++) { 2787 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i]; 2788 size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE; 2789 2790 dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i], 2791 le64_to_cpu(desc->addr), 2792 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN); 2793 } 2794 2795 kfree(dev->host_mem_desc_bufs); 2796 dev->host_mem_desc_bufs = NULL; 2797 } 2798 2799 static void nvme_free_host_mem(struct nvme_dev *dev) 2800 { 2801 if (dev->hmb_sgt) { 2802 dma_free_noncontiguous(dev->dev, dev->host_mem_size, 2803 dev->hmb_sgt, DMA_BIDIRECTIONAL); 2804 dev->hmb_sgt = NULL; 2805 } else { 2806 nvme_free_host_mem_multi(dev); 2807 } 2808 2809 dma_free_coherent(dev->dev, dev->host_mem_descs_size, 2810 dev->host_mem_descs, dev->host_mem_descs_dma); 2811 dev->host_mem_descs = NULL; 2812 dev->host_mem_descs_size = 0; 2813 dev->nr_host_mem_descs = 0; 2814 } 2815 2816 static int nvme_alloc_host_mem_single(struct nvme_dev *dev, u64 size) 2817 { 2818 dev->hmb_sgt = dma_alloc_noncontiguous(dev->dev, size, 2819 DMA_BIDIRECTIONAL, GFP_KERNEL, 0); 2820 if (!dev->hmb_sgt) 2821 return -ENOMEM; 2822 2823 dev->host_mem_descs = dma_alloc_coherent(dev->dev, 2824 sizeof(*dev->host_mem_descs), &dev->host_mem_descs_dma, 2825 GFP_KERNEL); 2826 if (!dev->host_mem_descs) { 2827 dma_free_noncontiguous(dev->dev, size, dev->hmb_sgt, 2828 DMA_BIDIRECTIONAL); 2829 dev->hmb_sgt = NULL; 2830 return -ENOMEM; 2831 } 2832 dev->host_mem_size = size; 2833 dev->host_mem_descs_size = sizeof(*dev->host_mem_descs); 2834 dev->nr_host_mem_descs = 1; 2835 2836 dev->host_mem_descs[0].addr = 2837 cpu_to_le64(dev->hmb_sgt->sgl->dma_address); 2838 dev->host_mem_descs[0].size = cpu_to_le32(size / NVME_CTRL_PAGE_SIZE); 2839 return 0; 2840 } 2841 2842 static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred, 2843 u32 chunk_size) 2844 { 2845 struct nvme_host_mem_buf_desc *descs; 2846 u32 max_entries, len, descs_size; 2847 dma_addr_t descs_dma; 2848 int i = 0; 2849 void **bufs; 2850 u64 size, tmp; 2851 2852 tmp = (preferred + chunk_size - 1); 2853 do_div(tmp, chunk_size); 2854 max_entries = tmp; 2855 2856 if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries) 2857 max_entries = dev->ctrl.hmmaxd; 2858 2859 descs_size = max_entries * sizeof(*descs); 2860 descs = dma_alloc_coherent(dev->dev, descs_size, &descs_dma, 2861 GFP_KERNEL); 2862 if (!descs) 2863 goto out; 2864 2865 bufs = kzalloc_objs(*bufs, max_entries); 2866 if (!bufs) 2867 goto out_free_descs; 2868 2869 for (size = 0; size < preferred && i < max_entries; size += len) { 2870 dma_addr_t dma_addr; 2871 2872 len = min_t(u64, chunk_size, preferred - size); 2873 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL, 2874 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN); 2875 if (!bufs[i]) 2876 break; 2877 2878 descs[i].addr = cpu_to_le64(dma_addr); 2879 descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE); 2880 i++; 2881 } 2882 2883 if (!size) 2884 goto out_free_bufs; 2885 2886 dev->nr_host_mem_descs = i; 2887 dev->host_mem_size = size; 2888 dev->host_mem_descs = descs; 2889 dev->host_mem_descs_dma = descs_dma; 2890 dev->host_mem_descs_size = descs_size; 2891 dev->host_mem_desc_bufs = bufs; 2892 return 0; 2893 2894 out_free_bufs: 2895 kfree(bufs); 2896 out_free_descs: 2897 dma_free_coherent(dev->dev, descs_size, descs, descs_dma); 2898 out: 2899 dev->host_mem_descs = NULL; 2900 return -ENOMEM; 2901 } 2902 2903 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) 2904 { 2905 unsigned long dma_merge_boundary = dma_get_merge_boundary(dev->dev); 2906 u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES); 2907 u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2); 2908 u64 chunk_size; 2909 2910 /* 2911 * If there is an IOMMU that can merge pages, try a virtually 2912 * non-contiguous allocation for a single segment first. 2913 */ 2914 if (dma_merge_boundary && (PAGE_SIZE & dma_merge_boundary) == 0) { 2915 if (!nvme_alloc_host_mem_single(dev, preferred)) 2916 return 0; 2917 } 2918 2919 /* start big and work our way down */ 2920 for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) { 2921 if (!nvme_alloc_host_mem_multi(dev, preferred, chunk_size)) { 2922 if (!min || dev->host_mem_size >= min) 2923 return 0; 2924 nvme_free_host_mem(dev); 2925 } 2926 } 2927 2928 return -ENOMEM; 2929 } 2930 2931 static int nvme_setup_host_mem(struct nvme_dev *dev) 2932 { 2933 u64 max = (u64)max_host_mem_size_mb * SZ_1M; 2934 u64 preferred = (u64)dev->ctrl.hmpre * 4096; 2935 u64 min = (u64)dev->ctrl.hmmin * 4096; 2936 u32 enable_bits = NVME_HOST_MEM_ENABLE; 2937 int ret; 2938 2939 if (!dev->ctrl.hmpre) 2940 return 0; 2941 2942 preferred = min(preferred, max); 2943 if (min > max) { 2944 dev_warn(dev->ctrl.device, 2945 "min host memory (%lld MiB) above limit (%d MiB).\n", 2946 min >> ilog2(SZ_1M), max_host_mem_size_mb); 2947 nvme_free_host_mem(dev); 2948 return 0; 2949 } 2950 2951 /* 2952 * If we already have a buffer allocated check if we can reuse it. 2953 */ 2954 if (dev->host_mem_descs) { 2955 if (dev->host_mem_size >= min) 2956 enable_bits |= NVME_HOST_MEM_RETURN; 2957 else 2958 nvme_free_host_mem(dev); 2959 } 2960 2961 if (!dev->host_mem_descs) { 2962 if (nvme_alloc_host_mem(dev, min, preferred)) { 2963 dev_warn(dev->ctrl.device, 2964 "failed to allocate host memory buffer.\n"); 2965 return 0; /* controller must work without HMB */ 2966 } 2967 2968 dev_info(dev->ctrl.device, 2969 "allocated %lld MiB host memory buffer (%u segment%s).\n", 2970 dev->host_mem_size >> ilog2(SZ_1M), 2971 dev->nr_host_mem_descs, 2972 str_plural(dev->nr_host_mem_descs)); 2973 } 2974 2975 ret = nvme_set_host_mem(dev, enable_bits); 2976 if (ret) 2977 nvme_free_host_mem(dev); 2978 return ret; 2979 } 2980 2981 static ssize_t cmb_show(struct device *dev, struct device_attribute *attr, 2982 char *buf) 2983 { 2984 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 2985 2986 return sysfs_emit(buf, "cmbloc : 0x%08x\ncmbsz : 0x%08x\n", 2987 ndev->cmbloc, ndev->cmbsz); 2988 } 2989 static DEVICE_ATTR_RO(cmb); 2990 2991 static ssize_t cmbloc_show(struct device *dev, struct device_attribute *attr, 2992 char *buf) 2993 { 2994 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 2995 2996 return sysfs_emit(buf, "%u\n", ndev->cmbloc); 2997 } 2998 static DEVICE_ATTR_RO(cmbloc); 2999 3000 static ssize_t cmbsz_show(struct device *dev, struct device_attribute *attr, 3001 char *buf) 3002 { 3003 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 3004 3005 return sysfs_emit(buf, "%u\n", ndev->cmbsz); 3006 } 3007 static DEVICE_ATTR_RO(cmbsz); 3008 3009 static ssize_t hmb_show(struct device *dev, struct device_attribute *attr, 3010 char *buf) 3011 { 3012 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 3013 3014 return sysfs_emit(buf, "%d\n", ndev->hmb); 3015 } 3016 3017 static ssize_t hmb_store(struct device *dev, struct device_attribute *attr, 3018 const char *buf, size_t count) 3019 { 3020 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev)); 3021 bool new; 3022 int ret; 3023 3024 if (kstrtobool(buf, &new) < 0) 3025 return -EINVAL; 3026 3027 if (new == ndev->hmb) 3028 return count; 3029 3030 if (new) { 3031 ret = nvme_setup_host_mem(ndev); 3032 } else { 3033 ret = nvme_set_host_mem(ndev, 0); 3034 if (!ret) 3035 nvme_free_host_mem(ndev); 3036 } 3037 3038 if (ret < 0) 3039 return ret; 3040 3041 return count; 3042 } 3043 static DEVICE_ATTR_RW(hmb); 3044 3045 static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, 3046 struct attribute *a, int n) 3047 { 3048 struct nvme_ctrl *ctrl = 3049 dev_get_drvdata(container_of(kobj, struct device, kobj)); 3050 struct nvme_dev *dev = to_nvme_dev(ctrl); 3051 3052 if (a == &dev_attr_cmb.attr || 3053 a == &dev_attr_cmbloc.attr || 3054 a == &dev_attr_cmbsz.attr) { 3055 if (!dev->cmbsz) 3056 return 0; 3057 } 3058 if (a == &dev_attr_hmb.attr && !ctrl->hmpre) 3059 return 0; 3060 3061 return a->mode; 3062 } 3063 3064 static struct attribute *nvme_pci_attrs[] = { 3065 &dev_attr_cmb.attr, 3066 &dev_attr_cmbloc.attr, 3067 &dev_attr_cmbsz.attr, 3068 &dev_attr_hmb.attr, 3069 NULL, 3070 }; 3071 3072 static const struct attribute_group nvme_pci_dev_attrs_group = { 3073 .attrs = nvme_pci_attrs, 3074 .is_visible = nvme_pci_attrs_are_visible, 3075 }; 3076 3077 static const struct attribute_group *nvme_pci_dev_attr_groups[] = { 3078 &nvme_dev_attrs_group, 3079 &nvme_pci_dev_attrs_group, 3080 &nvme_dev_diag_attrs_group, 3081 NULL, 3082 }; 3083 3084 static void nvme_update_attrs(struct nvme_dev *dev) 3085 { 3086 sysfs_update_group(&dev->ctrl.device->kobj, &nvme_pci_dev_attrs_group); 3087 } 3088 3089 /* 3090 * nirqs is the number of interrupts available for write and read 3091 * queues. The core already reserved an interrupt for the admin queue. 3092 */ 3093 static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs) 3094 { 3095 struct nvme_dev *dev = affd->priv; 3096 unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues; 3097 3098 /* 3099 * If there is no interrupt available for queues, ensure that 3100 * the default queue is set to 1. The affinity set size is 3101 * also set to one, but the irq core ignores it for this case. 3102 * 3103 * If only one interrupt is available or 'write_queue' == 0, combine 3104 * write and read queues. 3105 * 3106 * If 'write_queues' > 0, ensure it leaves room for at least one read 3107 * queue. 3108 */ 3109 if (!nrirqs) { 3110 nrirqs = 1; 3111 nr_read_queues = 0; 3112 } else if (nrirqs == 1 || !nr_write_queues) { 3113 nr_read_queues = 0; 3114 } else if (nr_write_queues >= nrirqs) { 3115 nr_read_queues = 1; 3116 } else { 3117 nr_read_queues = nrirqs - nr_write_queues; 3118 } 3119 3120 dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues; 3121 affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues; 3122 dev->io_queues[HCTX_TYPE_READ] = nr_read_queues; 3123 affd->set_size[HCTX_TYPE_READ] = nr_read_queues; 3124 affd->nr_sets = nr_read_queues ? 2 : 1; 3125 } 3126 3127 static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues) 3128 { 3129 struct pci_dev *pdev = to_pci_dev(dev->dev); 3130 struct irq_affinity affd = { 3131 .pre_vectors = 1, 3132 .calc_sets = nvme_calc_irq_sets, 3133 .priv = dev, 3134 }; 3135 unsigned int irq_queues, poll_queues; 3136 unsigned int flags = PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY; 3137 3138 /* 3139 * Poll queues don't need interrupts, but we need at least one I/O queue 3140 * left over for non-polled I/O. 3141 */ 3142 poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1); 3143 dev->io_queues[HCTX_TYPE_POLL] = poll_queues; 3144 3145 /* 3146 * Initialize for the single interrupt case, will be updated in 3147 * nvme_calc_irq_sets(). 3148 */ 3149 dev->io_queues[HCTX_TYPE_DEFAULT] = 1; 3150 dev->io_queues[HCTX_TYPE_READ] = 0; 3151 3152 /* 3153 * We need interrupts for the admin queue and each non-polled I/O queue, 3154 * but some Apple controllers require all queues to use the first 3155 * vector. 3156 */ 3157 irq_queues = 1; 3158 if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR)) 3159 irq_queues += (nr_io_queues - poll_queues); 3160 if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI) 3161 flags &= ~PCI_IRQ_MSI; 3162 return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues, flags, 3163 &affd); 3164 } 3165 3166 static unsigned int nvme_max_io_queues(struct nvme_dev *dev) 3167 { 3168 /* 3169 * If tags are shared with admin queue (Apple bug), then 3170 * make sure we only use one IO queue. 3171 */ 3172 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) 3173 return 1; 3174 return blk_mq_num_possible_queues(0) + dev->nr_write_queues + 3175 dev->nr_poll_queues; 3176 } 3177 3178 static int nvme_setup_io_queues(struct nvme_dev *dev) 3179 { 3180 struct nvme_queue *adminq = &dev->queues[0]; 3181 struct pci_dev *pdev = to_pci_dev(dev->dev); 3182 unsigned int nr_io_queues; 3183 unsigned long size; 3184 int result; 3185 3186 /* 3187 * Sample the module parameters once at reset time so that we have 3188 * stable values to work with. 3189 */ 3190 dev->nr_write_queues = write_queues; 3191 dev->nr_poll_queues = poll_queues; 3192 3193 if (dev->ctrl.tagset) { 3194 /* 3195 * The set's maps are allocated only once at initialization 3196 * time. We can't add special queues later if their mq_map 3197 * wasn't preallocated. 3198 */ 3199 if (dev->ctrl.tagset->nr_maps < 3) 3200 dev->nr_poll_queues = 0; 3201 if (dev->ctrl.tagset->nr_maps < 2) 3202 dev->nr_write_queues = 0; 3203 } 3204 3205 /* 3206 * The initial number of allocated queue slots may be too large if the 3207 * user reduced the special queue parameters. Cap the value to the 3208 * number we need for this round. 3209 */ 3210 nr_io_queues = min(nvme_max_io_queues(dev), 3211 dev->nr_allocated_queues - 1); 3212 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); 3213 if (result < 0) 3214 return result; 3215 3216 if (nr_io_queues == 0) 3217 return 0; 3218 3219 /* 3220 * Free IRQ resources as soon as NVMEQ_ENABLED bit transitions 3221 * from set to unset. If there is a window to it is truely freed, 3222 * pci_free_irq_vectors() jumping into this window will crash. 3223 * And take lock to avoid racing with pci_free_irq_vectors() in 3224 * nvme_dev_disable() path. 3225 */ 3226 result = nvme_setup_io_queues_trylock(dev); 3227 if (result) 3228 return result; 3229 if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags)) 3230 pci_free_irq(pdev, 0, adminq); 3231 3232 if (dev->cmb_use_sqes) { 3233 result = nvme_cmb_qdepth(dev, nr_io_queues, 3234 sizeof(struct nvme_command)); 3235 if (result > 0) { 3236 dev->q_depth = result; 3237 dev->ctrl.sqsize = result - 1; 3238 } else { 3239 dev->cmb_use_sqes = false; 3240 } 3241 } 3242 3243 do { 3244 size = db_bar_size(dev, nr_io_queues); 3245 result = nvme_remap_bar(dev, size); 3246 if (!result) 3247 break; 3248 if (!--nr_io_queues) { 3249 result = -ENOMEM; 3250 goto out_unlock; 3251 } 3252 } while (1); 3253 adminq->q_db = dev->dbs; 3254 3255 retry: 3256 /* Deregister the admin queue's interrupt */ 3257 if (test_and_clear_bit(NVMEQ_ENABLED, &adminq->flags)) 3258 pci_free_irq(pdev, 0, adminq); 3259 3260 /* 3261 * If we enable msix early due to not intx, disable it again before 3262 * setting up the full range we need. 3263 */ 3264 pci_free_irq_vectors(pdev); 3265 3266 result = nvme_setup_irqs(dev, nr_io_queues); 3267 if (result <= 0) { 3268 result = -EIO; 3269 goto out_unlock; 3270 } 3271 3272 dev->num_vecs = result; 3273 result = max(result - 1, 1); 3274 dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL]; 3275 3276 /* 3277 * Should investigate if there's a performance win from allocating 3278 * more queues than interrupt vectors; it might allow the submission 3279 * path to scale better, even if the receive path is limited by the 3280 * number of interrupts. 3281 */ 3282 result = queue_request_irq(adminq); 3283 if (result) 3284 goto out_unlock; 3285 set_bit(NVMEQ_ENABLED, &adminq->flags); 3286 mutex_unlock(&dev->shutdown_lock); 3287 3288 result = nvme_create_io_queues(dev); 3289 if (result || dev->online_queues < 2) 3290 return result; 3291 3292 if (dev->online_queues - 1 < dev->max_qid) { 3293 nr_io_queues = dev->online_queues - 1; 3294 nvme_delete_io_queues(dev); 3295 result = nvme_setup_io_queues_trylock(dev); 3296 if (result) 3297 return result; 3298 nvme_suspend_io_queues(dev); 3299 goto retry; 3300 } 3301 dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n", 3302 dev->io_queues[HCTX_TYPE_DEFAULT], 3303 dev->io_queues[HCTX_TYPE_READ], 3304 dev->io_queues[HCTX_TYPE_POLL]); 3305 return 0; 3306 out_unlock: 3307 mutex_unlock(&dev->shutdown_lock); 3308 return result; 3309 } 3310 3311 static enum rq_end_io_ret nvme_del_queue_end(struct request *req, 3312 blk_status_t error, 3313 const struct io_comp_batch *iob) 3314 { 3315 struct nvme_queue *nvmeq = req->end_io_data; 3316 3317 blk_mq_free_request(req); 3318 complete(&nvmeq->delete_done); 3319 return RQ_END_IO_NONE; 3320 } 3321 3322 static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) 3323 { 3324 struct request_queue *q = nvmeq->dev->ctrl.admin_q; 3325 struct request *req; 3326 struct nvme_command cmd = { }; 3327 3328 cmd.delete_queue.opcode = opcode; 3329 cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid); 3330 3331 req = blk_mq_alloc_request(q, nvme_req_op(&cmd), BLK_MQ_REQ_NOWAIT); 3332 if (IS_ERR(req)) 3333 return PTR_ERR(req); 3334 nvme_init_request(req, &cmd); 3335 3336 req->end_io = nvme_del_queue_end; 3337 req->end_io_data = nvmeq; 3338 3339 init_completion(&nvmeq->delete_done); 3340 blk_execute_rq_nowait(req, false); 3341 return 0; 3342 } 3343 3344 static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode) 3345 { 3346 int nr_queues = dev->online_queues - 1, sent = 0; 3347 unsigned long timeout; 3348 3349 retry: 3350 timeout = dev->ctrl.admin_timeout; 3351 while (nr_queues > 0) { 3352 if (nvme_delete_queue(&dev->queues[nr_queues], opcode)) 3353 break; 3354 nr_queues--; 3355 sent++; 3356 } 3357 while (sent) { 3358 struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent]; 3359 3360 timeout = wait_for_completion_io_timeout(&nvmeq->delete_done, 3361 timeout); 3362 if (timeout == 0) 3363 return false; 3364 3365 sent--; 3366 if (nr_queues) 3367 goto retry; 3368 } 3369 return true; 3370 } 3371 3372 static void nvme_delete_io_queues(struct nvme_dev *dev) 3373 { 3374 if (__nvme_delete_io_queues(dev, nvme_admin_delete_sq)) 3375 __nvme_delete_io_queues(dev, nvme_admin_delete_cq); 3376 } 3377 3378 static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev) 3379 { 3380 if (dev->io_queues[HCTX_TYPE_POLL]) 3381 return 3; 3382 if (dev->io_queues[HCTX_TYPE_READ]) 3383 return 2; 3384 return 1; 3385 } 3386 3387 static bool nvme_pci_update_nr_queues(struct nvme_dev *dev) 3388 { 3389 if (!dev->ctrl.tagset) { 3390 nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops, 3391 nvme_pci_nr_maps(dev), sizeof(struct nvme_iod)); 3392 return true; 3393 } 3394 3395 /* Give up if we are racing with nvme_dev_disable() */ 3396 if (!mutex_trylock(&dev->shutdown_lock)) 3397 return false; 3398 3399 /* Check if nvme_dev_disable() has been executed already */ 3400 if (!dev->online_queues) { 3401 mutex_unlock(&dev->shutdown_lock); 3402 return false; 3403 } 3404 3405 blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1); 3406 /* free previously allocated queues that are no longer usable */ 3407 nvme_free_queues(dev, dev->online_queues); 3408 mutex_unlock(&dev->shutdown_lock); 3409 return true; 3410 } 3411 3412 static int nvme_pci_enable(struct nvme_dev *dev) 3413 { 3414 int result = -ENOMEM; 3415 struct pci_dev *pdev = to_pci_dev(dev->dev); 3416 unsigned int flags = PCI_IRQ_ALL_TYPES; 3417 3418 if (pci_enable_device_mem(pdev)) 3419 return result; 3420 3421 pci_set_master(pdev); 3422 3423 if (readl(dev->bar + NVME_REG_CSTS) == -1) { 3424 dev_dbg(dev->ctrl.device, "reading CSTS register failed\n"); 3425 result = -ENODEV; 3426 goto disable; 3427 } 3428 3429 /* 3430 * Some devices and/or platforms don't advertise or work with INTx 3431 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll 3432 * adjust this later. 3433 */ 3434 if (dev->ctrl.quirks & NVME_QUIRK_BROKEN_MSI) 3435 flags &= ~PCI_IRQ_MSI; 3436 result = pci_alloc_irq_vectors(pdev, 1, 1, flags); 3437 if (result < 0) 3438 goto disable; 3439 3440 dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP); 3441 3442 dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1, 3443 io_queue_depth); 3444 dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap); 3445 dev->dbs = dev->bar + 4096; 3446 3447 /* 3448 * Some Apple controllers require a non-standard SQE size. 3449 * Interestingly they also seem to ignore the CC:IOSQES register 3450 * so we don't bother updating it here. 3451 */ 3452 if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES) 3453 dev->io_sqes = 7; 3454 else 3455 dev->io_sqes = NVME_NVM_IOSQES; 3456 3457 if (dev->ctrl.quirks & NVME_QUIRK_QDEPTH_ONE) { 3458 dev->q_depth = 2; 3459 } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG && 3460 (pdev->device == 0xa821 || pdev->device == 0xa822) && 3461 NVME_CAP_MQES(dev->ctrl.cap) == 0) { 3462 dev->q_depth = 64; 3463 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, " 3464 "set queue depth=%u\n", dev->q_depth); 3465 } 3466 3467 /* 3468 * Controllers with the shared tags quirk need the IO queue to be 3469 * big enough so that we get 32 tags for the admin queue 3470 */ 3471 if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) && 3472 (dev->q_depth < (NVME_AQ_DEPTH + 2))) { 3473 dev->q_depth = NVME_AQ_DEPTH + 2; 3474 dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n", 3475 dev->q_depth); 3476 } 3477 dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */ 3478 3479 nvme_map_cmb(dev); 3480 3481 pci_save_state(pdev); 3482 3483 result = nvme_pci_configure_admin_queue(dev); 3484 if (result) 3485 goto free_irq; 3486 return result; 3487 3488 free_irq: 3489 pci_free_irq_vectors(pdev); 3490 disable: 3491 pci_disable_device(pdev); 3492 return result; 3493 } 3494 3495 static void nvme_dev_unmap(struct nvme_dev *dev) 3496 { 3497 if (dev->bar) 3498 iounmap(dev->bar); 3499 pci_release_mem_regions(to_pci_dev(dev->dev)); 3500 } 3501 3502 static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev) 3503 { 3504 struct pci_dev *pdev = to_pci_dev(dev->dev); 3505 u32 csts; 3506 3507 if (!pci_is_enabled(pdev) || !pci_device_is_present(pdev)) 3508 return true; 3509 if (pdev->error_state != pci_channel_io_normal) 3510 return true; 3511 3512 csts = readl(dev->bar + NVME_REG_CSTS); 3513 return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY); 3514 } 3515 3516 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) 3517 { 3518 enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); 3519 struct pci_dev *pdev = to_pci_dev(dev->dev); 3520 bool dead; 3521 3522 mutex_lock(&dev->shutdown_lock); 3523 dead = nvme_pci_ctrl_is_dead(dev); 3524 if (state == NVME_CTRL_LIVE || state == NVME_CTRL_RESETTING) { 3525 if (pci_is_enabled(pdev)) 3526 nvme_start_freeze(&dev->ctrl); 3527 /* 3528 * Give the controller a chance to complete all entered requests 3529 * if doing a safe shutdown. 3530 */ 3531 if (!dead && shutdown) 3532 nvme_wait_freeze_timeout(&dev->ctrl); 3533 } 3534 3535 nvme_quiesce_io_queues(&dev->ctrl); 3536 3537 if (!dead && dev->ctrl.queue_count > 0) { 3538 nvme_delete_io_queues(dev); 3539 nvme_disable_ctrl(&dev->ctrl, shutdown); 3540 nvme_poll_irqdisable(&dev->queues[0]); 3541 } 3542 nvme_suspend_io_queues(dev); 3543 nvme_suspend_queue(dev, 0); 3544 pci_free_irq_vectors(pdev); 3545 if (pci_is_enabled(pdev)) 3546 pci_disable_device(pdev); 3547 nvme_reap_pending_cqes(dev); 3548 3549 nvme_cancel_tagset(&dev->ctrl); 3550 nvme_cancel_admin_tagset(&dev->ctrl); 3551 3552 /* 3553 * The driver will not be starting up queues again if shutting down so 3554 * must flush all entered requests to their failed completion to avoid 3555 * deadlocking blk-mq hot-cpu notifier. 3556 */ 3557 if (shutdown) { 3558 nvme_unquiesce_io_queues(&dev->ctrl); 3559 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) 3560 nvme_unquiesce_admin_queue(&dev->ctrl); 3561 } 3562 mutex_unlock(&dev->shutdown_lock); 3563 } 3564 3565 static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) 3566 { 3567 if (!nvme_wait_reset(&dev->ctrl)) 3568 return -EBUSY; 3569 nvme_dev_disable(dev, shutdown); 3570 return 0; 3571 } 3572 3573 static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev) 3574 { 3575 size_t alloc_size = sizeof(struct nvme_dma_vec) * NVME_MAX_SEGS; 3576 3577 dev->dmavec_mempool = mempool_create_node(1, 3578 mempool_kmalloc, mempool_kfree, 3579 (void *)alloc_size, GFP_KERNEL, 3580 dev_to_node(dev->dev)); 3581 if (!dev->dmavec_mempool) 3582 return -ENOMEM; 3583 return 0; 3584 } 3585 3586 static void nvme_free_tagset(struct nvme_dev *dev) 3587 { 3588 if (dev->tagset.tags) 3589 nvme_remove_io_tag_set(&dev->ctrl); 3590 dev->ctrl.tagset = NULL; 3591 } 3592 3593 /* pairs with nvme_pci_alloc_dev */ 3594 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) 3595 { 3596 struct nvme_dev *dev = to_nvme_dev(ctrl); 3597 3598 nvme_free_tagset(dev); 3599 put_device(dev->dev); 3600 kfree(dev->queues); 3601 kfree(dev); 3602 } 3603 3604 static void nvme_reset_work(struct work_struct *work) 3605 { 3606 struct nvme_dev *dev = 3607 container_of(work, struct nvme_dev, ctrl.reset_work); 3608 bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL); 3609 int result; 3610 3611 if (nvme_ctrl_state(&dev->ctrl) != NVME_CTRL_RESETTING) { 3612 dev_warn(dev->ctrl.device, "ctrl state %d is not RESETTING\n", 3613 dev->ctrl.state); 3614 result = -ENODEV; 3615 goto out; 3616 } 3617 3618 /* 3619 * If we're called to reset a live controller first shut it down before 3620 * moving on. 3621 */ 3622 if (dev->ctrl.ctrl_config & NVME_CC_ENABLE) 3623 nvme_dev_disable(dev, false); 3624 nvme_sync_queues(&dev->ctrl); 3625 3626 mutex_lock(&dev->shutdown_lock); 3627 result = nvme_pci_enable(dev); 3628 if (result) 3629 goto out_unlock; 3630 nvme_unquiesce_admin_queue(&dev->ctrl); 3631 mutex_unlock(&dev->shutdown_lock); 3632 3633 /* 3634 * Introduce CONNECTING state from nvme-fc/rdma transports to mark the 3635 * initializing procedure here. 3636 */ 3637 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) { 3638 dev_warn(dev->ctrl.device, 3639 "failed to mark controller CONNECTING\n"); 3640 result = -EBUSY; 3641 goto out; 3642 } 3643 3644 result = nvme_init_ctrl_finish(&dev->ctrl, was_suspend); 3645 if (result) 3646 goto out; 3647 3648 if (nvme_ctrl_meta_sgl_supported(&dev->ctrl)) 3649 dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS; 3650 else 3651 dev->ctrl.max_integrity_segments = 1; 3652 3653 nvme_dbbuf_dma_alloc(dev); 3654 3655 result = nvme_setup_host_mem(dev); 3656 if (result < 0) 3657 goto out; 3658 3659 nvme_update_attrs(dev); 3660 3661 result = nvme_setup_io_queues(dev); 3662 if (result) 3663 goto out; 3664 3665 /* 3666 * Freeze and update the number of I/O queues as those might have 3667 * changed. If there are no I/O queues left after this reset, keep the 3668 * controller around but remove all namespaces. 3669 */ 3670 if (dev->online_queues > 1) { 3671 nvme_dbbuf_set(dev); 3672 nvme_unquiesce_io_queues(&dev->ctrl); 3673 nvme_wait_freeze(&dev->ctrl); 3674 if (!nvme_pci_update_nr_queues(dev)) 3675 goto out; 3676 nvme_unfreeze(&dev->ctrl); 3677 } else { 3678 dev_warn(dev->ctrl.device, "IO queues lost\n"); 3679 nvme_mark_namespaces_dead(&dev->ctrl); 3680 nvme_unquiesce_io_queues(&dev->ctrl); 3681 nvme_remove_namespaces(&dev->ctrl); 3682 nvme_free_tagset(dev); 3683 } 3684 3685 /* 3686 * If only admin queue live, keep it to do further investigation or 3687 * recovery. 3688 */ 3689 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) { 3690 dev_warn(dev->ctrl.device, 3691 "failed to mark controller live state\n"); 3692 result = -ENODEV; 3693 goto out; 3694 } 3695 3696 nvme_start_ctrl(&dev->ctrl); 3697 return; 3698 3699 out_unlock: 3700 mutex_unlock(&dev->shutdown_lock); 3701 out: 3702 /* 3703 * Set state to deleting now to avoid blocking nvme_wait_reset(), which 3704 * may be holding this pci_dev's device lock. 3705 */ 3706 dev_warn(dev->ctrl.device, "Disabling device after reset failure: %d\n", 3707 result); 3708 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); 3709 nvme_dev_disable(dev, true); 3710 nvme_sync_queues(&dev->ctrl); 3711 nvme_mark_namespaces_dead(&dev->ctrl); 3712 nvme_unquiesce_io_queues(&dev->ctrl); 3713 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); 3714 } 3715 3716 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) 3717 { 3718 *val = readl(to_nvme_dev(ctrl)->bar + off); 3719 return 0; 3720 } 3721 3722 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) 3723 { 3724 writel(val, to_nvme_dev(ctrl)->bar + off); 3725 return 0; 3726 } 3727 3728 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) 3729 { 3730 *val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off); 3731 return 0; 3732 } 3733 3734 static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size) 3735 { 3736 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev); 3737 3738 return snprintf(buf, size, "%s\n", dev_name(&pdev->dev)); 3739 } 3740 3741 static void nvme_pci_print_device_info(struct nvme_ctrl *ctrl) 3742 { 3743 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev); 3744 struct nvme_subsystem *subsys = ctrl->subsys; 3745 3746 dev_err(ctrl->device, 3747 "VID:DID %04x:%04x model:%.*s firmware:%.*s\n", 3748 pdev->vendor, pdev->device, 3749 nvme_strlen(subsys->model, sizeof(subsys->model)), 3750 subsys->model, nvme_strlen(subsys->firmware_rev, 3751 sizeof(subsys->firmware_rev)), 3752 subsys->firmware_rev); 3753 } 3754 3755 static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl) 3756 { 3757 struct nvme_dev *dev = to_nvme_dev(ctrl); 3758 3759 return dma_pci_p2pdma_supported(dev->dev); 3760 } 3761 3762 static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl, 3763 bool is_admin) 3764 { 3765 if (!nvme_ctrl_sgl_supported(ctrl) || is_admin) 3766 return NVME_CTRL_PAGE_SIZE - 1; 3767 return 0; 3768 } 3769 3770 static int nvme_pci_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, 3771 struct dma_buf_io_ctx *ctx) 3772 { 3773 struct nvme_dev *dev = to_nvme_dev(ctrl); 3774 3775 return nvme_init_dma_buf_io_ctx(ctrl, &dev->dmabufs, ctx); 3776 } 3777 3778 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { 3779 .name = "pcie", 3780 .module = THIS_MODULE, 3781 .flags = NVME_F_METADATA_SUPPORTED, 3782 .dev_attr_groups = nvme_pci_dev_attr_groups, 3783 .reg_read32 = nvme_pci_reg_read32, 3784 .reg_write32 = nvme_pci_reg_write32, 3785 .reg_read64 = nvme_pci_reg_read64, 3786 .free_ctrl = nvme_pci_free_ctrl, 3787 .submit_async_event = nvme_pci_submit_async_event, 3788 .subsystem_reset = nvme_pci_subsystem_reset, 3789 .get_address = nvme_pci_get_address, 3790 .print_device_info = nvme_pci_print_device_info, 3791 .supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma, 3792 .get_virt_boundary = nvme_pci_get_virt_boundary, 3793 .init_dma_buf_io_ctx = nvme_pci_init_dma_buf_io_ctx, 3794 }; 3795 3796 static int nvme_dev_map(struct nvme_dev *dev) 3797 { 3798 struct pci_dev *pdev = to_pci_dev(dev->dev); 3799 3800 if (pci_request_mem_regions(pdev, "nvme")) 3801 return -ENODEV; 3802 3803 if (nvme_remap_bar(dev, NVME_REG_DBS + 4096)) 3804 goto release; 3805 3806 return 0; 3807 release: 3808 pci_release_mem_regions(pdev); 3809 return -ENODEV; 3810 } 3811 3812 static unsigned long check_vendor_combination_bug(struct pci_dev *pdev) 3813 { 3814 if (pdev->vendor == 0x144d && pdev->device == 0xa802) { 3815 /* 3816 * Several Samsung devices seem to drop off the PCIe bus 3817 * randomly when APST is on and uses the deepest sleep state. 3818 * This has been observed on a Samsung "SM951 NVMe SAMSUNG 3819 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD 3820 * 950 PRO 256GB", but it seems to be restricted to two Dell 3821 * laptops. 3822 */ 3823 if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") && 3824 (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") || 3825 dmi_match(DMI_PRODUCT_NAME, "Precision 5510"))) 3826 return NVME_QUIRK_NO_DEEPEST_PS; 3827 } else if (pdev->vendor == 0x144d && pdev->device == 0xa804) { 3828 /* 3829 * Samsung SSD 960 EVO drops off the PCIe bus after system 3830 * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as 3831 * within few minutes after bootup on a Coffee Lake board - 3832 * ASUS PRIME Z370-A 3833 */ 3834 if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") && 3835 (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") || 3836 dmi_match(DMI_BOARD_NAME, "PRIME Z370-A"))) 3837 return NVME_QUIRK_NO_APST; 3838 } else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 || 3839 pdev->device == 0xa808 || pdev->device == 0xa809)) || 3840 (pdev->vendor == 0x1e0f && pdev->device == 0x0001)) { 3841 /* 3842 * Forcing to use host managed nvme power settings for 3843 * lowest idle power with quick resume latency on 3844 * Samsung and Toshiba SSDs based on suspend behavior 3845 * on Coffee Lake board for LENOVO C640 3846 */ 3847 if ((dmi_match(DMI_BOARD_VENDOR, "LENOVO")) && 3848 dmi_match(DMI_BOARD_NAME, "LNVNB161216")) 3849 return NVME_QUIRK_SIMPLE_SUSPEND; 3850 } else if (pdev->vendor == 0x2646 && (pdev->device == 0x2263 || 3851 pdev->device == 0x500f)) { 3852 /* 3853 * Exclude some Kingston NV1 and A2000 devices from 3854 * NVME_QUIRK_SIMPLE_SUSPEND. Do a full suspend to save a 3855 * lot of energy with s2idle sleep on some TUXEDO platforms. 3856 */ 3857 if (dmi_match(DMI_BOARD_NAME, "NS5X_NS7XAU") || 3858 dmi_match(DMI_BOARD_NAME, "NS5x_7xAU") || 3859 dmi_match(DMI_BOARD_NAME, "NS5x_7xPU") || 3860 dmi_match(DMI_BOARD_NAME, "PH4PRX1_PH6PRX1")) 3861 return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND; 3862 } else if (pdev->vendor == 0x144d && pdev->device == 0xa80d) { 3863 /* 3864 * Exclude Samsung 990 Evo from NVME_QUIRK_SIMPLE_SUSPEND 3865 * because of high power consumption (> 2 Watt) in s2idle 3866 * sleep. Only some boards with Intel CPU are affected. 3867 * (Note for testing: Samsung 990 Evo Plus has same PCI ID) 3868 */ 3869 if (dmi_match(DMI_BOARD_NAME, "DN50Z-140HC-YD") || 3870 dmi_match(DMI_BOARD_NAME, "GMxPXxx") || 3871 dmi_match(DMI_BOARD_NAME, "GXxMRXx") || 3872 dmi_match(DMI_BOARD_NAME, "NS5X_NS7XAU") || 3873 dmi_match(DMI_BOARD_NAME, "PH4PG31") || 3874 dmi_match(DMI_BOARD_NAME, "PH4PRX1_PH6PRX1") || 3875 dmi_match(DMI_BOARD_NAME, "PH6PG01_PH6PG71")) 3876 return NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND; 3877 } 3878 3879 /* 3880 * NVMe SSD drops off the PCIe bus after system idle 3881 * for 10 hours on a Lenovo N60z board. 3882 */ 3883 if (dmi_match(DMI_BOARD_NAME, "LXKT-ZXEG-N6")) 3884 return NVME_QUIRK_NO_APST; 3885 3886 return 0; 3887 } 3888 3889 static struct quirk_entry *detect_dynamic_quirks(struct pci_dev *pdev) 3890 { 3891 int i; 3892 3893 for (i = 0; i < nvme_pci_quirk_count; i++) 3894 if (pdev->vendor == nvme_pci_quirk_list[i].vendor_id && 3895 pdev->device == nvme_pci_quirk_list[i].dev_id) 3896 return &nvme_pci_quirk_list[i]; 3897 3898 return NULL; 3899 } 3900 3901 static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev, 3902 const struct pci_device_id *id) 3903 { 3904 unsigned long quirks = id->driver_data; 3905 int node = dev_to_node(&pdev->dev); 3906 struct nvme_dev *dev; 3907 struct quirk_entry *qentry; 3908 int ret = -ENOMEM; 3909 3910 dev = kzalloc_node(struct_size(dev, descriptor_pools, nr_node_ids), 3911 GFP_KERNEL, node); 3912 if (!dev) 3913 return ERR_PTR(-ENOMEM); 3914 INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); 3915 mutex_init(&dev->shutdown_lock); 3916 nvme_init_buf_list(&dev->dmabufs); 3917 3918 dev->nr_write_queues = write_queues; 3919 dev->nr_poll_queues = poll_queues; 3920 dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1; 3921 dev->queues = kcalloc_node(dev->nr_allocated_queues, 3922 sizeof(struct nvme_queue), GFP_KERNEL, node); 3923 if (!dev->queues) 3924 goto out_free_dev; 3925 3926 dev->dev = get_device(&pdev->dev); 3927 3928 quirks |= check_vendor_combination_bug(pdev); 3929 if (!noacpi && 3930 !(quirks & NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND) && 3931 acpi_storage_d3(&pdev->dev)) { 3932 /* 3933 * Some systems use a bios work around to ask for D3 on 3934 * platforms that support kernel managed suspend. 3935 */ 3936 dev_info(&pdev->dev, 3937 "platform quirk: setting simple suspend\n"); 3938 quirks |= NVME_QUIRK_SIMPLE_SUSPEND; 3939 } 3940 qentry = detect_dynamic_quirks(pdev); 3941 if (qentry) { 3942 quirks |= qentry->enabled_quirks; 3943 quirks &= ~qentry->disabled_quirks; 3944 } 3945 ret = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops, 3946 quirks); 3947 if (ret) 3948 goto out_put_device; 3949 3950 if (dev->ctrl.quirks & NVME_QUIRK_DMA_ADDRESS_BITS_48) 3951 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); 3952 else 3953 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3954 dma_set_min_align_mask(&pdev->dev, NVME_CTRL_PAGE_SIZE - 1); 3955 dma_set_max_seg_size(&pdev->dev, 0xffffffff); 3956 3957 /* 3958 * Limit the max command size to prevent iod->sg allocations going 3959 * over a single page. 3960 */ 3961 dev->ctrl.max_hw_sectors = min_t(u32, 3962 NVME_MAX_BYTES >> SECTOR_SHIFT, 3963 dma_max_mapping_size(&pdev->dev) >> SECTOR_SHIFT); 3964 dev->ctrl.max_segments = NVME_MAX_SEGS; 3965 dev->ctrl.max_integrity_segments = 1; 3966 return dev; 3967 3968 out_put_device: 3969 put_device(dev->dev); 3970 kfree(dev->queues); 3971 out_free_dev: 3972 kfree(dev); 3973 return ERR_PTR(ret); 3974 } 3975 3976 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 3977 { 3978 struct nvme_dev *dev; 3979 int result = -ENOMEM; 3980 3981 dev = nvme_pci_alloc_dev(pdev, id); 3982 if (IS_ERR(dev)) 3983 return PTR_ERR(dev); 3984 3985 result = nvme_add_ctrl(&dev->ctrl); 3986 if (result) 3987 goto out_put_ctrl; 3988 3989 result = nvme_dev_map(dev); 3990 if (result) 3991 goto out_uninit_ctrl; 3992 3993 result = nvme_pci_alloc_iod_mempool(dev); 3994 if (result) 3995 goto out_dev_unmap; 3996 3997 dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev)); 3998 3999 result = nvme_pci_enable(dev); 4000 if (result) 4001 goto out_release_iod_mempool; 4002 4003 result = nvme_alloc_admin_tag_set(&dev->ctrl, &dev->admin_tagset, 4004 &nvme_mq_admin_ops, sizeof(struct nvme_iod)); 4005 if (result) 4006 goto out_disable; 4007 4008 /* 4009 * Mark the controller as connecting before sending admin commands to 4010 * allow the timeout handler to do the right thing. 4011 */ 4012 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) { 4013 dev_warn(dev->ctrl.device, 4014 "failed to mark controller CONNECTING\n"); 4015 result = -EBUSY; 4016 goto out_disable; 4017 } 4018 4019 result = nvme_init_ctrl_finish(&dev->ctrl, false); 4020 if (result) 4021 goto out_disable; 4022 4023 if (nvme_ctrl_meta_sgl_supported(&dev->ctrl)) 4024 dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS; 4025 else 4026 dev->ctrl.max_integrity_segments = 1; 4027 4028 nvme_dbbuf_dma_alloc(dev); 4029 4030 result = nvme_setup_host_mem(dev); 4031 if (result < 0) 4032 goto out_disable; 4033 4034 nvme_update_attrs(dev); 4035 4036 result = nvme_setup_io_queues(dev); 4037 if (result) 4038 goto out_disable; 4039 4040 if (dev->online_queues > 1) { 4041 nvme_alloc_io_tag_set(&dev->ctrl, &dev->tagset, &nvme_mq_ops, 4042 nvme_pci_nr_maps(dev), sizeof(struct nvme_iod)); 4043 nvme_dbbuf_set(dev); 4044 } 4045 4046 if (!dev->ctrl.tagset) 4047 dev_warn(dev->ctrl.device, "IO queues not created\n"); 4048 4049 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) { 4050 dev_warn(dev->ctrl.device, 4051 "failed to mark controller live state\n"); 4052 result = -ENODEV; 4053 goto out_disable; 4054 } 4055 4056 pci_set_drvdata(pdev, dev); 4057 4058 nvme_start_ctrl(&dev->ctrl); 4059 nvme_put_ctrl(&dev->ctrl); 4060 flush_work(&dev->ctrl.scan_work); 4061 return 0; 4062 4063 out_disable: 4064 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); 4065 nvme_dev_disable(dev, true); 4066 nvme_free_host_mem(dev); 4067 nvme_dev_remove_admin(dev); 4068 nvme_dbbuf_dma_free(dev); 4069 nvme_free_queues(dev, 0); 4070 nvme_release_descriptor_pools(dev); 4071 out_release_iod_mempool: 4072 mempool_destroy(dev->dmavec_mempool); 4073 out_dev_unmap: 4074 nvme_dev_unmap(dev); 4075 out_uninit_ctrl: 4076 nvme_uninit_ctrl(&dev->ctrl); 4077 out_put_ctrl: 4078 nvme_put_ctrl(&dev->ctrl); 4079 dev_err_probe(&pdev->dev, result, "probe failed\n"); 4080 return result; 4081 } 4082 4083 static void nvme_reset_prepare(struct pci_dev *pdev) 4084 { 4085 struct nvme_dev *dev = pci_get_drvdata(pdev); 4086 4087 /* 4088 * We don't need to check the return value from waiting for the reset 4089 * state as pci_dev device lock is held, making it impossible to race 4090 * with ->remove(). 4091 */ 4092 nvme_disable_prepare_reset(dev, false); 4093 nvme_sync_queues(&dev->ctrl); 4094 } 4095 4096 static void nvme_reset_done(struct pci_dev *pdev) 4097 { 4098 struct nvme_dev *dev = pci_get_drvdata(pdev); 4099 4100 if (!nvme_try_sched_reset(&dev->ctrl)) 4101 flush_work(&dev->ctrl.reset_work); 4102 } 4103 4104 static void nvme_shutdown(struct pci_dev *pdev) 4105 { 4106 struct nvme_dev *dev = pci_get_drvdata(pdev); 4107 4108 nvme_disable_prepare_reset(dev, true); 4109 } 4110 4111 /* 4112 * The driver's remove may be called on a device in a partially initialized 4113 * state. This function must not have any dependencies on the device state in 4114 * order to proceed. 4115 */ 4116 static void nvme_remove(struct pci_dev *pdev) 4117 { 4118 struct nvme_dev *dev = pci_get_drvdata(pdev); 4119 4120 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); 4121 pci_set_drvdata(pdev, NULL); 4122 4123 if (!pci_device_is_present(pdev)) { 4124 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD); 4125 nvme_dev_disable(dev, true); 4126 } 4127 4128 flush_work(&dev->ctrl.reset_work); 4129 nvme_stop_ctrl(&dev->ctrl); 4130 nvme_remove_namespaces(&dev->ctrl); 4131 nvme_dev_disable(dev, true); 4132 nvme_pci_remove_dmabuf(&dev->dmabufs); 4133 nvme_free_host_mem(dev); 4134 nvme_dev_remove_admin(dev); 4135 nvme_dbbuf_dma_free(dev); 4136 nvme_free_queues(dev, 0); 4137 mempool_destroy(dev->dmavec_mempool); 4138 nvme_release_descriptor_pools(dev); 4139 nvme_dev_unmap(dev); 4140 nvme_uninit_ctrl(&dev->ctrl); 4141 } 4142 4143 #ifdef CONFIG_PM_SLEEP 4144 static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps) 4145 { 4146 return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps); 4147 } 4148 4149 static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps) 4150 { 4151 return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL); 4152 } 4153 4154 static int nvme_resume(struct device *dev) 4155 { 4156 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); 4157 struct nvme_ctrl *ctrl = &ndev->ctrl; 4158 4159 if (ndev->last_ps == U32_MAX || 4160 nvme_set_power_state(ctrl, ndev->last_ps) != 0) 4161 goto reset; 4162 if (ctrl->hmpre && nvme_setup_host_mem(ndev)) 4163 goto reset; 4164 4165 return 0; 4166 reset: 4167 return nvme_try_sched_reset(ctrl); 4168 } 4169 4170 static int nvme_suspend(struct device *dev) 4171 { 4172 struct pci_dev *pdev = to_pci_dev(dev); 4173 struct nvme_dev *ndev = pci_get_drvdata(pdev); 4174 struct nvme_ctrl *ctrl = &ndev->ctrl; 4175 int ret = -EBUSY; 4176 4177 ndev->last_ps = U32_MAX; 4178 4179 /* 4180 * The platform does not remove power for a kernel managed suspend so 4181 * use host managed nvme power settings for lowest idle power if 4182 * possible. This should have quicker resume latency than a full device 4183 * shutdown. But if the firmware is involved after the suspend or the 4184 * platform has any limitation in waking from low power states or the 4185 * device does not support any non-default power states, shut down the 4186 * device fully. 4187 * 4188 * If ASPM is not enabled for the device, shut down the device and allow 4189 * the PCI bus layer to put it into D3 in order to take the PCIe link 4190 * down, so as to allow the platform to achieve its minimum low-power 4191 * state (which may not be possible if the link is up). 4192 */ 4193 if (!pci_suspend_retains_context(pdev) || !ctrl->npss || 4194 !pcie_aspm_enabled(pdev) || 4195 (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND)) 4196 return nvme_disable_prepare_reset(ndev, true); 4197 4198 nvme_start_freeze(ctrl); 4199 nvme_wait_freeze(ctrl); 4200 nvme_sync_queues(ctrl); 4201 4202 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 4203 goto unfreeze; 4204 4205 /* 4206 * Host memory access may not be successful in a system suspend state, 4207 * but the specification allows the controller to access memory in a 4208 * non-operational power state. 4209 */ 4210 if (ndev->hmb) { 4211 ret = nvme_set_host_mem(ndev, 0); 4212 if (ret < 0) 4213 goto unfreeze; 4214 if (ret) { 4215 /* 4216 * The controller may still access the HMB. Shut it 4217 * down instead, which forces a reset on resume. 4218 */ 4219 ret = nvme_disable_prepare_reset(ndev, true); 4220 goto unfreeze; 4221 } 4222 } 4223 4224 ret = nvme_get_power_state(ctrl, &ndev->last_ps); 4225 if (ret < 0) 4226 goto unfreeze; 4227 4228 /* 4229 * A saved state prevents pci pm from generically controlling the 4230 * device's power. If we're using protocol specific settings, we don't 4231 * want pci interfering. 4232 */ 4233 pci_save_state(pdev); 4234 4235 ret = nvme_set_power_state(ctrl, ctrl->npss); 4236 if (ret < 0) 4237 goto unfreeze; 4238 4239 if (ret) { 4240 /* discard the saved state */ 4241 pci_load_saved_state(pdev, NULL); 4242 4243 /* 4244 * Clearing npss forces a controller reset on resume. The 4245 * correct value will be rediscovered then. 4246 */ 4247 ret = nvme_disable_prepare_reset(ndev, true); 4248 ctrl->npss = 0; 4249 } 4250 unfreeze: 4251 nvme_unfreeze(ctrl); 4252 return ret; 4253 } 4254 4255 static int nvme_simple_suspend(struct device *dev) 4256 { 4257 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); 4258 4259 return nvme_disable_prepare_reset(ndev, true); 4260 } 4261 4262 static int nvme_simple_resume(struct device *dev) 4263 { 4264 struct pci_dev *pdev = to_pci_dev(dev); 4265 struct nvme_dev *ndev = pci_get_drvdata(pdev); 4266 4267 return nvme_try_sched_reset(&ndev->ctrl); 4268 } 4269 4270 static const struct dev_pm_ops nvme_dev_pm_ops = { 4271 .suspend = nvme_suspend, 4272 .resume = nvme_resume, 4273 .freeze = nvme_simple_suspend, 4274 .thaw = nvme_simple_resume, 4275 .poweroff = nvme_simple_suspend, 4276 .restore = nvme_simple_resume, 4277 }; 4278 #endif /* CONFIG_PM_SLEEP */ 4279 4280 static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev, 4281 pci_channel_state_t state) 4282 { 4283 struct nvme_dev *dev = pci_get_drvdata(pdev); 4284 4285 /* 4286 * A frozen channel requires a reset. When detected, this method will 4287 * shutdown the controller to quiesce. The controller will be restarted 4288 * after the slot reset through driver's slot_reset callback. 4289 */ 4290 switch (state) { 4291 case pci_channel_io_normal: 4292 return PCI_ERS_RESULT_CAN_RECOVER; 4293 case pci_channel_io_frozen: 4294 dev_warn(dev->ctrl.device, 4295 "frozen state error detected, reset controller\n"); 4296 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) { 4297 nvme_dev_disable(dev, true); 4298 return PCI_ERS_RESULT_DISCONNECT; 4299 } 4300 nvme_dev_disable(dev, false); 4301 return PCI_ERS_RESULT_NEED_RESET; 4302 case pci_channel_io_perm_failure: 4303 dev_warn(dev->ctrl.device, 4304 "failure state error detected, request disconnect\n"); 4305 return PCI_ERS_RESULT_DISCONNECT; 4306 } 4307 return PCI_ERS_RESULT_NEED_RESET; 4308 } 4309 4310 static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev) 4311 { 4312 struct nvme_dev *dev = pci_get_drvdata(pdev); 4313 4314 dev_info(dev->ctrl.device, "restart after slot reset\n"); 4315 pci_restore_state(pdev); 4316 if (nvme_try_sched_reset(&dev->ctrl)) 4317 nvme_unquiesce_io_queues(&dev->ctrl); 4318 return PCI_ERS_RESULT_RECOVERED; 4319 } 4320 4321 static void nvme_error_resume(struct pci_dev *pdev) 4322 { 4323 struct nvme_dev *dev = pci_get_drvdata(pdev); 4324 4325 flush_work(&dev->ctrl.reset_work); 4326 } 4327 4328 static const struct pci_error_handlers nvme_err_handler = { 4329 .error_detected = nvme_error_detected, 4330 .slot_reset = nvme_slot_reset, 4331 .resume = nvme_error_resume, 4332 .reset_prepare = nvme_reset_prepare, 4333 .reset_done = nvme_reset_done, 4334 }; 4335 4336 static const struct pci_device_id nvme_id_table[] = { 4337 { PCI_VDEVICE(INTEL, 0x0953), /* Intel 750/P3500/P3600/P3700 */ 4338 .driver_data = NVME_QUIRK_STRIPE_SIZE | 4339 NVME_QUIRK_DEALLOCATE_ZEROES, }, 4340 { PCI_VDEVICE(INTEL, 0x0a53), /* Intel P3520 */ 4341 .driver_data = NVME_QUIRK_STRIPE_SIZE | 4342 NVME_QUIRK_DEALLOCATE_ZEROES, }, 4343 { PCI_VDEVICE(INTEL, 0x0a54), /* Intel P4500/P4600 */ 4344 .driver_data = NVME_QUIRK_STRIPE_SIZE | 4345 NVME_QUIRK_IGNORE_DEV_SUBNQN | 4346 NVME_QUIRK_BOGUS_NID, }, 4347 { PCI_VDEVICE(INTEL, 0x0a55), /* Dell Express Flash P4600 */ 4348 .driver_data = NVME_QUIRK_STRIPE_SIZE, }, 4349 { PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */ 4350 .driver_data = NVME_QUIRK_NO_DEEPEST_PS | 4351 NVME_QUIRK_MEDIUM_PRIO_SQ | 4352 NVME_QUIRK_NO_TEMP_THRESH_CHANGE | 4353 NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4354 { PCI_VDEVICE(INTEL, 0xf1a6), /* Intel 760p/Pro 7600p */ 4355 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, }, 4356 { PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */ 4357 .driver_data = NVME_QUIRK_IDENTIFY_CNS | 4358 NVME_QUIRK_DISABLE_WRITE_ZEROES | 4359 NVME_QUIRK_BOGUS_NID, }, 4360 { PCI_VDEVICE(REDHAT, 0x0010), /* Qemu emulated controller */ 4361 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4362 { PCI_DEVICE(0x1217, 0x8760), /* O2 Micro 64GB Steam Deck */ 4363 .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, 4364 { PCI_DEVICE(0x126f, 0x1001), /* Silicon Motion generic */ 4365 .driver_data = NVME_QUIRK_NO_DEEPEST_PS | 4366 NVME_QUIRK_IGNORE_DEV_SUBNQN, }, 4367 { PCI_DEVICE(0x126f, 0x2262), /* Silicon Motion generic */ 4368 .driver_data = NVME_QUIRK_NO_DEEPEST_PS | 4369 NVME_QUIRK_BOGUS_NID, }, 4370 { PCI_DEVICE(0x126f, 0x2263), /* Silicon Motion unidentified */ 4371 .driver_data = NVME_QUIRK_NO_NS_DESC_LIST | 4372 NVME_QUIRK_BOGUS_NID, }, 4373 { PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */ 4374 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY | 4375 NVME_QUIRK_NO_NS_DESC_LIST, }, 4376 { PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */ 4377 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 4378 { PCI_DEVICE(0x1c58, 0x0023), /* WDC SN200 adapter */ 4379 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 4380 { PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */ 4381 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 4382 { PCI_DEVICE(0x1c5f, 0x0555), /* Memblaze Pblaze5 adapter */ 4383 .driver_data = NVME_QUIRK_NO_NS_DESC_LIST, }, 4384 { PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */ 4385 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, }, 4386 { PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */ 4387 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY | 4388 NVME_QUIRK_DISABLE_WRITE_ZEROES| 4389 NVME_QUIRK_IGNORE_DEV_SUBNQN, }, 4390 { PCI_DEVICE(0x15b7, 0x5008), /* Sandisk SN530 */ 4391 .driver_data = NVME_QUIRK_BROKEN_MSI }, 4392 { PCI_DEVICE(0x15b7, 0x5009), /* Sandisk SN550 */ 4393 .driver_data = NVME_QUIRK_BROKEN_MSI | 4394 NVME_QUIRK_NO_DEEPEST_PS }, 4395 { PCI_DEVICE(0x1987, 0x5012), /* Phison E12 */ 4396 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4397 { PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */ 4398 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN | 4399 NVME_QUIRK_BOGUS_NID, }, 4400 { PCI_DEVICE(0x1987, 0x5019), /* phison E19 */ 4401 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4402 { PCI_DEVICE(0x1987, 0x5021), /* Phison E21 */ 4403 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4404 { PCI_DEVICE(0x1b4b, 0x1092), /* Lexar 256 GB SSD */ 4405 .driver_data = NVME_QUIRK_NO_NS_DESC_LIST | 4406 NVME_QUIRK_IGNORE_DEV_SUBNQN, }, 4407 { PCI_DEVICE(0x1cc1, 0x33f8), /* ADATA IM2P33F8ABR1 1 TB */ 4408 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4409 { PCI_DEVICE(0x10ec, 0x5762), /* ADATA SX6000LNP */ 4410 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN | 4411 NVME_QUIRK_BOGUS_NID, }, 4412 { PCI_DEVICE(0x10ec, 0x5763), /* ADATA SX6000PNP */ 4413 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4414 { PCI_DEVICE(0x1cc1, 0x8201), /* ADATA SX8200PNP 512GB */ 4415 .driver_data = NVME_QUIRK_NO_DEEPEST_PS | 4416 NVME_QUIRK_IGNORE_DEV_SUBNQN, }, 4417 { PCI_DEVICE(0x1344, 0x5407), /* Micron Technology Inc NVMe SSD */ 4418 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN }, 4419 { PCI_DEVICE(0x1344, 0x6001), /* Micron Nitro NVMe */ 4420 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4421 { PCI_DEVICE(0x1344, 0x6004), /* Micron 4100AT */ 4422 .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, 4423 { PCI_DEVICE(0x1e0f, 0x0032), /* KIOXIA XG10d */ 4424 .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, 4425 { PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */ 4426 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4427 { PCI_DEVICE(0x1c5c, 0x174a), /* SK Hynix P31 SSD */ 4428 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4429 { PCI_DEVICE(0x1c5c, 0x1D59), /* SK Hynix BC901 */ 4430 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4431 { PCI_DEVICE(0x15b7, 0x2001), /* Sandisk Skyhawk */ 4432 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4433 { PCI_DEVICE(0x1d97, 0x2263), /* SPCC / Lexar NM610 */ 4434 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES | 4435 NVME_QUIRK_BOGUS_NID, }, 4436 { PCI_DEVICE(0x144d, 0xa80b), /* Samsung PM9B1 256G and 512G */ 4437 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES | 4438 NVME_QUIRK_BOGUS_NID, }, 4439 { PCI_DEVICE(0x144d, 0xa809), /* Samsung MZALQ256HBJD 256G */ 4440 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4441 { PCI_DEVICE(0x144d, 0xa802), /* Samsung SM953 */ 4442 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4443 { PCI_DEVICE(0x1cc4, 0x6303), /* UMIS RPJTJ512MGE1QDY 512G */ 4444 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4445 { PCI_DEVICE(0x1cc4, 0x6302), /* UMIS RPJTJ256MGE1QDY 256G */ 4446 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4447 { PCI_DEVICE(0x2646, 0x2262), /* KINGSTON SKC2000 NVMe SSD */ 4448 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, 4449 { PCI_DEVICE(0x2646, 0x2263), /* KINGSTON A2000 NVMe SSD */ 4450 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, 4451 { PCI_DEVICE(0x2646, 0x5013), /* Kingston KC3000, Kingston FURY Renegade */ 4452 .driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, }, 4453 { PCI_DEVICE(0x2646, 0x5018), /* KINGSTON OM8SFP4xxxxP OS21012 NVMe SSD */ 4454 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4455 { PCI_DEVICE(0x2646, 0x5016), /* KINGSTON OM3PGP4xxxxP OS21011 NVMe SSD */ 4456 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4457 { PCI_DEVICE(0x2646, 0x501A), /* KINGSTON OM8PGP4xxxxP OS21005 NVMe SSD */ 4458 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4459 { PCI_DEVICE(0x2646, 0x501B), /* KINGSTON OM8PGP4xxxxQ OS21005 NVMe SSD */ 4460 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4461 { PCI_DEVICE(0x2646, 0x501E), /* KINGSTON OM3PGP4xxxxQ OS21011 NVMe SSD */ 4462 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4463 { PCI_DEVICE(0x2646, 0x502F), /* KINGSTON OM3SGP4xxxxK NVMe SSD */ 4464 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, 4465 { PCI_DEVICE(0x1f40, 0x1202), /* Netac Technologies Co. NV3000 NVMe SSD */ 4466 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4467 { PCI_DEVICE(0x1f40, 0x5236), /* Netac Technologies Co. NV7000 NVMe SSD */ 4468 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4469 { PCI_DEVICE(0x1e4B, 0x1001), /* MAXIO MAP1001 */ 4470 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4471 { PCI_DEVICE(0x1e4B, 0x1002), /* MAXIO MAP1002 */ 4472 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4473 { PCI_DEVICE(0x1e4B, 0x1202), /* MAXIO MAP1202 */ 4474 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4475 { PCI_DEVICE(0x1e4B, 0x1602), /* MAXIO MAP1602 */ 4476 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4477 { PCI_DEVICE(0x1cc1, 0x5350), /* ADATA XPG GAMMIX S50 */ 4478 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4479 { PCI_DEVICE(0x1cc1, 0x5370), /* ADATA XPG GAMMIX S70 BLADE */ 4480 .driver_data = NVME_QUIRK_NO_FUA, }, 4481 { PCI_DEVICE(0x1dbe, 0x5216), /* Acer/INNOGRIT FA100/5216 NVMe SSD */ 4482 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4483 { PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */ 4484 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4485 { PCI_DEVICE(0x1e49, 0x0021), /* ZHITAI TiPro5000 NVMe SSD */ 4486 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, 4487 { PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */ 4488 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, 4489 { PCI_DEVICE(0x1fa0, 0x2283), /* Wodposit WPBSNM8-256GTP */ 4490 .driver_data = NVME_QUIRK_NO_SECONDARY_TEMP_THRESH, }, 4491 { PCI_DEVICE(0x025e, 0xf1ac), /* SOLIDIGM P44 pro SSDPFKKW020X7 */ 4492 .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, 4493 { PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */ 4494 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4495 { PCI_DEVICE(0x1d97, 0x1d97), /* Lexar NM620 */ 4496 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4497 { PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */ 4498 .driver_data = NVME_QUIRK_BOGUS_NID | 4499 NVME_QUIRK_IGNORE_DEV_SUBNQN, }, 4500 { PCI_DEVICE(0x10ec, 0x5763), /* TEAMGROUP T-FORCE CARDEA ZERO Z330 SSD */ 4501 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4502 { PCI_DEVICE(0x1e4b, 0x1602), /* HS-SSD-FUTURE 2048G */ 4503 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4504 { PCI_DEVICE(0x10ec, 0x5765), /* TEAMGROUP MP33 2TB SSD */ 4505 .driver_data = NVME_QUIRK_BOGUS_NID, }, 4506 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061), 4507 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, 4508 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065), 4509 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, 4510 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x8061), 4511 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, 4512 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd00), 4513 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, 4514 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd01), 4515 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, 4516 { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0xcd02), 4517 .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, 4518 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001), 4519 /* 4520 * Fix for the Apple controller found in the MacBook8,1 and 4521 * some MacBook7,1 to avoid controller resets and data loss. 4522 */ 4523 .driver_data = NVME_QUIRK_SINGLE_VECTOR | 4524 NVME_QUIRK_QDEPTH_ONE }, 4525 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) }, 4526 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2005), 4527 .driver_data = NVME_QUIRK_SINGLE_VECTOR | 4528 NVME_QUIRK_128_BYTES_SQES | 4529 NVME_QUIRK_SHARED_TAGS | 4530 NVME_QUIRK_SKIP_CID_GEN | 4531 NVME_QUIRK_IDENTIFY_CNS }, 4532 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 4533 { 0, } 4534 }; 4535 MODULE_DEVICE_TABLE(pci, nvme_id_table); 4536 4537 static struct pci_driver nvme_driver = { 4538 .name = "nvme", 4539 .id_table = nvme_id_table, 4540 .probe = nvme_probe, 4541 .remove = nvme_remove, 4542 .shutdown = nvme_shutdown, 4543 .driver = { 4544 .probe_type = PROBE_PREFER_ASYNCHRONOUS, 4545 #ifdef CONFIG_PM_SLEEP 4546 .pm = &nvme_dev_pm_ops, 4547 #endif 4548 }, 4549 .sriov_configure = pci_sriov_configure_simple, 4550 .err_handler = &nvme_err_handler, 4551 }; 4552 4553 static int __init nvme_init(void) 4554 { 4555 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 4556 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 4557 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 4558 BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2); 4559 4560 return pci_register_driver(&nvme_driver); 4561 } 4562 4563 static void __exit nvme_exit(void) 4564 { 4565 kfree(nvme_pci_quirk_list); 4566 pci_unregister_driver(&nvme_driver); 4567 flush_workqueue(nvme_wq); 4568 } 4569 4570 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 4571 MODULE_LICENSE("GPL"); 4572 MODULE_VERSION("1.0"); 4573 MODULE_DESCRIPTION("NVMe host PCIe transport driver"); 4574 MODULE_IMPORT_NS("DMA_BUF"); 4575 module_init(nvme_init); 4576 module_exit(nvme_exit); 4577