1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics RDMA target. 4 * Copyright (c) 2015-2016 HGST, a Western Digital Company. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/atomic.h> 8 #include <linux/blk-integrity.h> 9 #include <linux/ctype.h> 10 #include <linux/delay.h> 11 #include <linux/err.h> 12 #include <linux/init.h> 13 #include <linux/module.h> 14 #include <linux/nvme.h> 15 #include <linux/slab.h> 16 #include <linux/string.h> 17 #include <linux/wait.h> 18 #include <linux/inet.h> 19 #include <linux/unaligned.h> 20 21 #include <rdma/ib_verbs.h> 22 #include <rdma/rdma_cm.h> 23 #include <rdma/rw.h> 24 #include <rdma/ib_cm.h> 25 26 #include <linux/nvme-rdma.h> 27 #include "nvmet.h" 28 29 /* 30 * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data 31 */ 32 #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE PAGE_SIZE 33 #define NVMET_RDMA_MAX_INLINE_SGE 4 34 #define NVMET_RDMA_MAX_INLINE_DATA_SIZE max_t(int, SZ_16K, PAGE_SIZE) 35 36 /* Assume mpsmin == device_page_size == 4KB */ 37 #define NVMET_RDMA_MAX_MDTS 8 38 #define NVMET_RDMA_MAX_METADATA_MDTS 5 39 40 #define NVMET_RDMA_BACKLOG 128 41 42 #define NVMET_RDMA_DISCRETE_RSP_TAG -1 43 44 struct nvmet_rdma_srq; 45 46 struct nvmet_rdma_cmd { 47 struct ib_sge sge[NVMET_RDMA_MAX_INLINE_SGE + 1]; 48 struct ib_cqe cqe; 49 struct ib_recv_wr wr; 50 struct scatterlist inline_sg[NVMET_RDMA_MAX_INLINE_SGE]; 51 struct nvme_command *nvme_cmd; 52 struct nvmet_rdma_queue *queue; 53 struct nvmet_rdma_srq *nsrq; 54 }; 55 56 enum { 57 NVMET_RDMA_REQ_INLINE_DATA = (1 << 0), 58 }; 59 60 struct nvmet_rdma_rsp { 61 struct ib_sge send_sge; 62 struct ib_cqe send_cqe; 63 struct ib_send_wr send_wr; 64 65 struct nvmet_rdma_cmd *cmd; 66 struct nvmet_rdma_queue *queue; 67 68 struct ib_cqe read_cqe; 69 struct ib_cqe write_cqe; 70 struct rdma_rw_ctx rw; 71 72 struct nvmet_req req; 73 74 bool allocated; 75 u8 n_rdma; 76 u32 flags; 77 u32 invalidate_rkey; 78 79 struct list_head wait_list; 80 int tag; 81 }; 82 83 enum nvmet_rdma_queue_state { 84 NVMET_RDMA_Q_CONNECTING, 85 NVMET_RDMA_Q_LIVE, 86 NVMET_RDMA_Q_DISCONNECTING, 87 }; 88 89 struct nvmet_rdma_queue { 90 struct rdma_cm_id *cm_id; 91 struct ib_qp *qp; 92 struct nvmet_port *port; 93 struct ib_cq *cq; 94 atomic_t sq_wr_avail; 95 struct nvmet_rdma_device *dev; 96 struct nvmet_rdma_srq *nsrq; 97 spinlock_t state_lock; 98 enum nvmet_rdma_queue_state state; 99 struct nvmet_cq nvme_cq; 100 struct nvmet_sq nvme_sq; 101 102 struct nvmet_rdma_rsp *rsps; 103 struct sbitmap rsp_tags; 104 struct nvmet_rdma_cmd *cmds; 105 106 struct work_struct release_work; 107 struct list_head rsp_wait_list; 108 struct list_head rsp_wr_wait_list; 109 spinlock_t rsp_wr_wait_lock; 110 111 int idx; 112 int host_qid; 113 int comp_vector; 114 int recv_queue_size; 115 int send_queue_size; 116 117 struct list_head queue_list; 118 }; 119 120 struct nvmet_rdma_port { 121 struct nvmet_port *nport; 122 struct sockaddr_storage addr; 123 struct rdma_cm_id *cm_id; 124 struct delayed_work repair_work; 125 }; 126 127 struct nvmet_rdma_srq { 128 struct ib_srq *srq; 129 struct nvmet_rdma_cmd *cmds; 130 struct nvmet_rdma_device *ndev; 131 }; 132 133 struct nvmet_rdma_device { 134 struct ib_device *device; 135 struct ib_pd *pd; 136 struct nvmet_rdma_srq **srqs; 137 int srq_count; 138 size_t srq_size; 139 struct kref ref; 140 struct list_head entry; 141 int inline_data_size; 142 int inline_page_count; 143 }; 144 145 static bool nvmet_rdma_use_srq; 146 module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444); 147 MODULE_PARM_DESC(use_srq, "Use shared receive queue."); 148 149 static int srq_size_set(const char *val, const struct kernel_param *kp); 150 static const struct kernel_param_ops srq_size_ops = { 151 .set = srq_size_set, 152 .get = param_get_uint, 153 }; 154 155 static unsigned int nvmet_rdma_srq_size = 1024; 156 module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644); 157 MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)"); 158 159 static DEFINE_IDA(nvmet_rdma_queue_ida); 160 static LIST_HEAD(nvmet_rdma_queue_list); 161 static DEFINE_MUTEX(nvmet_rdma_queue_mutex); 162 163 static LIST_HEAD(device_list); 164 static DEFINE_MUTEX(device_list_mutex); 165 166 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp); 167 static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc); 168 static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc); 169 static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc); 170 static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc); 171 static void nvmet_rdma_qp_event(struct ib_event *event, void *priv); 172 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue); 173 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev, 174 struct nvmet_rdma_rsp *r); 175 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev, 176 struct nvmet_rdma_rsp *r, 177 int tag); 178 179 static const struct nvmet_fabrics_ops nvmet_rdma_ops; 180 181 static int srq_size_set(const char *val, const struct kernel_param *kp) 182 { 183 unsigned int n; 184 int ret; 185 186 ret = kstrtouint(val, 10, &n); 187 if (ret != 0 || n < 256) 188 return -EINVAL; 189 190 return param_set_uint(val, kp); 191 } 192 193 static int num_pages(int len) 194 { 195 return 1 + (((len - 1) & PAGE_MASK) >> PAGE_SHIFT); 196 } 197 198 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp *rsp) 199 { 200 return nvme_is_write(rsp->req.cmd) && 201 rsp->req.transfer_len && 202 !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA); 203 } 204 205 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp *rsp) 206 { 207 return !nvme_is_write(rsp->req.cmd) && 208 rsp->req.transfer_len && 209 !rsp->req.cqe->status && 210 !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA); 211 } 212 213 static inline struct nvmet_rdma_rsp * 214 nvmet_rdma_get_rsp(struct nvmet_rdma_queue *queue) 215 { 216 struct nvmet_rdma_rsp *rsp = NULL; 217 int tag; 218 219 tag = sbitmap_get(&queue->rsp_tags); 220 if (tag >= 0) 221 rsp = &queue->rsps[tag]; 222 223 if (unlikely(!rsp)) { 224 int ret; 225 226 rsp = kzalloc_obj(*rsp); 227 if (unlikely(!rsp)) 228 return NULL; 229 ret = nvmet_rdma_alloc_rsp(queue->dev, rsp, 230 NVMET_RDMA_DISCRETE_RSP_TAG); 231 if (unlikely(ret)) { 232 kfree(rsp); 233 return NULL; 234 } 235 } 236 237 return rsp; 238 } 239 240 static inline void 241 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp *rsp) 242 { 243 if (unlikely(rsp->tag == NVMET_RDMA_DISCRETE_RSP_TAG)) { 244 nvmet_rdma_free_rsp(rsp->queue->dev, rsp); 245 kfree(rsp); 246 return; 247 } 248 249 sbitmap_clear_bit(&rsp->queue->rsp_tags, rsp->tag); 250 } 251 252 static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device *ndev, 253 struct nvmet_rdma_cmd *c) 254 { 255 struct scatterlist *sg; 256 struct ib_sge *sge; 257 int i; 258 259 if (!ndev->inline_data_size) 260 return; 261 262 sg = c->inline_sg; 263 sge = &c->sge[1]; 264 265 for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) { 266 if (sge->length) 267 ib_dma_unmap_page(ndev->device, sge->addr, 268 sge->length, DMA_FROM_DEVICE); 269 if (sg_page(sg)) 270 __free_page(sg_page(sg)); 271 } 272 } 273 274 static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device *ndev, 275 struct nvmet_rdma_cmd *c) 276 { 277 struct scatterlist *sg; 278 struct ib_sge *sge; 279 struct page *pg; 280 int len; 281 int i; 282 283 if (!ndev->inline_data_size) 284 return 0; 285 286 sg = c->inline_sg; 287 sg_init_table(sg, ndev->inline_page_count); 288 sge = &c->sge[1]; 289 len = ndev->inline_data_size; 290 291 for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) { 292 pg = alloc_page(GFP_KERNEL); 293 if (!pg) 294 goto out_err; 295 sg_assign_page(sg, pg); 296 sge->addr = ib_dma_map_page(ndev->device, 297 pg, 0, PAGE_SIZE, DMA_FROM_DEVICE); 298 if (ib_dma_mapping_error(ndev->device, sge->addr)) 299 goto out_err; 300 sge->length = min_t(int, len, PAGE_SIZE); 301 sge->lkey = ndev->pd->local_dma_lkey; 302 len -= sge->length; 303 } 304 305 return 0; 306 out_err: 307 for (; i >= 0; i--, sg--, sge--) { 308 if (sge->length) 309 ib_dma_unmap_page(ndev->device, sge->addr, 310 sge->length, DMA_FROM_DEVICE); 311 if (sg_page(sg)) 312 __free_page(sg_page(sg)); 313 } 314 return -ENOMEM; 315 } 316 317 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device *ndev, 318 struct nvmet_rdma_cmd *c, bool admin) 319 { 320 /* NVMe command / RDMA RECV */ 321 c->nvme_cmd = kmalloc_obj(*c->nvme_cmd); 322 if (!c->nvme_cmd) 323 goto out; 324 325 c->sge[0].addr = ib_dma_map_single(ndev->device, c->nvme_cmd, 326 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE); 327 if (ib_dma_mapping_error(ndev->device, c->sge[0].addr)) 328 goto out_free_cmd; 329 330 c->sge[0].length = sizeof(*c->nvme_cmd); 331 c->sge[0].lkey = ndev->pd->local_dma_lkey; 332 333 if (!admin && nvmet_rdma_alloc_inline_pages(ndev, c)) 334 goto out_unmap_cmd; 335 336 c->cqe.done = nvmet_rdma_recv_done; 337 338 c->wr.wr_cqe = &c->cqe; 339 c->wr.sg_list = c->sge; 340 c->wr.num_sge = admin ? 1 : ndev->inline_page_count + 1; 341 342 return 0; 343 344 out_unmap_cmd: 345 ib_dma_unmap_single(ndev->device, c->sge[0].addr, 346 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE); 347 out_free_cmd: 348 kfree(c->nvme_cmd); 349 350 out: 351 return -ENOMEM; 352 } 353 354 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device *ndev, 355 struct nvmet_rdma_cmd *c, bool admin) 356 { 357 if (!admin) 358 nvmet_rdma_free_inline_pages(ndev, c); 359 ib_dma_unmap_single(ndev->device, c->sge[0].addr, 360 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE); 361 kfree(c->nvme_cmd); 362 } 363 364 static struct nvmet_rdma_cmd * 365 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device *ndev, 366 int nr_cmds, bool admin) 367 { 368 struct nvmet_rdma_cmd *cmds; 369 int ret = -EINVAL, i; 370 371 cmds = kvzalloc_objs(struct nvmet_rdma_cmd, nr_cmds); 372 if (!cmds) 373 goto out; 374 375 for (i = 0; i < nr_cmds; i++) { 376 ret = nvmet_rdma_alloc_cmd(ndev, cmds + i, admin); 377 if (ret) 378 goto out_free; 379 } 380 381 return cmds; 382 383 out_free: 384 while (--i >= 0) 385 nvmet_rdma_free_cmd(ndev, cmds + i, admin); 386 kvfree(cmds); 387 out: 388 return ERR_PTR(ret); 389 } 390 391 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device *ndev, 392 struct nvmet_rdma_cmd *cmds, int nr_cmds, bool admin) 393 { 394 int i; 395 396 for (i = 0; i < nr_cmds; i++) 397 nvmet_rdma_free_cmd(ndev, cmds + i, admin); 398 kvfree(cmds); 399 } 400 401 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev, 402 struct nvmet_rdma_rsp *r, int tag) 403 { 404 /* NVMe CQE / RDMA SEND */ 405 r->req.cqe = kmalloc_obj(*r->req.cqe); 406 if (!r->req.cqe) 407 goto out; 408 409 r->send_sge.addr = ib_dma_map_single(ndev->device, r->req.cqe, 410 sizeof(*r->req.cqe), DMA_TO_DEVICE); 411 if (ib_dma_mapping_error(ndev->device, r->send_sge.addr)) 412 goto out_free_rsp; 413 414 if (ib_dma_pci_p2p_dma_supported(ndev->device)) 415 r->req.p2p_client = &ndev->device->dev; 416 r->send_sge.length = sizeof(*r->req.cqe); 417 r->send_sge.lkey = ndev->pd->local_dma_lkey; 418 419 r->send_cqe.done = nvmet_rdma_send_done; 420 421 r->send_wr.wr_cqe = &r->send_cqe; 422 r->send_wr.sg_list = &r->send_sge; 423 r->send_wr.num_sge = 1; 424 r->send_wr.send_flags = IB_SEND_SIGNALED; 425 426 /* Data In / RDMA READ */ 427 r->read_cqe.done = nvmet_rdma_read_data_done; 428 /* Data Out / RDMA WRITE */ 429 r->write_cqe.done = nvmet_rdma_write_data_done; 430 r->tag = tag; 431 432 return 0; 433 434 out_free_rsp: 435 kfree(r->req.cqe); 436 out: 437 return -ENOMEM; 438 } 439 440 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev, 441 struct nvmet_rdma_rsp *r) 442 { 443 ib_dma_unmap_single(ndev->device, r->send_sge.addr, 444 sizeof(*r->req.cqe), DMA_TO_DEVICE); 445 kfree(r->req.cqe); 446 } 447 448 static int 449 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue *queue) 450 { 451 struct nvmet_rdma_device *ndev = queue->dev; 452 int nr_rsps = queue->recv_queue_size * 2; 453 int ret = -ENOMEM, i; 454 455 if (sbitmap_init_node(&queue->rsp_tags, nr_rsps, -1, GFP_KERNEL, 456 NUMA_NO_NODE, false, true)) 457 goto out; 458 459 queue->rsps = kvzalloc_objs(struct nvmet_rdma_rsp, nr_rsps); 460 if (!queue->rsps) 461 goto out_free_sbitmap; 462 463 for (i = 0; i < nr_rsps; i++) { 464 struct nvmet_rdma_rsp *rsp = &queue->rsps[i]; 465 466 ret = nvmet_rdma_alloc_rsp(ndev, rsp, i); 467 if (ret) 468 goto out_free; 469 } 470 471 return 0; 472 473 out_free: 474 while (--i >= 0) 475 nvmet_rdma_free_rsp(ndev, &queue->rsps[i]); 476 kvfree(queue->rsps); 477 out_free_sbitmap: 478 sbitmap_free(&queue->rsp_tags); 479 out: 480 return ret; 481 } 482 483 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue *queue) 484 { 485 struct nvmet_rdma_device *ndev = queue->dev; 486 int i, nr_rsps = queue->recv_queue_size * 2; 487 488 for (i = 0; i < nr_rsps; i++) 489 nvmet_rdma_free_rsp(ndev, &queue->rsps[i]); 490 kvfree(queue->rsps); 491 sbitmap_free(&queue->rsp_tags); 492 } 493 494 static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev, 495 struct nvmet_rdma_cmd *cmd) 496 { 497 int ret; 498 499 ib_dma_sync_single_for_device(ndev->device, 500 cmd->sge[0].addr, cmd->sge[0].length, 501 DMA_FROM_DEVICE); 502 503 if (cmd->nsrq) 504 ret = ib_post_srq_recv(cmd->nsrq->srq, &cmd->wr, NULL); 505 else 506 ret = ib_post_recv(cmd->queue->qp, &cmd->wr, NULL); 507 508 if (unlikely(ret)) 509 pr_err("post_recv cmd failed\n"); 510 511 return ret; 512 } 513 514 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue) 515 { 516 spin_lock(&queue->rsp_wr_wait_lock); 517 while (!list_empty(&queue->rsp_wr_wait_list)) { 518 struct nvmet_rdma_rsp *rsp; 519 bool ret; 520 521 rsp = list_entry(queue->rsp_wr_wait_list.next, 522 struct nvmet_rdma_rsp, wait_list); 523 list_del(&rsp->wait_list); 524 525 spin_unlock(&queue->rsp_wr_wait_lock); 526 ret = nvmet_rdma_execute_command(rsp); 527 spin_lock(&queue->rsp_wr_wait_lock); 528 529 if (!ret) { 530 list_add(&rsp->wait_list, &queue->rsp_wr_wait_list); 531 break; 532 } 533 } 534 spin_unlock(&queue->rsp_wr_wait_lock); 535 } 536 537 static u16 nvmet_rdma_check_pi_status(struct ib_mr *sig_mr) 538 { 539 struct ib_mr_status mr_status; 540 int ret; 541 u16 status = 0; 542 543 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 544 if (ret) { 545 pr_err("ib_check_mr_status failed, ret %d\n", ret); 546 return NVME_SC_INVALID_PI; 547 } 548 549 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 550 switch (mr_status.sig_err.err_type) { 551 case IB_SIG_BAD_GUARD: 552 status = NVME_SC_GUARD_CHECK; 553 break; 554 case IB_SIG_BAD_REFTAG: 555 status = NVME_SC_REFTAG_CHECK; 556 break; 557 case IB_SIG_BAD_APPTAG: 558 status = NVME_SC_APPTAG_CHECK; 559 break; 560 } 561 pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n", 562 mr_status.sig_err.err_type, 563 mr_status.sig_err.expected, 564 mr_status.sig_err.actual); 565 } 566 567 return status; 568 } 569 570 static void nvmet_rdma_set_sig_domain(struct blk_integrity *bi, 571 struct nvme_command *cmd, struct ib_sig_domain *domain, 572 u16 control, u8 pi_type) 573 { 574 domain->sig_type = IB_SIG_TYPE_T10_DIF; 575 domain->sig.dif.bg_type = IB_T10DIF_CRC; 576 domain->sig.dif.pi_interval = 1 << bi->interval_exp; 577 domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag); 578 if (control & NVME_RW_PRINFO_PRCHK_REF) 579 domain->sig.dif.ref_remap = true; 580 581 domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.lbat); 582 domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.lbatm); 583 domain->sig.dif.app_escape = true; 584 if (pi_type == NVME_NS_DPS_PI_TYPE3) 585 domain->sig.dif.ref_escape = true; 586 } 587 588 static void nvmet_rdma_set_sig_attrs(struct nvmet_req *req, 589 struct ib_sig_attrs *sig_attrs) 590 { 591 struct nvme_command *cmd = req->cmd; 592 u16 control = le16_to_cpu(cmd->rw.control); 593 u8 pi_type = req->ns->pi_type; 594 struct blk_integrity *bi; 595 596 bi = bdev_get_integrity(req->ns->bdev); 597 598 memset(sig_attrs, 0, sizeof(*sig_attrs)); 599 600 if (control & NVME_RW_PRINFO_PRACT) { 601 /* for WRITE_INSERT/READ_STRIP no wire domain */ 602 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 603 nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control, 604 pi_type); 605 /* Clear the PRACT bit since HCA will generate/verify the PI */ 606 control &= ~NVME_RW_PRINFO_PRACT; 607 cmd->rw.control = cpu_to_le16(control); 608 /* PI is added by the HW */ 609 req->transfer_len += req->metadata_len; 610 } else { 611 /* for WRITE_PASS/READ_PASS both wire/memory domains exist */ 612 nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control, 613 pi_type); 614 nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control, 615 pi_type); 616 } 617 618 if (control & NVME_RW_PRINFO_PRCHK_REF) 619 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 620 if (control & NVME_RW_PRINFO_PRCHK_GUARD) 621 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 622 if (control & NVME_RW_PRINFO_PRCHK_APP) 623 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 624 } 625 626 static int nvmet_rdma_rw_ctx_init(struct nvmet_rdma_rsp *rsp, u64 addr, u32 key, 627 struct ib_sig_attrs *sig_attrs) 628 { 629 struct rdma_cm_id *cm_id = rsp->queue->cm_id; 630 struct nvmet_req *req = &rsp->req; 631 int ret; 632 633 if (req->metadata_len) 634 ret = rdma_rw_ctx_signature_init(&rsp->rw, cm_id->qp, 635 cm_id->port_num, req->sg, req->sg_cnt, 636 req->metadata_sg, req->metadata_sg_cnt, sig_attrs, 637 addr, key, nvmet_data_dir(req)); 638 else 639 ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num, 640 req->sg, req->sg_cnt, 0, addr, key, 641 nvmet_data_dir(req)); 642 643 return ret; 644 } 645 646 static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) 647 { 648 struct rdma_cm_id *cm_id = rsp->queue->cm_id; 649 struct nvmet_req *req = &rsp->req; 650 651 if (req->metadata_len) 652 rdma_rw_ctx_destroy_signature(&rsp->rw, cm_id->qp, 653 cm_id->port_num, req->sg, req->sg_cnt, 654 req->metadata_sg, req->metadata_sg_cnt, 655 nvmet_data_dir(req)); 656 else 657 rdma_rw_ctx_destroy(&rsp->rw, cm_id->qp, cm_id->port_num, 658 req->sg, req->sg_cnt, nvmet_data_dir(req)); 659 } 660 661 static void nvmet_rdma_free_rsp_resources(struct nvmet_rdma_rsp *rsp) 662 { 663 struct nvmet_rdma_queue *queue = rsp->queue; 664 665 if (rsp->n_rdma) 666 nvmet_rdma_rw_ctx_destroy(rsp); 667 668 if (rsp->req.sg < rsp->cmd->inline_sg || 669 rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) 670 nvmet_req_free_sgls(&rsp->req); 671 } 672 673 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) 674 { 675 struct nvmet_rdma_queue *queue = rsp->queue; 676 677 atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); 678 679 nvmet_rdma_free_rsp_resources(rsp); 680 681 if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) 682 nvmet_rdma_process_wr_wait_list(queue); 683 684 nvmet_rdma_put_rsp(rsp); 685 } 686 687 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue *queue) 688 { 689 if (queue->nvme_sq.ctrl) { 690 nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); 691 } else { 692 /* 693 * we didn't setup the controller yet in case 694 * of admin connect error, just disconnect and 695 * cleanup the queue 696 */ 697 nvmet_rdma_queue_disconnect(queue); 698 } 699 } 700 701 static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc) 702 { 703 struct nvmet_rdma_rsp *rsp = 704 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, send_cqe); 705 struct nvmet_rdma_queue *queue = wc->qp->qp_context; 706 707 nvmet_rdma_release_rsp(rsp); 708 709 if (unlikely(wc->status != IB_WC_SUCCESS && 710 wc->status != IB_WC_WR_FLUSH_ERR)) { 711 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n", 712 wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status); 713 nvmet_rdma_error_comp(queue); 714 } 715 } 716 717 static void nvmet_rdma_queue_response(struct nvmet_req *req) 718 { 719 struct nvmet_rdma_rsp *rsp = 720 container_of(req, struct nvmet_rdma_rsp, req); 721 struct rdma_cm_id *cm_id = rsp->queue->cm_id; 722 struct ib_send_wr *first_wr; 723 724 if (rsp->invalidate_rkey) { 725 rsp->send_wr.opcode = IB_WR_SEND_WITH_INV; 726 rsp->send_wr.ex.invalidate_rkey = rsp->invalidate_rkey; 727 } else { 728 rsp->send_wr.opcode = IB_WR_SEND; 729 } 730 731 if (nvmet_rdma_need_data_out(rsp)) { 732 if (rsp->req.metadata_len) 733 first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, 734 cm_id->port_num, &rsp->write_cqe, NULL); 735 else 736 first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, 737 cm_id->port_num, NULL, &rsp->send_wr); 738 } else { 739 first_wr = &rsp->send_wr; 740 } 741 742 nvmet_rdma_post_recv(rsp->queue->dev, rsp->cmd); 743 744 ib_dma_sync_single_for_device(rsp->queue->dev->device, 745 rsp->send_sge.addr, rsp->send_sge.length, 746 DMA_TO_DEVICE); 747 748 if (unlikely(ib_post_send(cm_id->qp, first_wr, NULL))) { 749 pr_err("sending cmd response failed\n"); 750 nvmet_rdma_release_rsp(rsp); 751 } 752 } 753 754 static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc) 755 { 756 struct nvmet_rdma_rsp *rsp = 757 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, read_cqe); 758 struct nvmet_rdma_queue *queue = wc->qp->qp_context; 759 u16 status = 0; 760 761 WARN_ON(rsp->n_rdma <= 0); 762 atomic_add(rsp->n_rdma, &queue->sq_wr_avail); 763 rsp->n_rdma = 0; 764 765 if (unlikely(wc->status != IB_WC_SUCCESS)) { 766 nvmet_rdma_rw_ctx_destroy(rsp); 767 nvmet_req_uninit(&rsp->req); 768 nvmet_rdma_release_rsp(rsp); 769 if (wc->status != IB_WC_WR_FLUSH_ERR) { 770 pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n", 771 wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status); 772 nvmet_rdma_error_comp(queue); 773 } 774 return; 775 } 776 777 if (rsp->req.metadata_len) 778 status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr); 779 nvmet_rdma_rw_ctx_destroy(rsp); 780 781 if (unlikely(status)) 782 nvmet_req_complete(&rsp->req, status); 783 else 784 rsp->req.execute(&rsp->req); 785 } 786 787 static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) 788 { 789 struct nvmet_rdma_rsp *rsp = 790 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, write_cqe); 791 struct nvmet_rdma_queue *queue = wc->qp->qp_context; 792 struct rdma_cm_id *cm_id = rsp->queue->cm_id; 793 u16 status; 794 795 if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) 796 return; 797 798 WARN_ON(rsp->n_rdma <= 0); 799 atomic_add(rsp->n_rdma, &queue->sq_wr_avail); 800 rsp->n_rdma = 0; 801 802 if (unlikely(wc->status != IB_WC_SUCCESS)) { 803 nvmet_rdma_rw_ctx_destroy(rsp); 804 nvmet_req_uninit(&rsp->req); 805 nvmet_rdma_release_rsp(rsp); 806 if (wc->status != IB_WC_WR_FLUSH_ERR) { 807 pr_info("RDMA WRITE for CQE failed with status %s (%d).\n", 808 ib_wc_status_msg(wc->status), wc->status); 809 nvmet_rdma_error_comp(queue); 810 } 811 return; 812 } 813 814 /* 815 * Upon RDMA completion check the signature status 816 * - if succeeded send good NVMe response 817 * - if failed send bad NVMe response with appropriate error 818 */ 819 status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr); 820 if (unlikely(status)) 821 rsp->req.cqe->status = cpu_to_le16(status << 1); 822 nvmet_rdma_rw_ctx_destroy(rsp); 823 824 if (unlikely(ib_post_send(cm_id->qp, &rsp->send_wr, NULL))) { 825 pr_err("sending cmd response failed\n"); 826 nvmet_rdma_release_rsp(rsp); 827 } 828 } 829 830 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len, 831 u64 off) 832 { 833 u64 page_off = off % PAGE_SIZE; 834 u64 page_idx = off / PAGE_SIZE; 835 int sg_count = num_pages(page_off + len); 836 struct scatterlist *sg; 837 int i; 838 839 sg = &rsp->cmd->inline_sg[page_idx]; 840 for (i = 0; i < sg_count; i++, sg++) { 841 if (i < sg_count - 1) 842 sg_unmark_end(sg); 843 else 844 sg_mark_end(sg); 845 sg->offset = page_off; 846 sg->length = min_t(u64, len, PAGE_SIZE - page_off); 847 len -= sg->length; 848 page_off = 0; 849 } 850 851 rsp->req.sg = &rsp->cmd->inline_sg[page_idx]; 852 rsp->req.sg_cnt = sg_count; 853 } 854 855 static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp) 856 { 857 struct nvme_sgl_desc *sgl = &rsp->req.cmd->common.dptr.sgl; 858 u64 off = le64_to_cpu(sgl->addr); 859 u32 len = le32_to_cpu(sgl->length); 860 861 if (!nvme_is_write(rsp->req.cmd)) { 862 rsp->req.error_loc = 863 offsetof(struct nvme_common_command, opcode); 864 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 865 } 866 867 if (off + len > rsp->queue->dev->inline_data_size) { 868 pr_err("invalid inline data offset!\n"); 869 return NVME_SC_SGL_INVALID_OFFSET | NVME_STATUS_DNR; 870 } 871 872 /* no data command? */ 873 if (!len) 874 return 0; 875 876 nvmet_rdma_use_inline_sg(rsp, len, off); 877 rsp->flags |= NVMET_RDMA_REQ_INLINE_DATA; 878 rsp->req.transfer_len += len; 879 return 0; 880 } 881 882 static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp, 883 struct nvme_keyed_sgl_desc *sgl, bool invalidate) 884 { 885 u64 addr = le64_to_cpu(sgl->addr); 886 u32 key = get_unaligned_le32(sgl->key); 887 struct ib_sig_attrs sig_attrs; 888 int ret; 889 890 rsp->req.transfer_len = get_unaligned_le24(sgl->length); 891 892 /* no data command? */ 893 if (!rsp->req.transfer_len) 894 return 0; 895 896 if (rsp->req.metadata_len) 897 nvmet_rdma_set_sig_attrs(&rsp->req, &sig_attrs); 898 899 ret = nvmet_req_alloc_sgls(&rsp->req); 900 if (unlikely(ret < 0)) 901 goto error_out; 902 903 ret = nvmet_rdma_rw_ctx_init(rsp, addr, key, &sig_attrs); 904 if (unlikely(ret < 0)) 905 goto error_out; 906 rsp->n_rdma += ret; 907 908 if (invalidate) 909 rsp->invalidate_rkey = key; 910 911 return 0; 912 913 error_out: 914 rsp->req.transfer_len = 0; 915 return NVME_SC_INTERNAL; 916 } 917 918 static u16 nvmet_rdma_map_sgl(struct nvmet_rdma_rsp *rsp) 919 { 920 struct nvme_keyed_sgl_desc *sgl = &rsp->req.cmd->common.dptr.ksgl; 921 922 switch (sgl->type >> 4) { 923 case NVME_SGL_FMT_DATA_DESC: 924 switch (sgl->type & 0xf) { 925 case NVME_SGL_FMT_OFFSET: 926 return nvmet_rdma_map_sgl_inline(rsp); 927 default: 928 pr_err("invalid SGL subtype: %#x\n", sgl->type); 929 rsp->req.error_loc = 930 offsetof(struct nvme_common_command, dptr); 931 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 932 } 933 case NVME_KEY_SGL_FMT_DATA_DESC: 934 switch (sgl->type & 0xf) { 935 case NVME_SGL_FMT_ADDRESS | NVME_SGL_FMT_INVALIDATE: 936 return nvmet_rdma_map_sgl_keyed(rsp, sgl, true); 937 case NVME_SGL_FMT_ADDRESS: 938 return nvmet_rdma_map_sgl_keyed(rsp, sgl, false); 939 default: 940 pr_err("invalid SGL subtype: %#x\n", sgl->type); 941 rsp->req.error_loc = 942 offsetof(struct nvme_common_command, dptr); 943 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 944 } 945 default: 946 pr_err("invalid SGL type: %#x\n", sgl->type); 947 rsp->req.error_loc = offsetof(struct nvme_common_command, dptr); 948 return NVME_SC_SGL_INVALID_TYPE | NVME_STATUS_DNR; 949 } 950 } 951 952 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp) 953 { 954 struct nvmet_rdma_queue *queue = rsp->queue; 955 956 if (unlikely(atomic_sub_return(1 + rsp->n_rdma, 957 &queue->sq_wr_avail) < 0)) { 958 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n", 959 1 + rsp->n_rdma, queue->idx, 960 queue->nvme_sq.ctrl->cntlid); 961 atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); 962 return false; 963 } 964 965 if (nvmet_rdma_need_data_in(rsp)) { 966 if (rdma_rw_ctx_post(&rsp->rw, queue->qp, 967 queue->cm_id->port_num, &rsp->read_cqe, NULL)) 968 nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR); 969 } else { 970 rsp->req.execute(&rsp->req); 971 } 972 973 return true; 974 } 975 976 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue, 977 struct nvmet_rdma_rsp *cmd) 978 { 979 u16 status; 980 981 ib_dma_sync_single_for_cpu(queue->dev->device, 982 cmd->cmd->sge[0].addr, cmd->cmd->sge[0].length, 983 DMA_FROM_DEVICE); 984 ib_dma_sync_single_for_cpu(queue->dev->device, 985 cmd->send_sge.addr, cmd->send_sge.length, 986 DMA_TO_DEVICE); 987 988 if (!nvmet_req_init(&cmd->req, &queue->nvme_sq, &nvmet_rdma_ops)) 989 return; 990 991 status = nvmet_rdma_map_sgl(cmd); 992 if (status) 993 goto out_err; 994 995 if (unlikely(!nvmet_rdma_execute_command(cmd))) { 996 spin_lock(&queue->rsp_wr_wait_lock); 997 list_add_tail(&cmd->wait_list, &queue->rsp_wr_wait_list); 998 spin_unlock(&queue->rsp_wr_wait_lock); 999 } 1000 1001 return; 1002 1003 out_err: 1004 nvmet_req_complete(&cmd->req, status); 1005 } 1006 1007 static bool nvmet_rdma_recv_not_live(struct nvmet_rdma_queue *queue, 1008 struct nvmet_rdma_rsp *rsp) 1009 { 1010 unsigned long flags; 1011 bool ret = true; 1012 1013 spin_lock_irqsave(&queue->state_lock, flags); 1014 /* 1015 * recheck queue state is not live to prevent a race condition 1016 * with RDMA_CM_EVENT_ESTABLISHED handler. 1017 */ 1018 if (queue->state == NVMET_RDMA_Q_LIVE) 1019 ret = false; 1020 else if (queue->state == NVMET_RDMA_Q_CONNECTING) 1021 list_add_tail(&rsp->wait_list, &queue->rsp_wait_list); 1022 else 1023 nvmet_rdma_put_rsp(rsp); 1024 spin_unlock_irqrestore(&queue->state_lock, flags); 1025 return ret; 1026 } 1027 1028 static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1029 { 1030 struct nvmet_rdma_cmd *cmd = 1031 container_of(wc->wr_cqe, struct nvmet_rdma_cmd, cqe); 1032 struct nvmet_rdma_queue *queue = wc->qp->qp_context; 1033 struct nvmet_rdma_rsp *rsp; 1034 1035 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1036 if (wc->status != IB_WC_WR_FLUSH_ERR) { 1037 pr_err("RECV for CQE 0x%p failed with status %s (%d)\n", 1038 wc->wr_cqe, ib_wc_status_msg(wc->status), 1039 wc->status); 1040 nvmet_rdma_error_comp(queue); 1041 } 1042 return; 1043 } 1044 1045 if (unlikely(wc->byte_len < sizeof(struct nvme_command))) { 1046 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n"); 1047 nvmet_rdma_error_comp(queue); 1048 return; 1049 } 1050 1051 cmd->queue = queue; 1052 rsp = nvmet_rdma_get_rsp(queue); 1053 if (unlikely(!rsp)) { 1054 /* 1055 * we get here only under memory pressure, 1056 * silently drop and have the host retry 1057 * as we can't even fail it. 1058 */ 1059 nvmet_rdma_post_recv(queue->dev, cmd); 1060 return; 1061 } 1062 rsp->queue = queue; 1063 rsp->cmd = cmd; 1064 rsp->flags = 0; 1065 rsp->req.cmd = cmd->nvme_cmd; 1066 rsp->req.port = queue->port; 1067 rsp->n_rdma = 0; 1068 rsp->invalidate_rkey = 0; 1069 1070 if (unlikely(queue->state != NVMET_RDMA_Q_LIVE) && 1071 nvmet_rdma_recv_not_live(queue, rsp)) 1072 return; 1073 1074 nvmet_rdma_handle_command(queue, rsp); 1075 } 1076 1077 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_srq *nsrq) 1078 { 1079 nvmet_rdma_free_cmds(nsrq->ndev, nsrq->cmds, nsrq->ndev->srq_size, 1080 false); 1081 ib_destroy_srq(nsrq->srq); 1082 1083 kfree(nsrq); 1084 } 1085 1086 static void nvmet_rdma_destroy_srqs(struct nvmet_rdma_device *ndev) 1087 { 1088 int i; 1089 1090 if (!ndev->srqs) 1091 return; 1092 1093 for (i = 0; i < ndev->srq_count; i++) 1094 nvmet_rdma_destroy_srq(ndev->srqs[i]); 1095 1096 kfree(ndev->srqs); 1097 } 1098 1099 static struct nvmet_rdma_srq * 1100 nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev) 1101 { 1102 struct ib_srq_init_attr srq_attr = { NULL, }; 1103 size_t srq_size = ndev->srq_size; 1104 struct nvmet_rdma_srq *nsrq; 1105 struct ib_srq *srq; 1106 int ret, i; 1107 1108 nsrq = kzalloc_obj(*nsrq); 1109 if (!nsrq) 1110 return ERR_PTR(-ENOMEM); 1111 1112 srq_attr.attr.max_wr = srq_size; 1113 srq_attr.attr.max_sge = 1 + ndev->inline_page_count; 1114 srq_attr.attr.srq_limit = 0; 1115 srq_attr.srq_type = IB_SRQT_BASIC; 1116 srq = ib_create_srq(ndev->pd, &srq_attr); 1117 if (IS_ERR(srq)) { 1118 ret = PTR_ERR(srq); 1119 goto out_free; 1120 } 1121 1122 nsrq->cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false); 1123 if (IS_ERR(nsrq->cmds)) { 1124 ret = PTR_ERR(nsrq->cmds); 1125 goto out_destroy_srq; 1126 } 1127 1128 nsrq->srq = srq; 1129 nsrq->ndev = ndev; 1130 1131 for (i = 0; i < srq_size; i++) { 1132 nsrq->cmds[i].nsrq = nsrq; 1133 ret = nvmet_rdma_post_recv(ndev, &nsrq->cmds[i]); 1134 if (ret) 1135 goto out_free_cmds; 1136 } 1137 1138 return nsrq; 1139 1140 out_free_cmds: 1141 nvmet_rdma_free_cmds(ndev, nsrq->cmds, srq_size, false); 1142 out_destroy_srq: 1143 ib_destroy_srq(srq); 1144 out_free: 1145 kfree(nsrq); 1146 return ERR_PTR(ret); 1147 } 1148 1149 static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev) 1150 { 1151 int i, ret; 1152 1153 if (!ndev->device->attrs.max_srq_wr || !ndev->device->attrs.max_srq) { 1154 /* 1155 * If SRQs aren't supported we just go ahead and use normal 1156 * non-shared receive queues. 1157 */ 1158 pr_info("SRQ requested but not supported.\n"); 1159 return 0; 1160 } 1161 1162 ndev->srq_size = min(ndev->device->attrs.max_srq_wr, 1163 nvmet_rdma_srq_size); 1164 ndev->srq_count = min_t(u32, ndev->device->num_comp_vectors, 1165 ndev->device->attrs.max_srq); 1166 1167 ndev->srqs = kzalloc_objs(*ndev->srqs, ndev->srq_count); 1168 if (!ndev->srqs) 1169 return -ENOMEM; 1170 1171 for (i = 0; i < ndev->srq_count; i++) { 1172 ndev->srqs[i] = nvmet_rdma_init_srq(ndev); 1173 if (IS_ERR(ndev->srqs[i])) { 1174 ret = PTR_ERR(ndev->srqs[i]); 1175 goto err_srq; 1176 } 1177 } 1178 1179 return 0; 1180 1181 err_srq: 1182 while (--i >= 0) 1183 nvmet_rdma_destroy_srq(ndev->srqs[i]); 1184 kfree(ndev->srqs); 1185 return ret; 1186 } 1187 1188 static void nvmet_rdma_free_dev(struct kref *ref) 1189 { 1190 struct nvmet_rdma_device *ndev = 1191 container_of(ref, struct nvmet_rdma_device, ref); 1192 1193 mutex_lock(&device_list_mutex); 1194 list_del(&ndev->entry); 1195 mutex_unlock(&device_list_mutex); 1196 1197 nvmet_rdma_destroy_srqs(ndev); 1198 ib_dealloc_pd(ndev->pd); 1199 1200 kfree(ndev); 1201 } 1202 1203 static struct nvmet_rdma_device * 1204 nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id) 1205 { 1206 struct nvmet_rdma_port *port = cm_id->context; 1207 struct nvmet_port *nport = port->nport; 1208 struct nvmet_rdma_device *ndev; 1209 int inline_page_count; 1210 u32 inline_sge_count; 1211 int ret; 1212 1213 mutex_lock(&device_list_mutex); 1214 list_for_each_entry(ndev, &device_list, entry) { 1215 if (ndev->device->node_guid == cm_id->device->node_guid && 1216 kref_get_unless_zero(&ndev->ref)) 1217 goto out_unlock; 1218 } 1219 1220 ndev = kzalloc_obj(*ndev); 1221 if (!ndev) 1222 goto out_err; 1223 1224 inline_page_count = num_pages(nport->inline_data_size); 1225 inline_sge_count = max(cm_id->device->attrs.max_sge_rd, 1226 cm_id->device->attrs.max_recv_sge); 1227 if (inline_sge_count) 1228 inline_sge_count--; 1229 if (inline_page_count > inline_sge_count) { 1230 pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n", 1231 nport->inline_data_size, cm_id->device->name, 1232 inline_sge_count * PAGE_SIZE); 1233 nport->inline_data_size = inline_sge_count * PAGE_SIZE; 1234 inline_page_count = inline_sge_count; 1235 } 1236 ndev->inline_data_size = nport->inline_data_size; 1237 ndev->inline_page_count = inline_page_count; 1238 1239 if (nport->pi_enable && !(cm_id->device->attrs.kernel_cap_flags & 1240 IBK_INTEGRITY_HANDOVER)) { 1241 pr_warn("T10-PI is not supported by device %s. Disabling it\n", 1242 cm_id->device->name); 1243 nport->pi_enable = false; 1244 } 1245 1246 ndev->device = cm_id->device; 1247 kref_init(&ndev->ref); 1248 1249 ndev->pd = ib_alloc_pd(ndev->device, 0); 1250 if (IS_ERR(ndev->pd)) 1251 goto out_free_dev; 1252 1253 if (nvmet_rdma_use_srq) { 1254 ret = nvmet_rdma_init_srqs(ndev); 1255 if (ret) 1256 goto out_free_pd; 1257 } 1258 1259 list_add(&ndev->entry, &device_list); 1260 out_unlock: 1261 mutex_unlock(&device_list_mutex); 1262 pr_debug("added %s.\n", ndev->device->name); 1263 return ndev; 1264 1265 out_free_pd: 1266 ib_dealloc_pd(ndev->pd); 1267 out_free_dev: 1268 kfree(ndev); 1269 out_err: 1270 mutex_unlock(&device_list_mutex); 1271 return NULL; 1272 } 1273 1274 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) 1275 { 1276 struct ib_qp_init_attr qp_attr = { }; 1277 struct nvmet_rdma_device *ndev = queue->dev; 1278 int nr_cqe, ret, i, factor; 1279 1280 /* 1281 * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND. 1282 */ 1283 nr_cqe = queue->recv_queue_size + 2 * queue->send_queue_size; 1284 1285 queue->cq = ib_cq_pool_get(ndev->device, nr_cqe + 1, 1286 queue->comp_vector, IB_POLL_WORKQUEUE); 1287 if (IS_ERR(queue->cq)) { 1288 ret = PTR_ERR(queue->cq); 1289 pr_err("failed to create CQ cqe= %d ret= %d\n", 1290 nr_cqe + 1, ret); 1291 goto out; 1292 } 1293 1294 qp_attr.qp_context = queue; 1295 qp_attr.event_handler = nvmet_rdma_qp_event; 1296 qp_attr.send_cq = queue->cq; 1297 qp_attr.recv_cq = queue->cq; 1298 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 1299 qp_attr.qp_type = IB_QPT_RC; 1300 /* +1 for drain */ 1301 qp_attr.cap.max_send_wr = queue->send_queue_size + 1; 1302 factor = rdma_rw_mr_factor(ndev->device, queue->cm_id->port_num, 1303 1 << NVMET_RDMA_MAX_MDTS); 1304 qp_attr.cap.max_rdma_ctxs = queue->send_queue_size * factor; 1305 qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd, 1306 ndev->device->attrs.max_send_sge); 1307 1308 if (queue->nsrq) { 1309 qp_attr.srq = queue->nsrq->srq; 1310 } else { 1311 /* +1 for drain */ 1312 qp_attr.cap.max_recv_wr = 1 + queue->recv_queue_size; 1313 qp_attr.cap.max_recv_sge = 1 + ndev->inline_page_count; 1314 } 1315 1316 if (queue->port->pi_enable && queue->host_qid) 1317 qp_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 1318 1319 ret = rdma_create_qp(queue->cm_id, ndev->pd, &qp_attr); 1320 if (ret) { 1321 pr_err("failed to create_qp ret= %d\n", ret); 1322 goto err_destroy_cq; 1323 } 1324 queue->qp = queue->cm_id->qp; 1325 1326 atomic_set(&queue->sq_wr_avail, qp_attr.cap.max_send_wr); 1327 1328 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n", 1329 __func__, queue->cq->cqe, qp_attr.cap.max_send_sge, 1330 qp_attr.cap.max_send_wr, queue->cm_id); 1331 1332 if (!queue->nsrq) { 1333 for (i = 0; i < queue->recv_queue_size; i++) { 1334 queue->cmds[i].queue = queue; 1335 ret = nvmet_rdma_post_recv(ndev, &queue->cmds[i]); 1336 if (ret) 1337 goto err_destroy_qp; 1338 } 1339 } 1340 1341 out: 1342 return ret; 1343 1344 err_destroy_qp: 1345 rdma_destroy_qp(queue->cm_id); 1346 err_destroy_cq: 1347 ib_cq_pool_put(queue->cq, nr_cqe + 1); 1348 goto out; 1349 } 1350 1351 static bool nvmet_rdma_reclaim_rsp(struct sbitmap *sb, unsigned int bitnr, 1352 void *data) 1353 { 1354 struct nvmet_rdma_queue *queue = data; 1355 1356 nvmet_rdma_free_rsp_resources(&queue->rsps[bitnr]); 1357 1358 return true; 1359 } 1360 1361 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) 1362 { 1363 ib_drain_qp(queue->qp); 1364 1365 /* 1366 * Reclaim resources of a response that is still in-flight when the 1367 * queue is being torn down. This happens when the connection was 1368 * forcefully disconnected while an I/O is in flight. 1369 */ 1370 sbitmap_for_each_set(&queue->rsp_tags, nvmet_rdma_reclaim_rsp, queue); 1371 1372 if (queue->cm_id) 1373 rdma_destroy_id(queue->cm_id); 1374 ib_destroy_qp(queue->qp); 1375 ib_cq_pool_put(queue->cq, queue->recv_queue_size + 2 * 1376 queue->send_queue_size + 1); 1377 } 1378 1379 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue) 1380 { 1381 pr_debug("freeing queue %d\n", queue->idx); 1382 1383 nvmet_sq_destroy(&queue->nvme_sq); 1384 nvmet_cq_put(&queue->nvme_cq); 1385 1386 nvmet_rdma_destroy_queue_ib(queue); 1387 if (!queue->nsrq) { 1388 nvmet_rdma_free_cmds(queue->dev, queue->cmds, 1389 queue->recv_queue_size, 1390 !queue->host_qid); 1391 } 1392 nvmet_rdma_free_rsps(queue); 1393 ida_free(&nvmet_rdma_queue_ida, queue->idx); 1394 kfree(queue); 1395 } 1396 1397 static void nvmet_rdma_release_queue_work(struct work_struct *w) 1398 { 1399 struct nvmet_rdma_queue *queue = 1400 container_of(w, struct nvmet_rdma_queue, release_work); 1401 struct nvmet_rdma_device *dev = queue->dev; 1402 1403 nvmet_rdma_free_queue(queue); 1404 1405 kref_put(&dev->ref, nvmet_rdma_free_dev); 1406 } 1407 1408 static int 1409 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn, 1410 struct nvmet_rdma_queue *queue) 1411 { 1412 struct nvme_rdma_cm_req *req; 1413 1414 req = (struct nvme_rdma_cm_req *)conn->private_data; 1415 if (!req || conn->private_data_len == 0) 1416 return NVME_RDMA_CM_INVALID_LEN; 1417 1418 if (le16_to_cpu(req->recfmt) != NVME_RDMA_CM_FMT_1_0) 1419 return NVME_RDMA_CM_INVALID_RECFMT; 1420 1421 queue->host_qid = le16_to_cpu(req->qid); 1422 1423 /* 1424 * req->hsqsize corresponds to our recv queue size plus 1 1425 * req->hrqsize corresponds to our send queue size 1426 */ 1427 queue->recv_queue_size = le16_to_cpu(req->hsqsize) + 1; 1428 queue->send_queue_size = le16_to_cpu(req->hrqsize); 1429 1430 if (!queue->host_qid && queue->recv_queue_size > NVME_AQ_DEPTH) 1431 return NVME_RDMA_CM_INVALID_HSQSIZE; 1432 1433 /* XXX: Should we enforce some kind of max for IO queues? */ 1434 1435 return 0; 1436 } 1437 1438 static int nvmet_rdma_cm_reject(struct rdma_cm_id *cm_id, 1439 enum nvme_rdma_cm_status status) 1440 { 1441 struct nvme_rdma_cm_rej rej; 1442 1443 pr_debug("rejecting connect request: status %d (%s)\n", 1444 status, nvme_rdma_cm_msg(status)); 1445 1446 rej.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0); 1447 rej.sts = cpu_to_le16(status); 1448 1449 return rdma_reject(cm_id, (void *)&rej, sizeof(rej), 1450 IB_CM_REJ_CONSUMER_DEFINED); 1451 } 1452 1453 static struct nvmet_rdma_queue * 1454 nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev, 1455 struct rdma_cm_id *cm_id, 1456 struct rdma_cm_event *event) 1457 { 1458 struct nvmet_rdma_port *port = cm_id->context; 1459 struct nvmet_rdma_queue *queue; 1460 int ret; 1461 1462 queue = kzalloc_obj(*queue); 1463 if (!queue) { 1464 ret = NVME_RDMA_CM_NO_RSC; 1465 goto out_reject; 1466 } 1467 1468 nvmet_cq_init(&queue->nvme_cq); 1469 ret = nvmet_sq_init(&queue->nvme_sq, &queue->nvme_cq); 1470 if (ret) { 1471 ret = NVME_RDMA_CM_NO_RSC; 1472 goto out_free_queue; 1473 } 1474 1475 ret = nvmet_rdma_parse_cm_connect_req(&event->param.conn, queue); 1476 if (ret) 1477 goto out_destroy_sq; 1478 1479 /* 1480 * Schedules the actual release because calling rdma_destroy_id from 1481 * inside a CM callback would trigger a deadlock. (great API design..) 1482 */ 1483 INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work); 1484 queue->dev = ndev; 1485 queue->cm_id = cm_id; 1486 queue->port = port->nport; 1487 1488 spin_lock_init(&queue->state_lock); 1489 queue->state = NVMET_RDMA_Q_CONNECTING; 1490 INIT_LIST_HEAD(&queue->rsp_wait_list); 1491 INIT_LIST_HEAD(&queue->rsp_wr_wait_list); 1492 spin_lock_init(&queue->rsp_wr_wait_lock); 1493 INIT_LIST_HEAD(&queue->queue_list); 1494 1495 queue->idx = ida_alloc(&nvmet_rdma_queue_ida, GFP_KERNEL); 1496 if (queue->idx < 0) { 1497 ret = NVME_RDMA_CM_NO_RSC; 1498 goto out_destroy_sq; 1499 } 1500 1501 /* 1502 * Spread the io queues across completion vectors, 1503 * but still keep all admin queues on vector 0. 1504 */ 1505 queue->comp_vector = !queue->host_qid ? 0 : 1506 queue->idx % ndev->device->num_comp_vectors; 1507 1508 1509 ret = nvmet_rdma_alloc_rsps(queue); 1510 if (ret) { 1511 ret = NVME_RDMA_CM_NO_RSC; 1512 goto out_ida_remove; 1513 } 1514 1515 if (ndev->srqs) { 1516 queue->nsrq = ndev->srqs[queue->comp_vector % ndev->srq_count]; 1517 } else { 1518 queue->cmds = nvmet_rdma_alloc_cmds(ndev, 1519 queue->recv_queue_size, 1520 !queue->host_qid); 1521 if (IS_ERR(queue->cmds)) { 1522 ret = NVME_RDMA_CM_NO_RSC; 1523 goto out_free_responses; 1524 } 1525 } 1526 1527 ret = nvmet_rdma_create_queue_ib(queue); 1528 if (ret) { 1529 pr_err("%s: creating RDMA queue failed (%d).\n", 1530 __func__, ret); 1531 ret = NVME_RDMA_CM_NO_RSC; 1532 goto out_free_cmds; 1533 } 1534 1535 return queue; 1536 1537 out_free_cmds: 1538 if (!queue->nsrq) { 1539 nvmet_rdma_free_cmds(queue->dev, queue->cmds, 1540 queue->recv_queue_size, 1541 !queue->host_qid); 1542 } 1543 out_free_responses: 1544 nvmet_rdma_free_rsps(queue); 1545 out_ida_remove: 1546 ida_free(&nvmet_rdma_queue_ida, queue->idx); 1547 out_destroy_sq: 1548 nvmet_sq_destroy(&queue->nvme_sq); 1549 out_free_queue: 1550 nvmet_cq_put(&queue->nvme_cq); 1551 kfree(queue); 1552 out_reject: 1553 nvmet_rdma_cm_reject(cm_id, ret); 1554 return NULL; 1555 } 1556 1557 static void nvmet_rdma_qp_event(struct ib_event *event, void *priv) 1558 { 1559 struct nvmet_rdma_queue *queue = priv; 1560 1561 switch (event->event) { 1562 case IB_EVENT_COMM_EST: 1563 rdma_notify(queue->cm_id, event->event); 1564 break; 1565 case IB_EVENT_QP_LAST_WQE_REACHED: 1566 pr_debug("received last WQE reached event for queue=0x%p\n", 1567 queue); 1568 break; 1569 default: 1570 pr_err("received IB QP event: %s (%d)\n", 1571 ib_event_msg(event->event), event->event); 1572 break; 1573 } 1574 } 1575 1576 static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id, 1577 struct nvmet_rdma_queue *queue, 1578 struct rdma_conn_param *p) 1579 { 1580 struct rdma_conn_param param = { }; 1581 struct nvme_rdma_cm_rep priv = { }; 1582 int ret = -ENOMEM; 1583 1584 param.rnr_retry_count = 7; 1585 param.flow_control = 1; 1586 param.initiator_depth = min3(p->initiator_depth, 1587 queue->dev->device->attrs.max_qp_init_rd_atom, 1588 U8_MAX); 1589 param.private_data = &priv; 1590 param.private_data_len = sizeof(priv); 1591 priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0); 1592 priv.crqsize = cpu_to_le16(queue->recv_queue_size); 1593 1594 ret = rdma_accept(cm_id, ¶m); 1595 if (ret) 1596 pr_err("rdma_accept failed (error code = %d)\n", ret); 1597 1598 return ret; 1599 } 1600 1601 static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, 1602 struct rdma_cm_event *event) 1603 { 1604 struct nvmet_rdma_device *ndev; 1605 struct nvmet_rdma_queue *queue; 1606 int ret = -EINVAL; 1607 1608 ndev = nvmet_rdma_find_get_device(cm_id); 1609 if (!ndev) { 1610 nvmet_rdma_cm_reject(cm_id, NVME_RDMA_CM_NO_RSC); 1611 return -ECONNREFUSED; 1612 } 1613 1614 queue = nvmet_rdma_alloc_queue(ndev, cm_id, event); 1615 if (!queue) { 1616 ret = -ENOMEM; 1617 goto put_device; 1618 } 1619 1620 if (queue->host_qid == 0) { 1621 struct nvmet_rdma_queue *q; 1622 int pending = 0; 1623 1624 /* Check for pending controller teardown */ 1625 mutex_lock(&nvmet_rdma_queue_mutex); 1626 list_for_each_entry(q, &nvmet_rdma_queue_list, queue_list) { 1627 if (q->nvme_sq.ctrl == queue->nvme_sq.ctrl && 1628 q->state == NVMET_RDMA_Q_DISCONNECTING) 1629 pending++; 1630 } 1631 mutex_unlock(&nvmet_rdma_queue_mutex); 1632 if (pending > NVMET_RDMA_BACKLOG) { 1633 ret = NVME_SC_CONNECT_CTRL_BUSY; 1634 goto put_device; 1635 } 1636 } 1637 1638 ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); 1639 if (ret) { 1640 /* 1641 * Don't destroy the cm_id in free path, as we implicitly 1642 * destroy the cm_id here with non-zero ret code. 1643 */ 1644 queue->cm_id = NULL; 1645 goto free_queue; 1646 } 1647 1648 mutex_lock(&nvmet_rdma_queue_mutex); 1649 list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list); 1650 mutex_unlock(&nvmet_rdma_queue_mutex); 1651 1652 return 0; 1653 1654 free_queue: 1655 nvmet_rdma_free_queue(queue); 1656 put_device: 1657 kref_put(&ndev->ref, nvmet_rdma_free_dev); 1658 1659 return ret; 1660 } 1661 1662 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue *queue) 1663 { 1664 unsigned long flags; 1665 1666 spin_lock_irqsave(&queue->state_lock, flags); 1667 if (queue->state != NVMET_RDMA_Q_CONNECTING) { 1668 pr_warn("trying to establish a connected queue\n"); 1669 goto out_unlock; 1670 } 1671 queue->state = NVMET_RDMA_Q_LIVE; 1672 1673 while (!list_empty(&queue->rsp_wait_list)) { 1674 struct nvmet_rdma_rsp *cmd; 1675 1676 cmd = list_first_entry(&queue->rsp_wait_list, 1677 struct nvmet_rdma_rsp, wait_list); 1678 list_del(&cmd->wait_list); 1679 1680 spin_unlock_irqrestore(&queue->state_lock, flags); 1681 nvmet_rdma_handle_command(queue, cmd); 1682 spin_lock_irqsave(&queue->state_lock, flags); 1683 } 1684 1685 out_unlock: 1686 spin_unlock_irqrestore(&queue->state_lock, flags); 1687 } 1688 1689 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) 1690 { 1691 bool disconnect = false; 1692 unsigned long flags; 1693 1694 pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state); 1695 1696 spin_lock_irqsave(&queue->state_lock, flags); 1697 switch (queue->state) { 1698 case NVMET_RDMA_Q_CONNECTING: 1699 while (!list_empty(&queue->rsp_wait_list)) { 1700 struct nvmet_rdma_rsp *rsp; 1701 1702 rsp = list_first_entry(&queue->rsp_wait_list, 1703 struct nvmet_rdma_rsp, 1704 wait_list); 1705 list_del(&rsp->wait_list); 1706 nvmet_rdma_put_rsp(rsp); 1707 } 1708 fallthrough; 1709 case NVMET_RDMA_Q_LIVE: 1710 queue->state = NVMET_RDMA_Q_DISCONNECTING; 1711 disconnect = true; 1712 break; 1713 case NVMET_RDMA_Q_DISCONNECTING: 1714 break; 1715 } 1716 spin_unlock_irqrestore(&queue->state_lock, flags); 1717 1718 if (disconnect) { 1719 rdma_disconnect(queue->cm_id); 1720 queue_work(nvmet_wq, &queue->release_work); 1721 } 1722 } 1723 1724 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) 1725 { 1726 bool disconnect = false; 1727 1728 mutex_lock(&nvmet_rdma_queue_mutex); 1729 if (!list_empty(&queue->queue_list)) { 1730 list_del_init(&queue->queue_list); 1731 disconnect = true; 1732 } 1733 mutex_unlock(&nvmet_rdma_queue_mutex); 1734 1735 if (disconnect) 1736 __nvmet_rdma_queue_disconnect(queue); 1737 } 1738 1739 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, 1740 struct nvmet_rdma_queue *queue) 1741 { 1742 WARN_ON_ONCE(queue->state != NVMET_RDMA_Q_CONNECTING); 1743 1744 mutex_lock(&nvmet_rdma_queue_mutex); 1745 if (!list_empty(&queue->queue_list)) 1746 list_del_init(&queue->queue_list); 1747 mutex_unlock(&nvmet_rdma_queue_mutex); 1748 1749 pr_err("failed to connect queue %d\n", queue->idx); 1750 queue_work(nvmet_wq, &queue->release_work); 1751 } 1752 1753 /** 1754 * nvmet_rdma_device_removal() - Handle RDMA device removal 1755 * @cm_id: rdma_cm id, used for nvmet port 1756 * @queue: nvmet rdma queue (cm id qp_context) 1757 * 1758 * DEVICE_REMOVAL event notifies us that the RDMA device is about 1759 * to unplug. Note that this event can be generated on a normal 1760 * queue cm_id and/or a device bound listener cm_id (where in this 1761 * case queue will be null). 1762 * 1763 * We registered an ib_client to handle device removal for queues, 1764 * so we only need to handle the listening port cm_ids. In this case 1765 * we nullify the priv to prevent double cm_id destruction and destroying 1766 * the cm_id implicitly by returning a non-zero rc to the callout. 1767 */ 1768 static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, 1769 struct nvmet_rdma_queue *queue) 1770 { 1771 struct nvmet_rdma_port *port; 1772 1773 if (queue) { 1774 /* 1775 * This is a queue cm_id. we have registered 1776 * an ib_client to handle queues removal 1777 * so don't interfere and just return. 1778 */ 1779 return 0; 1780 } 1781 1782 port = cm_id->context; 1783 1784 /* 1785 * This is a listener cm_id. Make sure that 1786 * future remove_port won't invoke a double 1787 * cm_id destroy. use atomic xchg to make sure 1788 * we don't compete with remove_port. 1789 */ 1790 if (xchg(&port->cm_id, NULL) != cm_id) 1791 return 0; 1792 1793 /* 1794 * We need to return 1 so that the core will destroy 1795 * its own ID. What a great API design.. 1796 */ 1797 return 1; 1798 } 1799 1800 static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id, 1801 struct rdma_cm_event *event) 1802 { 1803 struct nvmet_rdma_queue *queue = NULL; 1804 int ret = 0; 1805 1806 if (cm_id->qp) 1807 queue = cm_id->qp->qp_context; 1808 1809 pr_debug("%s (%d): status %d id %p\n", 1810 rdma_event_msg(event->event), event->event, 1811 event->status, cm_id); 1812 1813 switch (event->event) { 1814 case RDMA_CM_EVENT_CONNECT_REQUEST: 1815 ret = nvmet_rdma_queue_connect(cm_id, event); 1816 break; 1817 case RDMA_CM_EVENT_ESTABLISHED: 1818 nvmet_rdma_queue_established(queue); 1819 break; 1820 case RDMA_CM_EVENT_ADDR_CHANGE: 1821 if (!queue) { 1822 struct nvmet_rdma_port *port = cm_id->context; 1823 1824 queue_delayed_work(nvmet_wq, &port->repair_work, 0); 1825 break; 1826 } 1827 fallthrough; 1828 case RDMA_CM_EVENT_DISCONNECTED: 1829 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 1830 nvmet_rdma_queue_disconnect(queue); 1831 break; 1832 case RDMA_CM_EVENT_DEVICE_REMOVAL: 1833 ret = nvmet_rdma_device_removal(cm_id, queue); 1834 break; 1835 case RDMA_CM_EVENT_REJECTED: 1836 pr_debug("Connection rejected: %s\n", 1837 rdma_reject_msg(cm_id, event->status)); 1838 fallthrough; 1839 case RDMA_CM_EVENT_UNREACHABLE: 1840 case RDMA_CM_EVENT_CONNECT_ERROR: 1841 nvmet_rdma_queue_connect_fail(cm_id, queue); 1842 break; 1843 default: 1844 pr_err("received unrecognized RDMA CM event %d\n", 1845 event->event); 1846 break; 1847 } 1848 1849 return ret; 1850 } 1851 1852 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl *ctrl) 1853 { 1854 struct nvmet_rdma_queue *queue, *n; 1855 1856 mutex_lock(&nvmet_rdma_queue_mutex); 1857 list_for_each_entry_safe(queue, n, &nvmet_rdma_queue_list, queue_list) { 1858 if (queue->nvme_sq.ctrl != ctrl) 1859 continue; 1860 list_del_init(&queue->queue_list); 1861 __nvmet_rdma_queue_disconnect(queue); 1862 } 1863 mutex_unlock(&nvmet_rdma_queue_mutex); 1864 } 1865 1866 static void nvmet_rdma_destroy_port_queues(struct nvmet_rdma_port *port) 1867 { 1868 struct nvmet_rdma_queue *queue, *tmp; 1869 struct nvmet_port *nport = port->nport; 1870 1871 mutex_lock(&nvmet_rdma_queue_mutex); 1872 list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list, 1873 queue_list) { 1874 if (queue->port != nport) 1875 continue; 1876 1877 list_del_init(&queue->queue_list); 1878 __nvmet_rdma_queue_disconnect(queue); 1879 } 1880 mutex_unlock(&nvmet_rdma_queue_mutex); 1881 } 1882 1883 static void nvmet_rdma_disable_port(struct nvmet_rdma_port *port) 1884 { 1885 struct rdma_cm_id *cm_id = xchg(&port->cm_id, NULL); 1886 1887 if (cm_id) 1888 rdma_destroy_id(cm_id); 1889 1890 /* 1891 * Destroy the remaining queues, which are not belong to any 1892 * controller yet. Do it here after the RDMA-CM was destroyed 1893 * guarantees that no new queue will be created. 1894 */ 1895 nvmet_rdma_destroy_port_queues(port); 1896 } 1897 1898 static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port) 1899 { 1900 struct sockaddr *addr = (struct sockaddr *)&port->addr; 1901 struct rdma_cm_id *cm_id; 1902 int ret; 1903 1904 cm_id = rdma_create_id(&init_net, nvmet_rdma_cm_handler, port, 1905 RDMA_PS_TCP, IB_QPT_RC); 1906 if (IS_ERR(cm_id)) { 1907 pr_err("CM ID creation failed\n"); 1908 return PTR_ERR(cm_id); 1909 } 1910 1911 /* 1912 * Allow both IPv4 and IPv6 sockets to bind a single port 1913 * at the same time. 1914 */ 1915 ret = rdma_set_afonly(cm_id, 1); 1916 if (ret) { 1917 pr_err("rdma_set_afonly failed (%d)\n", ret); 1918 goto out_destroy_id; 1919 } 1920 1921 ret = rdma_bind_addr(cm_id, addr); 1922 if (ret) { 1923 pr_err("binding CM ID to %pISpcs failed (%d)\n", addr, ret); 1924 goto out_destroy_id; 1925 } 1926 1927 ret = rdma_listen(cm_id, NVMET_RDMA_BACKLOG); 1928 if (ret) { 1929 pr_err("listening to %pISpcs failed (%d)\n", addr, ret); 1930 goto out_destroy_id; 1931 } 1932 1933 port->cm_id = cm_id; 1934 return 0; 1935 1936 out_destroy_id: 1937 rdma_destroy_id(cm_id); 1938 return ret; 1939 } 1940 1941 static void nvmet_rdma_repair_port_work(struct work_struct *w) 1942 { 1943 struct nvmet_rdma_port *port = container_of(to_delayed_work(w), 1944 struct nvmet_rdma_port, repair_work); 1945 int ret; 1946 1947 nvmet_rdma_disable_port(port); 1948 ret = nvmet_rdma_enable_port(port); 1949 if (ret) 1950 queue_delayed_work(nvmet_wq, &port->repair_work, 5 * HZ); 1951 } 1952 1953 static int nvmet_rdma_add_port(struct nvmet_port *nport) 1954 { 1955 struct nvmet_rdma_port *port; 1956 __kernel_sa_family_t af; 1957 int ret; 1958 1959 port = kzalloc_obj(*port); 1960 if (!port) 1961 return -ENOMEM; 1962 1963 nport->priv = port; 1964 port->nport = nport; 1965 INIT_DELAYED_WORK(&port->repair_work, nvmet_rdma_repair_port_work); 1966 1967 switch (nport->disc_addr.adrfam) { 1968 case NVMF_ADDR_FAMILY_IP4: 1969 af = AF_INET; 1970 break; 1971 case NVMF_ADDR_FAMILY_IP6: 1972 af = AF_INET6; 1973 break; 1974 default: 1975 pr_err("address family %d not supported\n", 1976 nport->disc_addr.adrfam); 1977 ret = -EINVAL; 1978 goto out_free_port; 1979 } 1980 1981 if (nport->inline_data_size < 0) { 1982 nport->inline_data_size = NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE; 1983 } else if (nport->inline_data_size > NVMET_RDMA_MAX_INLINE_DATA_SIZE) { 1984 pr_warn("inline_data_size %u is too large, reducing to %u\n", 1985 nport->inline_data_size, 1986 NVMET_RDMA_MAX_INLINE_DATA_SIZE); 1987 nport->inline_data_size = NVMET_RDMA_MAX_INLINE_DATA_SIZE; 1988 } 1989 1990 if (nport->max_queue_size < 0) { 1991 nport->max_queue_size = NVME_RDMA_DEFAULT_QUEUE_SIZE; 1992 } else if (nport->max_queue_size > NVME_RDMA_MAX_QUEUE_SIZE) { 1993 pr_warn("max_queue_size %u is too large, reducing to %u\n", 1994 nport->max_queue_size, NVME_RDMA_MAX_QUEUE_SIZE); 1995 nport->max_queue_size = NVME_RDMA_MAX_QUEUE_SIZE; 1996 } 1997 1998 ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr, 1999 nport->disc_addr.trsvcid, &port->addr); 2000 if (ret) { 2001 pr_err("malformed ip/port passed: %s:%s\n", 2002 nport->disc_addr.traddr, nport->disc_addr.trsvcid); 2003 goto out_free_port; 2004 } 2005 2006 ret = nvmet_rdma_enable_port(port); 2007 if (ret) 2008 goto out_free_port; 2009 2010 pr_info("enabling port %d (%pISpcs)\n", 2011 le16_to_cpu(nport->disc_addr.portid), 2012 (struct sockaddr *)&port->addr); 2013 2014 return 0; 2015 2016 out_free_port: 2017 kfree(port); 2018 return ret; 2019 } 2020 2021 static void nvmet_rdma_remove_port(struct nvmet_port *nport) 2022 { 2023 struct nvmet_rdma_port *port = nport->priv; 2024 2025 cancel_delayed_work_sync(&port->repair_work); 2026 nvmet_rdma_disable_port(port); 2027 kfree(port); 2028 } 2029 2030 static void nvmet_rdma_disc_port_addr(struct nvmet_req *req, 2031 struct nvmet_port *nport, char *traddr) 2032 { 2033 struct nvmet_rdma_port *port = nport->priv; 2034 struct rdma_cm_id *cm_id = port->cm_id; 2035 2036 if (inet_addr_is_any(&cm_id->route.addr.src_addr)) { 2037 struct nvmet_rdma_rsp *rsp = 2038 container_of(req, struct nvmet_rdma_rsp, req); 2039 struct rdma_cm_id *req_cm_id = rsp->queue->cm_id; 2040 struct sockaddr *addr = (void *)&req_cm_id->route.addr.src_addr; 2041 2042 sprintf(traddr, "%pISc", addr); 2043 } else { 2044 memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE); 2045 } 2046 } 2047 2048 static ssize_t nvmet_rdma_host_port_addr(struct nvmet_ctrl *ctrl, 2049 char *traddr, size_t traddr_len) 2050 { 2051 struct nvmet_sq *nvme_sq = ctrl->sqs[0]; 2052 struct nvmet_rdma_queue *queue = 2053 container_of(nvme_sq, struct nvmet_rdma_queue, nvme_sq); 2054 2055 return snprintf(traddr, traddr_len, "%pISc", 2056 (struct sockaddr *)&queue->cm_id->route.addr.dst_addr); 2057 } 2058 2059 static u8 nvmet_rdma_get_mdts(const struct nvmet_ctrl *ctrl) 2060 { 2061 if (ctrl->pi_support) 2062 return NVMET_RDMA_MAX_METADATA_MDTS; 2063 return NVMET_RDMA_MAX_MDTS; 2064 } 2065 2066 static u16 nvmet_rdma_get_max_queue_size(const struct nvmet_ctrl *ctrl) 2067 { 2068 if (ctrl->pi_support) 2069 return NVME_RDMA_MAX_METADATA_QUEUE_SIZE; 2070 return NVME_RDMA_MAX_QUEUE_SIZE; 2071 } 2072 2073 static const struct nvmet_fabrics_ops nvmet_rdma_ops = { 2074 .owner = THIS_MODULE, 2075 .type = NVMF_TRTYPE_RDMA, 2076 .msdbd = 1, 2077 .flags = NVMF_KEYED_SGLS | NVMF_METADATA_SUPPORTED, 2078 .add_port = nvmet_rdma_add_port, 2079 .remove_port = nvmet_rdma_remove_port, 2080 .queue_response = nvmet_rdma_queue_response, 2081 .delete_ctrl = nvmet_rdma_delete_ctrl, 2082 .disc_traddr = nvmet_rdma_disc_port_addr, 2083 .host_traddr = nvmet_rdma_host_port_addr, 2084 .get_mdts = nvmet_rdma_get_mdts, 2085 .get_max_queue_size = nvmet_rdma_get_max_queue_size, 2086 }; 2087 2088 static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data) 2089 { 2090 struct nvmet_rdma_queue *queue, *tmp; 2091 struct nvmet_rdma_device *ndev; 2092 bool found = false; 2093 2094 mutex_lock(&device_list_mutex); 2095 list_for_each_entry(ndev, &device_list, entry) { 2096 if (ndev->device == ib_device) { 2097 found = true; 2098 break; 2099 } 2100 } 2101 mutex_unlock(&device_list_mutex); 2102 2103 if (!found) 2104 return; 2105 2106 /* 2107 * IB Device that is used by nvmet controllers is being removed, 2108 * delete all queues using this device. 2109 */ 2110 mutex_lock(&nvmet_rdma_queue_mutex); 2111 list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list, 2112 queue_list) { 2113 if (queue->dev->device != ib_device) 2114 continue; 2115 2116 pr_info("Removing queue %d\n", queue->idx); 2117 list_del_init(&queue->queue_list); 2118 __nvmet_rdma_queue_disconnect(queue); 2119 } 2120 mutex_unlock(&nvmet_rdma_queue_mutex); 2121 2122 flush_workqueue(nvmet_wq); 2123 flush_workqueue(nvmet_aen_wq); 2124 } 2125 2126 static struct ib_client nvmet_rdma_ib_client = { 2127 .name = "nvmet_rdma", 2128 .remove = nvmet_rdma_remove_one 2129 }; 2130 2131 static int __init nvmet_rdma_init(void) 2132 { 2133 int ret; 2134 2135 ret = ib_register_client(&nvmet_rdma_ib_client); 2136 if (ret) 2137 return ret; 2138 2139 ret = nvmet_register_transport(&nvmet_rdma_ops); 2140 if (ret) 2141 goto err_ib_client; 2142 2143 return 0; 2144 2145 err_ib_client: 2146 ib_unregister_client(&nvmet_rdma_ib_client); 2147 return ret; 2148 } 2149 2150 static void __exit nvmet_rdma_exit(void) 2151 { 2152 nvmet_unregister_transport(&nvmet_rdma_ops); 2153 ib_unregister_client(&nvmet_rdma_ib_client); 2154 WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list)); 2155 ida_destroy(&nvmet_rdma_queue_ida); 2156 } 2157 2158 module_init(nvmet_rdma_init); 2159 module_exit(nvmet_rdma_exit); 2160 2161 MODULE_DESCRIPTION("NVMe target RDMA transport driver"); 2162 MODULE_LICENSE("GPL v2"); 2163 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */ 2164