1 /* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/module.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/err.h> 39 #include <linux/string.h> 40 #include <linux/parser.h> 41 #include <linux/random.h> 42 #include <linux/jiffies.h> 43 #include <linux/lockdep.h> 44 #include <linux/inet.h> 45 #include <rdma/ib_cache.h> 46 47 #include <linux/atomic.h> 48 49 #include <scsi/scsi.h> 50 #include <scsi/scsi_device.h> 51 #include <scsi/scsi_dbg.h> 52 #include <scsi/scsi_tcq.h> 53 #include <scsi/srp.h> 54 #include <scsi/scsi_transport_srp.h> 55 56 #include "ib_srp.h" 57 58 #define DRV_NAME "ib_srp" 59 #define PFX DRV_NAME ": " 60 61 MODULE_AUTHOR("Roland Dreier"); 62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); 63 MODULE_LICENSE("Dual BSD/GPL"); 64 65 static unsigned int srp_sg_tablesize; 66 static unsigned int cmd_sg_entries; 67 static unsigned int indirect_sg_entries; 68 static bool allow_ext_sg; 69 static bool register_always = true; 70 static bool never_register; 71 static int topspin_workarounds = 1; 72 73 module_param(srp_sg_tablesize, uint, 0444); 74 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); 75 76 module_param(cmd_sg_entries, uint, 0444); 77 MODULE_PARM_DESC(cmd_sg_entries, 78 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); 79 80 module_param(indirect_sg_entries, uint, 0444); 81 MODULE_PARM_DESC(indirect_sg_entries, 82 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")"); 83 84 module_param(allow_ext_sg, bool, 0444); 85 MODULE_PARM_DESC(allow_ext_sg, 86 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)"); 87 88 module_param(topspin_workarounds, int, 0444); 89 MODULE_PARM_DESC(topspin_workarounds, 90 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0"); 91 92 module_param(register_always, bool, 0444); 93 MODULE_PARM_DESC(register_always, 94 "Use memory registration even for contiguous memory regions"); 95 96 module_param(never_register, bool, 0444); 97 MODULE_PARM_DESC(never_register, "Never register memory"); 98 99 static const struct kernel_param_ops srp_tmo_ops; 100 101 static int srp_reconnect_delay = 10; 102 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 103 S_IRUGO | S_IWUSR); 104 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts"); 105 106 static int srp_fast_io_fail_tmo = 15; 107 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo, 108 S_IRUGO | S_IWUSR); 109 MODULE_PARM_DESC(fast_io_fail_tmo, 110 "Number of seconds between the observation of a transport" 111 " layer error and failing all I/O. \"off\" means that this" 112 " functionality is disabled."); 113 114 static int srp_dev_loss_tmo = 600; 115 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo, 116 S_IRUGO | S_IWUSR); 117 MODULE_PARM_DESC(dev_loss_tmo, 118 "Maximum number of seconds that the SRP transport should" 119 " insulate transport layer errors. After this time has been" 120 " exceeded the SCSI host is removed. Should be" 121 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT) 122 " if fast_io_fail_tmo has not been set. \"off\" means that" 123 " this functionality is disabled."); 124 125 static bool srp_use_imm_data = true; 126 module_param_named(use_imm_data, srp_use_imm_data, bool, 0644); 127 MODULE_PARM_DESC(use_imm_data, 128 "Whether or not to request permission to use immediate data during SRP login."); 129 130 static unsigned int srp_max_imm_data = 8 * 1024; 131 module_param_named(max_imm_data, srp_max_imm_data, uint, 0644); 132 MODULE_PARM_DESC(max_imm_data, "Maximum immediate data size."); 133 134 static unsigned ch_count; 135 module_param(ch_count, uint, 0444); 136 MODULE_PARM_DESC(ch_count, 137 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA."); 138 139 static int srp_add_one(struct ib_device *device); 140 static void srp_remove_one(struct ib_device *device, void *client_data); 141 static void srp_rename_dev(struct ib_device *device, void *client_data); 142 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc); 143 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 144 const char *opname); 145 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 146 const struct ib_cm_event *event); 147 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 148 struct rdma_cm_event *event); 149 150 static struct scsi_transport_template *ib_srp_transport_template; 151 static struct workqueue_struct *srp_remove_wq; 152 153 static struct ib_client srp_client = { 154 .name = "srp", 155 .add = srp_add_one, 156 .remove = srp_remove_one, 157 .rename = srp_rename_dev 158 }; 159 160 static struct ib_sa_client srp_sa_client; 161 162 static int srp_tmo_get(char *buffer, const struct kernel_param *kp) 163 { 164 int tmo = *(int *)kp->arg; 165 166 if (tmo >= 0) 167 return sysfs_emit(buffer, "%d\n", tmo); 168 else 169 return sysfs_emit(buffer, "off\n"); 170 } 171 172 static int srp_tmo_set(const char *val, const struct kernel_param *kp) 173 { 174 int tmo, res; 175 176 res = srp_parse_tmo(&tmo, val); 177 if (res) 178 goto out; 179 180 if (kp->arg == &srp_reconnect_delay) 181 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo, 182 srp_dev_loss_tmo); 183 else if (kp->arg == &srp_fast_io_fail_tmo) 184 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo); 185 else 186 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo, 187 tmo); 188 if (res) 189 goto out; 190 *(int *)kp->arg = tmo; 191 192 out: 193 return res; 194 } 195 196 static const struct kernel_param_ops srp_tmo_ops = { 197 .get = srp_tmo_get, 198 .set = srp_tmo_set, 199 }; 200 201 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) 202 { 203 return (struct srp_target_port *) host->hostdata; 204 } 205 206 static const char *srp_target_info(struct Scsi_Host *host) 207 { 208 return host_to_target(host)->target_name; 209 } 210 211 static int srp_target_is_topspin(struct srp_target_port *target) 212 { 213 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad }; 214 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d }; 215 216 return topspin_workarounds && 217 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) || 218 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui)); 219 } 220 221 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size, 222 gfp_t gfp_mask, 223 enum dma_data_direction direction) 224 { 225 struct srp_iu *iu; 226 227 iu = kmalloc(sizeof *iu, gfp_mask); 228 if (!iu) 229 goto out; 230 231 iu->buf = kzalloc(size, gfp_mask); 232 if (!iu->buf) 233 goto out_free_iu; 234 235 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size, 236 direction); 237 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma)) 238 goto out_free_buf; 239 240 iu->size = size; 241 iu->direction = direction; 242 243 return iu; 244 245 out_free_buf: 246 kfree(iu->buf); 247 out_free_iu: 248 kfree(iu); 249 out: 250 return NULL; 251 } 252 253 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu) 254 { 255 if (!iu) 256 return; 257 258 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size, 259 iu->direction); 260 kfree(iu->buf); 261 kfree(iu); 262 } 263 264 static void srp_qp_event(struct ib_event *event, void *context) 265 { 266 pr_debug("QP event %s (%d)\n", 267 ib_event_msg(event->event), event->event); 268 } 269 270 static int srp_init_ib_qp(struct srp_target_port *target, 271 struct ib_qp *qp) 272 { 273 struct ib_qp_attr *attr; 274 int ret; 275 276 attr = kmalloc(sizeof *attr, GFP_KERNEL); 277 if (!attr) 278 return -ENOMEM; 279 280 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 281 target->srp_host->port, 282 be16_to_cpu(target->ib_cm.pkey), 283 &attr->pkey_index); 284 if (ret) 285 goto out; 286 287 attr->qp_state = IB_QPS_INIT; 288 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ | 289 IB_ACCESS_REMOTE_WRITE); 290 attr->port_num = target->srp_host->port; 291 292 ret = ib_modify_qp(qp, attr, 293 IB_QP_STATE | 294 IB_QP_PKEY_INDEX | 295 IB_QP_ACCESS_FLAGS | 296 IB_QP_PORT); 297 298 out: 299 kfree(attr); 300 return ret; 301 } 302 303 static int srp_new_ib_cm_id(struct srp_rdma_ch *ch) 304 { 305 struct srp_target_port *target = ch->target; 306 struct ib_cm_id *new_cm_id; 307 308 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, 309 srp_ib_cm_handler, ch); 310 if (IS_ERR(new_cm_id)) 311 return PTR_ERR(new_cm_id); 312 313 if (ch->ib_cm.cm_id) 314 ib_destroy_cm_id(ch->ib_cm.cm_id); 315 ch->ib_cm.cm_id = new_cm_id; 316 if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev, 317 target->srp_host->port)) 318 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA; 319 else 320 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB; 321 ch->ib_cm.path.sgid = target->sgid; 322 ch->ib_cm.path.dgid = target->ib_cm.orig_dgid; 323 ch->ib_cm.path.pkey = target->ib_cm.pkey; 324 ch->ib_cm.path.service_id = target->ib_cm.service_id; 325 326 return 0; 327 } 328 329 static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch) 330 { 331 struct srp_target_port *target = ch->target; 332 struct rdma_cm_id *new_cm_id; 333 int ret; 334 335 new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch, 336 RDMA_PS_TCP, IB_QPT_RC); 337 if (IS_ERR(new_cm_id)) { 338 ret = PTR_ERR(new_cm_id); 339 new_cm_id = NULL; 340 goto out; 341 } 342 343 init_completion(&ch->done); 344 ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ? 345 &target->rdma_cm.src.sa : NULL, 346 &target->rdma_cm.dst.sa, 347 SRP_PATH_REC_TIMEOUT_MS); 348 if (ret) { 349 pr_err("No route available from %pISpsc to %pISpsc (%d)\n", 350 &target->rdma_cm.src, &target->rdma_cm.dst, ret); 351 goto out; 352 } 353 ret = wait_for_completion_interruptible(&ch->done); 354 if (ret < 0) 355 goto out; 356 357 ret = ch->status; 358 if (ret) { 359 pr_err("Resolving address %pISpsc failed (%d)\n", 360 &target->rdma_cm.dst, ret); 361 goto out; 362 } 363 364 swap(ch->rdma_cm.cm_id, new_cm_id); 365 366 out: 367 if (new_cm_id) 368 rdma_destroy_id(new_cm_id); 369 370 return ret; 371 } 372 373 static int srp_new_cm_id(struct srp_rdma_ch *ch) 374 { 375 struct srp_target_port *target = ch->target; 376 377 return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) : 378 srp_new_ib_cm_id(ch); 379 } 380 381 /** 382 * srp_destroy_fr_pool() - free the resources owned by a pool 383 * @pool: Fast registration pool to be destroyed. 384 */ 385 static void srp_destroy_fr_pool(struct srp_fr_pool *pool) 386 { 387 int i; 388 struct srp_fr_desc *d; 389 390 if (!pool) 391 return; 392 393 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 394 if (d->mr) 395 ib_dereg_mr(d->mr); 396 } 397 kfree(pool); 398 } 399 400 /** 401 * srp_create_fr_pool() - allocate and initialize a pool for fast registration 402 * @device: IB device to allocate fast registration descriptors for. 403 * @pd: Protection domain associated with the FR descriptors. 404 * @pool_size: Number of descriptors to allocate. 405 * @max_page_list_len: Maximum fast registration work request page list length. 406 */ 407 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, 408 struct ib_pd *pd, int pool_size, 409 int max_page_list_len) 410 { 411 struct srp_fr_pool *pool; 412 struct srp_fr_desc *d; 413 struct ib_mr *mr; 414 int i, ret = -EINVAL; 415 enum ib_mr_type mr_type; 416 417 if (pool_size <= 0) 418 goto err; 419 ret = -ENOMEM; 420 pool = kzalloc(struct_size(pool, desc, pool_size), GFP_KERNEL); 421 if (!pool) 422 goto err; 423 pool->size = pool_size; 424 pool->max_page_list_len = max_page_list_len; 425 spin_lock_init(&pool->lock); 426 INIT_LIST_HEAD(&pool->free_list); 427 428 if (device->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) 429 mr_type = IB_MR_TYPE_SG_GAPS; 430 else 431 mr_type = IB_MR_TYPE_MEM_REG; 432 433 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 434 mr = ib_alloc_mr(pd, mr_type, max_page_list_len); 435 if (IS_ERR(mr)) { 436 ret = PTR_ERR(mr); 437 if (ret == -ENOMEM) 438 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n", 439 dev_name(&device->dev)); 440 goto destroy_pool; 441 } 442 d->mr = mr; 443 list_add_tail(&d->entry, &pool->free_list); 444 } 445 446 out: 447 return pool; 448 449 destroy_pool: 450 srp_destroy_fr_pool(pool); 451 452 err: 453 pool = ERR_PTR(ret); 454 goto out; 455 } 456 457 /** 458 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration 459 * @pool: Pool to obtain descriptor from. 460 */ 461 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool) 462 { 463 struct srp_fr_desc *d = NULL; 464 unsigned long flags; 465 466 spin_lock_irqsave(&pool->lock, flags); 467 if (!list_empty(&pool->free_list)) { 468 d = list_first_entry(&pool->free_list, typeof(*d), entry); 469 list_del(&d->entry); 470 } 471 spin_unlock_irqrestore(&pool->lock, flags); 472 473 return d; 474 } 475 476 /** 477 * srp_fr_pool_put() - put an FR descriptor back in the free list 478 * @pool: Pool the descriptor was allocated from. 479 * @desc: Pointer to an array of fast registration descriptor pointers. 480 * @n: Number of descriptors to put back. 481 * 482 * Note: The caller must already have queued an invalidation request for 483 * desc->mr->rkey before calling this function. 484 */ 485 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc, 486 int n) 487 { 488 unsigned long flags; 489 int i; 490 491 spin_lock_irqsave(&pool->lock, flags); 492 for (i = 0; i < n; i++) 493 list_add(&desc[i]->entry, &pool->free_list); 494 spin_unlock_irqrestore(&pool->lock, flags); 495 } 496 497 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) 498 { 499 struct srp_device *dev = target->srp_host->srp_dev; 500 501 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size, 502 dev->max_pages_per_mr); 503 } 504 505 /** 506 * srp_destroy_qp() - destroy an RDMA queue pair 507 * @ch: SRP RDMA channel. 508 * 509 * Drain the qp before destroying it. This avoids that the receive 510 * completion handler can access the queue pair while it is 511 * being destroyed. 512 */ 513 static void srp_destroy_qp(struct srp_rdma_ch *ch) 514 { 515 spin_lock_irq(&ch->lock); 516 ib_process_cq_direct(ch->send_cq, -1); 517 spin_unlock_irq(&ch->lock); 518 519 ib_drain_qp(ch->qp); 520 ib_destroy_qp(ch->qp); 521 } 522 523 static int srp_create_ch_ib(struct srp_rdma_ch *ch) 524 { 525 struct srp_target_port *target = ch->target; 526 struct srp_device *dev = target->srp_host->srp_dev; 527 const struct ib_device_attr *attr = &dev->dev->attrs; 528 struct ib_qp_init_attr *init_attr; 529 struct ib_cq *recv_cq, *send_cq; 530 struct ib_qp *qp; 531 struct srp_fr_pool *fr_pool = NULL; 532 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2; 533 int ret; 534 535 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL); 536 if (!init_attr) 537 return -ENOMEM; 538 539 /* queue_size + 1 for ib_drain_rq() */ 540 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1, 541 ch->comp_vector, IB_POLL_SOFTIRQ); 542 if (IS_ERR(recv_cq)) { 543 ret = PTR_ERR(recv_cq); 544 goto err; 545 } 546 547 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size, 548 ch->comp_vector, IB_POLL_DIRECT); 549 if (IS_ERR(send_cq)) { 550 ret = PTR_ERR(send_cq); 551 goto err_recv_cq; 552 } 553 554 init_attr->event_handler = srp_qp_event; 555 init_attr->cap.max_send_wr = m * target->queue_size; 556 init_attr->cap.max_recv_wr = target->queue_size + 1; 557 init_attr->cap.max_recv_sge = 1; 558 init_attr->cap.max_send_sge = min(SRP_MAX_SGE, attr->max_send_sge); 559 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 560 init_attr->qp_type = IB_QPT_RC; 561 init_attr->send_cq = send_cq; 562 init_attr->recv_cq = recv_cq; 563 564 ch->max_imm_sge = min(init_attr->cap.max_send_sge - 1U, 255U); 565 566 if (target->using_rdma_cm) { 567 ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr); 568 qp = ch->rdma_cm.cm_id->qp; 569 } else { 570 qp = ib_create_qp(dev->pd, init_attr); 571 if (!IS_ERR(qp)) { 572 ret = srp_init_ib_qp(target, qp); 573 if (ret) 574 ib_destroy_qp(qp); 575 } else { 576 ret = PTR_ERR(qp); 577 } 578 } 579 if (ret) { 580 pr_err("QP creation failed for dev %s: %d\n", 581 dev_name(&dev->dev->dev), ret); 582 goto err_send_cq; 583 } 584 585 if (dev->use_fast_reg) { 586 fr_pool = srp_alloc_fr_pool(target); 587 if (IS_ERR(fr_pool)) { 588 ret = PTR_ERR(fr_pool); 589 shost_printk(KERN_WARNING, target->scsi_host, PFX 590 "FR pool allocation failed (%d)\n", ret); 591 goto err_qp; 592 } 593 } 594 595 if (ch->qp) 596 srp_destroy_qp(ch); 597 if (ch->recv_cq) 598 ib_free_cq(ch->recv_cq); 599 if (ch->send_cq) 600 ib_free_cq(ch->send_cq); 601 602 ch->qp = qp; 603 ch->recv_cq = recv_cq; 604 ch->send_cq = send_cq; 605 606 if (dev->use_fast_reg) { 607 if (ch->fr_pool) 608 srp_destroy_fr_pool(ch->fr_pool); 609 ch->fr_pool = fr_pool; 610 } 611 612 kfree(init_attr); 613 return 0; 614 615 err_qp: 616 if (target->using_rdma_cm) 617 rdma_destroy_qp(ch->rdma_cm.cm_id); 618 else 619 ib_destroy_qp(qp); 620 621 err_send_cq: 622 ib_free_cq(send_cq); 623 624 err_recv_cq: 625 ib_free_cq(recv_cq); 626 627 err: 628 kfree(init_attr); 629 return ret; 630 } 631 632 /* 633 * Note: this function may be called without srp_alloc_iu_bufs() having been 634 * invoked. Hence the ch->[rt]x_ring checks. 635 */ 636 static void srp_free_ch_ib(struct srp_target_port *target, 637 struct srp_rdma_ch *ch) 638 { 639 struct srp_device *dev = target->srp_host->srp_dev; 640 int i; 641 642 if (!ch->target) 643 return; 644 645 if (target->using_rdma_cm) { 646 if (ch->rdma_cm.cm_id) { 647 rdma_destroy_id(ch->rdma_cm.cm_id); 648 ch->rdma_cm.cm_id = NULL; 649 } 650 } else { 651 if (ch->ib_cm.cm_id) { 652 ib_destroy_cm_id(ch->ib_cm.cm_id); 653 ch->ib_cm.cm_id = NULL; 654 } 655 } 656 657 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */ 658 if (!ch->qp) 659 return; 660 661 if (dev->use_fast_reg) { 662 if (ch->fr_pool) 663 srp_destroy_fr_pool(ch->fr_pool); 664 } 665 666 srp_destroy_qp(ch); 667 ib_free_cq(ch->send_cq); 668 ib_free_cq(ch->recv_cq); 669 670 /* 671 * Avoid that the SCSI error handler tries to use this channel after 672 * it has been freed. The SCSI error handler can namely continue 673 * trying to perform recovery actions after scsi_remove_host() 674 * returned. 675 */ 676 ch->target = NULL; 677 678 ch->qp = NULL; 679 ch->send_cq = ch->recv_cq = NULL; 680 681 if (ch->rx_ring) { 682 for (i = 0; i < target->queue_size; ++i) 683 srp_free_iu(target->srp_host, ch->rx_ring[i]); 684 kfree(ch->rx_ring); 685 ch->rx_ring = NULL; 686 } 687 if (ch->tx_ring) { 688 for (i = 0; i < target->queue_size; ++i) 689 srp_free_iu(target->srp_host, ch->tx_ring[i]); 690 kfree(ch->tx_ring); 691 ch->tx_ring = NULL; 692 } 693 } 694 695 static void srp_path_rec_completion(int status, 696 struct sa_path_rec *pathrec, 697 unsigned int num_paths, void *ch_ptr) 698 { 699 struct srp_rdma_ch *ch = ch_ptr; 700 struct srp_target_port *target = ch->target; 701 702 ch->status = status; 703 if (status) 704 shost_printk(KERN_ERR, target->scsi_host, 705 PFX "Got failed path rec status %d\n", status); 706 else 707 ch->ib_cm.path = *pathrec; 708 complete(&ch->done); 709 } 710 711 static int srp_ib_lookup_path(struct srp_rdma_ch *ch) 712 { 713 struct srp_target_port *target = ch->target; 714 int ret; 715 716 ch->ib_cm.path.numb_path = 1; 717 718 init_completion(&ch->done); 719 720 ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client, 721 target->srp_host->srp_dev->dev, 722 target->srp_host->port, 723 &ch->ib_cm.path, 724 IB_SA_PATH_REC_SERVICE_ID | 725 IB_SA_PATH_REC_DGID | 726 IB_SA_PATH_REC_SGID | 727 IB_SA_PATH_REC_NUMB_PATH | 728 IB_SA_PATH_REC_PKEY, 729 SRP_PATH_REC_TIMEOUT_MS, 730 GFP_KERNEL, 731 srp_path_rec_completion, 732 ch, &ch->ib_cm.path_query); 733 if (ch->ib_cm.path_query_id < 0) 734 return ch->ib_cm.path_query_id; 735 736 ret = wait_for_completion_interruptible(&ch->done); 737 if (ret < 0) 738 return ret; 739 740 if (ch->status < 0) 741 shost_printk(KERN_WARNING, target->scsi_host, 742 PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n", 743 ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw, 744 be16_to_cpu(target->ib_cm.pkey), 745 be64_to_cpu(target->ib_cm.service_id)); 746 747 return ch->status; 748 } 749 750 static int srp_rdma_lookup_path(struct srp_rdma_ch *ch) 751 { 752 struct srp_target_port *target = ch->target; 753 int ret; 754 755 init_completion(&ch->done); 756 757 ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS); 758 if (ret) 759 return ret; 760 761 wait_for_completion_interruptible(&ch->done); 762 763 if (ch->status != 0) 764 shost_printk(KERN_WARNING, target->scsi_host, 765 PFX "Path resolution failed\n"); 766 767 return ch->status; 768 } 769 770 static int srp_lookup_path(struct srp_rdma_ch *ch) 771 { 772 struct srp_target_port *target = ch->target; 773 774 return target->using_rdma_cm ? srp_rdma_lookup_path(ch) : 775 srp_ib_lookup_path(ch); 776 } 777 778 static u8 srp_get_subnet_timeout(struct srp_host *host) 779 { 780 struct ib_port_attr attr; 781 int ret; 782 u8 subnet_timeout = 18; 783 784 ret = ib_query_port(host->srp_dev->dev, host->port, &attr); 785 if (ret == 0) 786 subnet_timeout = attr.subnet_timeout; 787 788 if (unlikely(subnet_timeout < 15)) 789 pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n", 790 dev_name(&host->srp_dev->dev->dev), subnet_timeout); 791 792 return subnet_timeout; 793 } 794 795 static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len, 796 bool multich) 797 { 798 struct srp_target_port *target = ch->target; 799 struct { 800 struct rdma_conn_param rdma_param; 801 struct srp_login_req_rdma rdma_req; 802 struct ib_cm_req_param ib_param; 803 struct srp_login_req ib_req; 804 } *req = NULL; 805 char *ipi, *tpi; 806 int status; 807 808 req = kzalloc(sizeof *req, GFP_KERNEL); 809 if (!req) 810 return -ENOMEM; 811 812 req->ib_param.flow_control = 1; 813 req->ib_param.retry_count = target->tl_retry_count; 814 815 /* 816 * Pick some arbitrary defaults here; we could make these 817 * module parameters if anyone cared about setting them. 818 */ 819 req->ib_param.responder_resources = 4; 820 req->ib_param.rnr_retry_count = 7; 821 req->ib_param.max_cm_retries = 15; 822 823 req->ib_req.opcode = SRP_LOGIN_REQ; 824 req->ib_req.tag = 0; 825 req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len); 826 req->ib_req.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | 827 SRP_BUF_FORMAT_INDIRECT); 828 req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI : 829 SRP_MULTICHAN_SINGLE); 830 if (srp_use_imm_data) { 831 req->ib_req.req_flags |= SRP_IMMED_REQUESTED; 832 req->ib_req.imm_data_offset = cpu_to_be16(SRP_IMM_DATA_OFFSET); 833 } 834 835 if (target->using_rdma_cm) { 836 req->rdma_param.flow_control = req->ib_param.flow_control; 837 req->rdma_param.responder_resources = 838 req->ib_param.responder_resources; 839 req->rdma_param.initiator_depth = req->ib_param.initiator_depth; 840 req->rdma_param.retry_count = req->ib_param.retry_count; 841 req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count; 842 req->rdma_param.private_data = &req->rdma_req; 843 req->rdma_param.private_data_len = sizeof(req->rdma_req); 844 845 req->rdma_req.opcode = req->ib_req.opcode; 846 req->rdma_req.tag = req->ib_req.tag; 847 req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len; 848 req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt; 849 req->rdma_req.req_flags = req->ib_req.req_flags; 850 req->rdma_req.imm_data_offset = req->ib_req.imm_data_offset; 851 852 ipi = req->rdma_req.initiator_port_id; 853 tpi = req->rdma_req.target_port_id; 854 } else { 855 u8 subnet_timeout; 856 857 subnet_timeout = srp_get_subnet_timeout(target->srp_host); 858 859 req->ib_param.primary_path = &ch->ib_cm.path; 860 req->ib_param.alternate_path = NULL; 861 req->ib_param.service_id = target->ib_cm.service_id; 862 get_random_bytes(&req->ib_param.starting_psn, 4); 863 req->ib_param.starting_psn &= 0xffffff; 864 req->ib_param.qp_num = ch->qp->qp_num; 865 req->ib_param.qp_type = ch->qp->qp_type; 866 req->ib_param.local_cm_response_timeout = subnet_timeout + 2; 867 req->ib_param.remote_cm_response_timeout = subnet_timeout + 2; 868 req->ib_param.private_data = &req->ib_req; 869 req->ib_param.private_data_len = sizeof(req->ib_req); 870 871 ipi = req->ib_req.initiator_port_id; 872 tpi = req->ib_req.target_port_id; 873 } 874 875 /* 876 * In the published SRP specification (draft rev. 16a), the 877 * port identifier format is 8 bytes of ID extension followed 878 * by 8 bytes of GUID. Older drafts put the two halves in the 879 * opposite order, so that the GUID comes first. 880 * 881 * Targets conforming to these obsolete drafts can be 882 * recognized by the I/O Class they report. 883 */ 884 if (target->io_class == SRP_REV10_IB_IO_CLASS) { 885 memcpy(ipi, &target->sgid.global.interface_id, 8); 886 memcpy(ipi + 8, &target->initiator_ext, 8); 887 memcpy(tpi, &target->ioc_guid, 8); 888 memcpy(tpi + 8, &target->id_ext, 8); 889 } else { 890 memcpy(ipi, &target->initiator_ext, 8); 891 memcpy(ipi + 8, &target->sgid.global.interface_id, 8); 892 memcpy(tpi, &target->id_ext, 8); 893 memcpy(tpi + 8, &target->ioc_guid, 8); 894 } 895 896 /* 897 * Topspin/Cisco SRP targets will reject our login unless we 898 * zero out the first 8 bytes of our initiator port ID and set 899 * the second 8 bytes to the local node GUID. 900 */ 901 if (srp_target_is_topspin(target)) { 902 shost_printk(KERN_DEBUG, target->scsi_host, 903 PFX "Topspin/Cisco initiator port ID workaround " 904 "activated for target GUID %016llx\n", 905 be64_to_cpu(target->ioc_guid)); 906 memset(ipi, 0, 8); 907 memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8); 908 } 909 910 if (target->using_rdma_cm) 911 status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param); 912 else 913 status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param); 914 915 kfree(req); 916 917 return status; 918 } 919 920 static bool srp_queue_remove_work(struct srp_target_port *target) 921 { 922 bool changed = false; 923 924 spin_lock_irq(&target->lock); 925 if (target->state != SRP_TARGET_REMOVED) { 926 target->state = SRP_TARGET_REMOVED; 927 changed = true; 928 } 929 spin_unlock_irq(&target->lock); 930 931 if (changed) 932 queue_work(srp_remove_wq, &target->remove_work); 933 934 return changed; 935 } 936 937 static void srp_disconnect_target(struct srp_target_port *target) 938 { 939 struct srp_rdma_ch *ch; 940 int i, ret; 941 942 /* XXX should send SRP_I_LOGOUT request */ 943 944 for (i = 0; i < target->ch_count; i++) { 945 ch = &target->ch[i]; 946 ch->connected = false; 947 ret = 0; 948 if (target->using_rdma_cm) { 949 if (ch->rdma_cm.cm_id) 950 rdma_disconnect(ch->rdma_cm.cm_id); 951 } else { 952 if (ch->ib_cm.cm_id) 953 ret = ib_send_cm_dreq(ch->ib_cm.cm_id, 954 NULL, 0); 955 } 956 if (ret < 0) { 957 shost_printk(KERN_DEBUG, target->scsi_host, 958 PFX "Sending CM DREQ failed\n"); 959 } 960 } 961 } 962 963 static int srp_exit_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 964 { 965 struct srp_target_port *target = host_to_target(shost); 966 struct srp_device *dev = target->srp_host->srp_dev; 967 struct ib_device *ibdev = dev->dev; 968 struct srp_request *req = scsi_cmd_priv(cmd); 969 970 kfree(req->fr_list); 971 if (req->indirect_dma_addr) { 972 ib_dma_unmap_single(ibdev, req->indirect_dma_addr, 973 target->indirect_size, 974 DMA_TO_DEVICE); 975 } 976 kfree(req->indirect_desc); 977 978 return 0; 979 } 980 981 static int srp_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 982 { 983 struct srp_target_port *target = host_to_target(shost); 984 struct srp_device *srp_dev = target->srp_host->srp_dev; 985 struct ib_device *ibdev = srp_dev->dev; 986 struct srp_request *req = scsi_cmd_priv(cmd); 987 dma_addr_t dma_addr; 988 int ret = -ENOMEM; 989 990 if (srp_dev->use_fast_reg) { 991 req->fr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *), 992 GFP_KERNEL); 993 if (!req->fr_list) 994 goto out; 995 } 996 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL); 997 if (!req->indirect_desc) 998 goto out; 999 1000 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc, 1001 target->indirect_size, 1002 DMA_TO_DEVICE); 1003 if (ib_dma_mapping_error(ibdev, dma_addr)) { 1004 srp_exit_cmd_priv(shost, cmd); 1005 goto out; 1006 } 1007 1008 req->indirect_dma_addr = dma_addr; 1009 ret = 0; 1010 1011 out: 1012 return ret; 1013 } 1014 1015 /** 1016 * srp_del_scsi_host_attr() - Remove attributes defined in the host template. 1017 * @shost: SCSI host whose attributes to remove from sysfs. 1018 * 1019 * Note: Any attributes defined in the host template and that did not exist 1020 * before invocation of this function will be ignored. 1021 */ 1022 static void srp_del_scsi_host_attr(struct Scsi_Host *shost) 1023 { 1024 const struct attribute_group **g; 1025 struct attribute **attr; 1026 1027 for (g = shost->hostt->shost_groups; *g; ++g) { 1028 for (attr = (*g)->attrs; *attr; ++attr) { 1029 struct device_attribute *dev_attr = 1030 container_of(*attr, typeof(*dev_attr), attr); 1031 1032 device_remove_file(&shost->shost_dev, dev_attr); 1033 } 1034 } 1035 } 1036 1037 static void srp_remove_target(struct srp_target_port *target) 1038 { 1039 struct srp_rdma_ch *ch; 1040 int i; 1041 1042 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1043 1044 srp_del_scsi_host_attr(target->scsi_host); 1045 srp_rport_get(target->rport); 1046 srp_remove_host(target->scsi_host); 1047 scsi_remove_host(target->scsi_host); 1048 srp_stop_rport_timers(target->rport); 1049 srp_disconnect_target(target); 1050 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 1051 for (i = 0; i < target->ch_count; i++) { 1052 ch = &target->ch[i]; 1053 srp_free_ch_ib(target, ch); 1054 } 1055 cancel_work_sync(&target->tl_err_work); 1056 srp_rport_put(target->rport); 1057 kfree(target->ch); 1058 target->ch = NULL; 1059 1060 spin_lock(&target->srp_host->target_lock); 1061 list_del(&target->list); 1062 spin_unlock(&target->srp_host->target_lock); 1063 1064 scsi_host_put(target->scsi_host); 1065 } 1066 1067 static void srp_remove_work(struct work_struct *work) 1068 { 1069 struct srp_target_port *target = 1070 container_of(work, struct srp_target_port, remove_work); 1071 1072 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1073 1074 srp_remove_target(target); 1075 } 1076 1077 static void srp_rport_delete(struct srp_rport *rport) 1078 { 1079 struct srp_target_port *target = rport->lld_data; 1080 1081 srp_queue_remove_work(target); 1082 } 1083 1084 /** 1085 * srp_connected_ch() - number of connected channels 1086 * @target: SRP target port. 1087 */ 1088 static int srp_connected_ch(struct srp_target_port *target) 1089 { 1090 int i, c = 0; 1091 1092 for (i = 0; i < target->ch_count; i++) 1093 c += target->ch[i].connected; 1094 1095 return c; 1096 } 1097 1098 static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len, 1099 bool multich) 1100 { 1101 struct srp_target_port *target = ch->target; 1102 int ret; 1103 1104 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0); 1105 1106 ret = srp_lookup_path(ch); 1107 if (ret) 1108 goto out; 1109 1110 while (1) { 1111 init_completion(&ch->done); 1112 ret = srp_send_req(ch, max_iu_len, multich); 1113 if (ret) 1114 goto out; 1115 ret = wait_for_completion_interruptible(&ch->done); 1116 if (ret < 0) 1117 goto out; 1118 1119 /* 1120 * The CM event handling code will set status to 1121 * SRP_PORT_REDIRECT if we get a port redirect REJ 1122 * back, or SRP_DLID_REDIRECT if we get a lid/qp 1123 * redirect REJ back. 1124 */ 1125 ret = ch->status; 1126 switch (ret) { 1127 case 0: 1128 ch->connected = true; 1129 goto out; 1130 1131 case SRP_PORT_REDIRECT: 1132 ret = srp_lookup_path(ch); 1133 if (ret) 1134 goto out; 1135 break; 1136 1137 case SRP_DLID_REDIRECT: 1138 break; 1139 1140 case SRP_STALE_CONN: 1141 shost_printk(KERN_ERR, target->scsi_host, PFX 1142 "giving up on stale connection\n"); 1143 ret = -ECONNRESET; 1144 goto out; 1145 1146 default: 1147 goto out; 1148 } 1149 } 1150 1151 out: 1152 return ret <= 0 ? ret : -ENODEV; 1153 } 1154 1155 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc) 1156 { 1157 srp_handle_qp_err(cq, wc, "INV RKEY"); 1158 } 1159 1160 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch, 1161 u32 rkey) 1162 { 1163 struct ib_send_wr wr = { 1164 .opcode = IB_WR_LOCAL_INV, 1165 .next = NULL, 1166 .num_sge = 0, 1167 .send_flags = 0, 1168 .ex.invalidate_rkey = rkey, 1169 }; 1170 1171 wr.wr_cqe = &req->reg_cqe; 1172 req->reg_cqe.done = srp_inv_rkey_err_done; 1173 return ib_post_send(ch->qp, &wr, NULL); 1174 } 1175 1176 static void srp_unmap_data(struct scsi_cmnd *scmnd, 1177 struct srp_rdma_ch *ch, 1178 struct srp_request *req) 1179 { 1180 struct srp_target_port *target = ch->target; 1181 struct srp_device *dev = target->srp_host->srp_dev; 1182 struct ib_device *ibdev = dev->dev; 1183 int i, res; 1184 1185 if (!scsi_sglist(scmnd) || 1186 (scmnd->sc_data_direction != DMA_TO_DEVICE && 1187 scmnd->sc_data_direction != DMA_FROM_DEVICE)) 1188 return; 1189 1190 if (dev->use_fast_reg) { 1191 struct srp_fr_desc **pfr; 1192 1193 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { 1194 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey); 1195 if (res < 0) { 1196 shost_printk(KERN_ERR, target->scsi_host, PFX 1197 "Queueing INV WR for rkey %#x failed (%d)\n", 1198 (*pfr)->mr->rkey, res); 1199 queue_work(system_long_wq, 1200 &target->tl_err_work); 1201 } 1202 } 1203 if (req->nmdesc) 1204 srp_fr_pool_put(ch->fr_pool, req->fr_list, 1205 req->nmdesc); 1206 } 1207 1208 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd), 1209 scmnd->sc_data_direction); 1210 } 1211 1212 /** 1213 * srp_claim_req - Take ownership of the scmnd associated with a request. 1214 * @ch: SRP RDMA channel. 1215 * @req: SRP request. 1216 * @sdev: If not NULL, only take ownership for this SCSI device. 1217 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take 1218 * ownership of @req->scmnd if it equals @scmnd. 1219 * 1220 * Return value: 1221 * Either NULL or a pointer to the SCSI command the caller became owner of. 1222 */ 1223 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch, 1224 struct srp_request *req, 1225 struct scsi_device *sdev, 1226 struct scsi_cmnd *scmnd) 1227 { 1228 unsigned long flags; 1229 1230 spin_lock_irqsave(&ch->lock, flags); 1231 if (req->scmnd && 1232 (!sdev || req->scmnd->device == sdev) && 1233 (!scmnd || req->scmnd == scmnd)) { 1234 scmnd = req->scmnd; 1235 req->scmnd = NULL; 1236 } else { 1237 scmnd = NULL; 1238 } 1239 spin_unlock_irqrestore(&ch->lock, flags); 1240 1241 return scmnd; 1242 } 1243 1244 /** 1245 * srp_free_req() - Unmap data and adjust ch->req_lim. 1246 * @ch: SRP RDMA channel. 1247 * @req: Request to be freed. 1248 * @scmnd: SCSI command associated with @req. 1249 * @req_lim_delta: Amount to be added to @target->req_lim. 1250 */ 1251 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req, 1252 struct scsi_cmnd *scmnd, s32 req_lim_delta) 1253 { 1254 unsigned long flags; 1255 1256 srp_unmap_data(scmnd, ch, req); 1257 1258 spin_lock_irqsave(&ch->lock, flags); 1259 ch->req_lim += req_lim_delta; 1260 spin_unlock_irqrestore(&ch->lock, flags); 1261 } 1262 1263 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req, 1264 struct scsi_device *sdev, int result) 1265 { 1266 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL); 1267 1268 if (scmnd) { 1269 srp_free_req(ch, req, scmnd, 0); 1270 scmnd->result = result; 1271 scsi_done(scmnd); 1272 } 1273 } 1274 1275 struct srp_terminate_context { 1276 struct srp_target_port *srp_target; 1277 int scsi_result; 1278 }; 1279 1280 static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr) 1281 { 1282 struct srp_terminate_context *context = context_ptr; 1283 struct srp_target_port *target = context->srp_target; 1284 u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); 1285 struct srp_rdma_ch *ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 1286 struct srp_request *req = scsi_cmd_priv(scmnd); 1287 1288 srp_finish_req(ch, req, NULL, context->scsi_result); 1289 1290 return true; 1291 } 1292 1293 static void srp_terminate_io(struct srp_rport *rport) 1294 { 1295 struct srp_target_port *target = rport->lld_data; 1296 struct srp_terminate_context context = { .srp_target = target, 1297 .scsi_result = DID_TRANSPORT_FAILFAST << 16 }; 1298 1299 scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, &context); 1300 } 1301 1302 /* Calculate maximum initiator to target information unit length. */ 1303 static uint32_t srp_max_it_iu_len(int cmd_sg_cnt, bool use_imm_data, 1304 uint32_t max_it_iu_size) 1305 { 1306 uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN + 1307 sizeof(struct srp_indirect_buf) + 1308 cmd_sg_cnt * sizeof(struct srp_direct_buf); 1309 1310 if (use_imm_data) 1311 max_iu_len = max(max_iu_len, SRP_IMM_DATA_OFFSET + 1312 srp_max_imm_data); 1313 1314 if (max_it_iu_size) 1315 max_iu_len = min(max_iu_len, max_it_iu_size); 1316 1317 pr_debug("max_iu_len = %d\n", max_iu_len); 1318 1319 return max_iu_len; 1320 } 1321 1322 /* 1323 * It is up to the caller to ensure that srp_rport_reconnect() calls are 1324 * serialized and that no concurrent srp_queuecommand(), srp_abort(), 1325 * srp_reset_device() or srp_reset_host() calls will occur while this function 1326 * is in progress. One way to realize that is not to call this function 1327 * directly but to call srp_reconnect_rport() instead since that last function 1328 * serializes calls of this function via rport->mutex and also blocks 1329 * srp_queuecommand() calls before invoking this function. 1330 */ 1331 static int srp_rport_reconnect(struct srp_rport *rport) 1332 { 1333 struct srp_target_port *target = rport->lld_data; 1334 struct srp_rdma_ch *ch; 1335 uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 1336 srp_use_imm_data, 1337 target->max_it_iu_size); 1338 int i, j, ret = 0; 1339 bool multich = false; 1340 1341 srp_disconnect_target(target); 1342 1343 if (target->state == SRP_TARGET_SCANNING) 1344 return -ENODEV; 1345 1346 /* 1347 * Now get a new local CM ID so that we avoid confusing the target in 1348 * case things are really fouled up. Doing so also ensures that all CM 1349 * callbacks will have finished before a new QP is allocated. 1350 */ 1351 for (i = 0; i < target->ch_count; i++) { 1352 ch = &target->ch[i]; 1353 ret += srp_new_cm_id(ch); 1354 } 1355 { 1356 struct srp_terminate_context context = { 1357 .srp_target = target, .scsi_result = DID_RESET << 16}; 1358 1359 scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, 1360 &context); 1361 } 1362 for (i = 0; i < target->ch_count; i++) { 1363 ch = &target->ch[i]; 1364 /* 1365 * Whether or not creating a new CM ID succeeded, create a new 1366 * QP. This guarantees that all completion callback function 1367 * invocations have finished before request resetting starts. 1368 */ 1369 ret += srp_create_ch_ib(ch); 1370 1371 INIT_LIST_HEAD(&ch->free_tx); 1372 for (j = 0; j < target->queue_size; ++j) 1373 list_add(&ch->tx_ring[j]->list, &ch->free_tx); 1374 } 1375 1376 target->qp_in_error = false; 1377 1378 for (i = 0; i < target->ch_count; i++) { 1379 ch = &target->ch[i]; 1380 if (ret) 1381 break; 1382 ret = srp_connect_ch(ch, max_iu_len, multich); 1383 multich = true; 1384 } 1385 1386 if (ret == 0) 1387 shost_printk(KERN_INFO, target->scsi_host, 1388 PFX "reconnect succeeded\n"); 1389 1390 return ret; 1391 } 1392 1393 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, 1394 unsigned int dma_len, u32 rkey) 1395 { 1396 struct srp_direct_buf *desc = state->desc; 1397 1398 WARN_ON_ONCE(!dma_len); 1399 1400 desc->va = cpu_to_be64(dma_addr); 1401 desc->key = cpu_to_be32(rkey); 1402 desc->len = cpu_to_be32(dma_len); 1403 1404 state->total_len += dma_len; 1405 state->desc++; 1406 state->ndesc++; 1407 } 1408 1409 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc) 1410 { 1411 srp_handle_qp_err(cq, wc, "FAST REG"); 1412 } 1413 1414 /* 1415 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset 1416 * where to start in the first element. If sg_offset_p != NULL then 1417 * *sg_offset_p is updated to the offset in state->sg[retval] of the first 1418 * byte that has not yet been mapped. 1419 */ 1420 static int srp_map_finish_fr(struct srp_map_state *state, 1421 struct srp_request *req, 1422 struct srp_rdma_ch *ch, int sg_nents, 1423 unsigned int *sg_offset_p) 1424 { 1425 struct srp_target_port *target = ch->target; 1426 struct srp_device *dev = target->srp_host->srp_dev; 1427 struct ib_reg_wr wr; 1428 struct srp_fr_desc *desc; 1429 u32 rkey; 1430 int n, err; 1431 1432 if (state->fr.next >= state->fr.end) { 1433 shost_printk(KERN_ERR, ch->target->scsi_host, 1434 PFX "Out of MRs (mr_per_cmd = %d)\n", 1435 ch->target->mr_per_cmd); 1436 return -ENOMEM; 1437 } 1438 1439 WARN_ON_ONCE(!dev->use_fast_reg); 1440 1441 if (sg_nents == 1 && target->global_rkey) { 1442 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1443 1444 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset, 1445 sg_dma_len(state->sg) - sg_offset, 1446 target->global_rkey); 1447 if (sg_offset_p) 1448 *sg_offset_p = 0; 1449 return 1; 1450 } 1451 1452 desc = srp_fr_pool_get(ch->fr_pool); 1453 if (!desc) 1454 return -ENOMEM; 1455 1456 rkey = ib_inc_rkey(desc->mr->rkey); 1457 ib_update_fast_reg_key(desc->mr, rkey); 1458 1459 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p, 1460 dev->mr_page_size); 1461 if (unlikely(n < 0)) { 1462 srp_fr_pool_put(ch->fr_pool, &desc, 1); 1463 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n", 1464 dev_name(&req->scmnd->device->sdev_gendev), sg_nents, 1465 sg_offset_p ? *sg_offset_p : -1, n); 1466 return n; 1467 } 1468 1469 WARN_ON_ONCE(desc->mr->length == 0); 1470 1471 req->reg_cqe.done = srp_reg_mr_err_done; 1472 1473 wr.wr.next = NULL; 1474 wr.wr.opcode = IB_WR_REG_MR; 1475 wr.wr.wr_cqe = &req->reg_cqe; 1476 wr.wr.num_sge = 0; 1477 wr.wr.send_flags = 0; 1478 wr.mr = desc->mr; 1479 wr.key = desc->mr->rkey; 1480 wr.access = (IB_ACCESS_LOCAL_WRITE | 1481 IB_ACCESS_REMOTE_READ | 1482 IB_ACCESS_REMOTE_WRITE); 1483 1484 *state->fr.next++ = desc; 1485 state->nmdesc++; 1486 1487 srp_map_desc(state, desc->mr->iova, 1488 desc->mr->length, desc->mr->rkey); 1489 1490 err = ib_post_send(ch->qp, &wr.wr, NULL); 1491 if (unlikely(err)) { 1492 WARN_ON_ONCE(err == -ENOMEM); 1493 return err; 1494 } 1495 1496 return n; 1497 } 1498 1499 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1500 struct srp_request *req, struct scatterlist *scat, 1501 int count) 1502 { 1503 unsigned int sg_offset = 0; 1504 1505 state->fr.next = req->fr_list; 1506 state->fr.end = req->fr_list + ch->target->mr_per_cmd; 1507 state->sg = scat; 1508 1509 if (count == 0) 1510 return 0; 1511 1512 while (count) { 1513 int i, n; 1514 1515 n = srp_map_finish_fr(state, req, ch, count, &sg_offset); 1516 if (unlikely(n < 0)) 1517 return n; 1518 1519 count -= n; 1520 for (i = 0; i < n; i++) 1521 state->sg = sg_next(state->sg); 1522 } 1523 1524 return 0; 1525 } 1526 1527 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch, 1528 struct srp_request *req, struct scatterlist *scat, 1529 int count) 1530 { 1531 struct srp_target_port *target = ch->target; 1532 struct scatterlist *sg; 1533 int i; 1534 1535 for_each_sg(scat, sg, count, i) { 1536 srp_map_desc(state, sg_dma_address(sg), sg_dma_len(sg), 1537 target->global_rkey); 1538 } 1539 1540 return 0; 1541 } 1542 1543 /* 1544 * Register the indirect data buffer descriptor with the HCA. 1545 * 1546 * Note: since the indirect data buffer descriptor has been allocated with 1547 * kmalloc() it is guaranteed that this buffer is a physically contiguous 1548 * memory buffer. 1549 */ 1550 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req, 1551 void **next_mr, void **end_mr, u32 idb_len, 1552 __be32 *idb_rkey) 1553 { 1554 struct srp_target_port *target = ch->target; 1555 struct srp_device *dev = target->srp_host->srp_dev; 1556 struct srp_map_state state; 1557 struct srp_direct_buf idb_desc; 1558 struct scatterlist idb_sg[1]; 1559 int ret; 1560 1561 memset(&state, 0, sizeof(state)); 1562 memset(&idb_desc, 0, sizeof(idb_desc)); 1563 state.gen.next = next_mr; 1564 state.gen.end = end_mr; 1565 state.desc = &idb_desc; 1566 state.base_dma_addr = req->indirect_dma_addr; 1567 state.dma_len = idb_len; 1568 1569 if (dev->use_fast_reg) { 1570 state.sg = idb_sg; 1571 sg_init_one(idb_sg, req->indirect_desc, idb_len); 1572 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */ 1573 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1574 idb_sg->dma_length = idb_sg->length; /* hack^2 */ 1575 #endif 1576 ret = srp_map_finish_fr(&state, req, ch, 1, NULL); 1577 if (ret < 0) 1578 return ret; 1579 WARN_ON_ONCE(ret < 1); 1580 } else { 1581 return -EINVAL; 1582 } 1583 1584 *idb_rkey = idb_desc.key; 1585 1586 return 0; 1587 } 1588 1589 static void srp_check_mapping(struct srp_map_state *state, 1590 struct srp_rdma_ch *ch, struct srp_request *req, 1591 struct scatterlist *scat, int count) 1592 { 1593 struct srp_device *dev = ch->target->srp_host->srp_dev; 1594 struct srp_fr_desc **pfr; 1595 u64 desc_len = 0, mr_len = 0; 1596 int i; 1597 1598 for (i = 0; i < state->ndesc; i++) 1599 desc_len += be32_to_cpu(req->indirect_desc[i].len); 1600 if (dev->use_fast_reg) 1601 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++) 1602 mr_len += (*pfr)->mr->length; 1603 if (desc_len != scsi_bufflen(req->scmnd) || 1604 mr_len > scsi_bufflen(req->scmnd)) 1605 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n", 1606 scsi_bufflen(req->scmnd), desc_len, mr_len, 1607 state->ndesc, state->nmdesc); 1608 } 1609 1610 /** 1611 * srp_map_data() - map SCSI data buffer onto an SRP request 1612 * @scmnd: SCSI command to map 1613 * @ch: SRP RDMA channel 1614 * @req: SRP request 1615 * 1616 * Returns the length in bytes of the SRP_CMD IU or a negative value if 1617 * mapping failed. The size of any immediate data is not included in the 1618 * return value. 1619 */ 1620 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch, 1621 struct srp_request *req) 1622 { 1623 struct srp_target_port *target = ch->target; 1624 struct scatterlist *scat, *sg; 1625 struct srp_cmd *cmd = req->cmd->buf; 1626 int i, len, nents, count, ret; 1627 struct srp_device *dev; 1628 struct ib_device *ibdev; 1629 struct srp_map_state state; 1630 struct srp_indirect_buf *indirect_hdr; 1631 u64 data_len; 1632 u32 idb_len, table_len; 1633 __be32 idb_rkey; 1634 u8 fmt; 1635 1636 req->cmd->num_sge = 1; 1637 1638 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE) 1639 return sizeof(struct srp_cmd) + cmd->add_cdb_len; 1640 1641 if (scmnd->sc_data_direction != DMA_FROM_DEVICE && 1642 scmnd->sc_data_direction != DMA_TO_DEVICE) { 1643 shost_printk(KERN_WARNING, target->scsi_host, 1644 PFX "Unhandled data direction %d\n", 1645 scmnd->sc_data_direction); 1646 return -EINVAL; 1647 } 1648 1649 nents = scsi_sg_count(scmnd); 1650 scat = scsi_sglist(scmnd); 1651 data_len = scsi_bufflen(scmnd); 1652 1653 dev = target->srp_host->srp_dev; 1654 ibdev = dev->dev; 1655 1656 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction); 1657 if (unlikely(count == 0)) 1658 return -EIO; 1659 1660 if (ch->use_imm_data && 1661 count <= ch->max_imm_sge && 1662 SRP_IMM_DATA_OFFSET + data_len <= ch->max_it_iu_len && 1663 scmnd->sc_data_direction == DMA_TO_DEVICE) { 1664 struct srp_imm_buf *buf; 1665 struct ib_sge *sge = &req->cmd->sge[1]; 1666 1667 fmt = SRP_DATA_DESC_IMM; 1668 len = SRP_IMM_DATA_OFFSET; 1669 req->nmdesc = 0; 1670 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1671 buf->len = cpu_to_be32(data_len); 1672 WARN_ON_ONCE((void *)(buf + 1) > (void *)cmd + len); 1673 for_each_sg(scat, sg, count, i) { 1674 sge[i].addr = sg_dma_address(sg); 1675 sge[i].length = sg_dma_len(sg); 1676 sge[i].lkey = target->lkey; 1677 } 1678 req->cmd->num_sge += count; 1679 goto map_complete; 1680 } 1681 1682 fmt = SRP_DATA_DESC_DIRECT; 1683 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1684 sizeof(struct srp_direct_buf); 1685 1686 if (count == 1 && target->global_rkey) { 1687 /* 1688 * The midlayer only generated a single gather/scatter 1689 * entry, or DMA mapping coalesced everything to a 1690 * single entry. So a direct descriptor along with 1691 * the DMA MR suffices. 1692 */ 1693 struct srp_direct_buf *buf; 1694 1695 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1696 buf->va = cpu_to_be64(sg_dma_address(scat)); 1697 buf->key = cpu_to_be32(target->global_rkey); 1698 buf->len = cpu_to_be32(sg_dma_len(scat)); 1699 1700 req->nmdesc = 0; 1701 goto map_complete; 1702 } 1703 1704 /* 1705 * We have more than one scatter/gather entry, so build our indirect 1706 * descriptor table, trying to merge as many entries as we can. 1707 */ 1708 indirect_hdr = (void *)cmd->add_data + cmd->add_cdb_len; 1709 1710 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr, 1711 target->indirect_size, DMA_TO_DEVICE); 1712 1713 memset(&state, 0, sizeof(state)); 1714 state.desc = req->indirect_desc; 1715 if (dev->use_fast_reg) 1716 ret = srp_map_sg_fr(&state, ch, req, scat, count); 1717 else 1718 ret = srp_map_sg_dma(&state, ch, req, scat, count); 1719 req->nmdesc = state.nmdesc; 1720 if (ret < 0) 1721 goto unmap; 1722 1723 { 1724 DEFINE_DYNAMIC_DEBUG_METADATA(ddm, 1725 "Memory mapping consistency check"); 1726 if (DYNAMIC_DEBUG_BRANCH(ddm)) 1727 srp_check_mapping(&state, ch, req, scat, count); 1728 } 1729 1730 /* We've mapped the request, now pull as much of the indirect 1731 * descriptor table as we can into the command buffer. If this 1732 * target is not using an external indirect table, we are 1733 * guaranteed to fit into the command, as the SCSI layer won't 1734 * give us more S/G entries than we allow. 1735 */ 1736 if (state.ndesc == 1) { 1737 /* 1738 * Memory registration collapsed the sg-list into one entry, 1739 * so use a direct descriptor. 1740 */ 1741 struct srp_direct_buf *buf; 1742 1743 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1744 *buf = req->indirect_desc[0]; 1745 goto map_complete; 1746 } 1747 1748 if (unlikely(target->cmd_sg_cnt < state.ndesc && 1749 !target->allow_ext_sg)) { 1750 shost_printk(KERN_ERR, target->scsi_host, 1751 "Could not fit S/G list into SRP_CMD\n"); 1752 ret = -EIO; 1753 goto unmap; 1754 } 1755 1756 count = min(state.ndesc, target->cmd_sg_cnt); 1757 table_len = state.ndesc * sizeof (struct srp_direct_buf); 1758 idb_len = sizeof(struct srp_indirect_buf) + table_len; 1759 1760 fmt = SRP_DATA_DESC_INDIRECT; 1761 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1762 sizeof(struct srp_indirect_buf); 1763 len += count * sizeof (struct srp_direct_buf); 1764 1765 memcpy(indirect_hdr->desc_list, req->indirect_desc, 1766 count * sizeof (struct srp_direct_buf)); 1767 1768 if (!target->global_rkey) { 1769 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end, 1770 idb_len, &idb_rkey); 1771 if (ret < 0) 1772 goto unmap; 1773 req->nmdesc++; 1774 } else { 1775 idb_rkey = cpu_to_be32(target->global_rkey); 1776 } 1777 1778 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr); 1779 indirect_hdr->table_desc.key = idb_rkey; 1780 indirect_hdr->table_desc.len = cpu_to_be32(table_len); 1781 indirect_hdr->len = cpu_to_be32(state.total_len); 1782 1783 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1784 cmd->data_out_desc_cnt = count; 1785 else 1786 cmd->data_in_desc_cnt = count; 1787 1788 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len, 1789 DMA_TO_DEVICE); 1790 1791 map_complete: 1792 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1793 cmd->buf_fmt = fmt << 4; 1794 else 1795 cmd->buf_fmt = fmt; 1796 1797 return len; 1798 1799 unmap: 1800 srp_unmap_data(scmnd, ch, req); 1801 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size) 1802 ret = -E2BIG; 1803 return ret; 1804 } 1805 1806 /* 1807 * Return an IU and possible credit to the free pool 1808 */ 1809 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu, 1810 enum srp_iu_type iu_type) 1811 { 1812 unsigned long flags; 1813 1814 spin_lock_irqsave(&ch->lock, flags); 1815 list_add(&iu->list, &ch->free_tx); 1816 if (iu_type != SRP_IU_RSP) 1817 ++ch->req_lim; 1818 spin_unlock_irqrestore(&ch->lock, flags); 1819 } 1820 1821 /* 1822 * Must be called with ch->lock held to protect req_lim and free_tx. 1823 * If IU is not sent, it must be returned using srp_put_tx_iu(). 1824 * 1825 * Note: 1826 * An upper limit for the number of allocated information units for each 1827 * request type is: 1828 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues 1829 * more than Scsi_Host.can_queue requests. 1830 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE. 1831 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than 1832 * one unanswered SRP request to an initiator. 1833 */ 1834 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch, 1835 enum srp_iu_type iu_type) 1836 { 1837 struct srp_target_port *target = ch->target; 1838 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; 1839 struct srp_iu *iu; 1840 1841 lockdep_assert_held(&ch->lock); 1842 1843 ib_process_cq_direct(ch->send_cq, -1); 1844 1845 if (list_empty(&ch->free_tx)) 1846 return NULL; 1847 1848 /* Initiator responses to target requests do not consume credits */ 1849 if (iu_type != SRP_IU_RSP) { 1850 if (ch->req_lim <= rsv) { 1851 ++target->zero_req_lim; 1852 return NULL; 1853 } 1854 1855 --ch->req_lim; 1856 } 1857 1858 iu = list_first_entry(&ch->free_tx, struct srp_iu, list); 1859 list_del(&iu->list); 1860 return iu; 1861 } 1862 1863 /* 1864 * Note: if this function is called from inside ib_drain_sq() then it will 1865 * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE 1866 * with status IB_WC_SUCCESS then that's a bug. 1867 */ 1868 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc) 1869 { 1870 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1871 struct srp_rdma_ch *ch = cq->cq_context; 1872 1873 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1874 srp_handle_qp_err(cq, wc, "SEND"); 1875 return; 1876 } 1877 1878 lockdep_assert_held(&ch->lock); 1879 1880 list_add(&iu->list, &ch->free_tx); 1881 } 1882 1883 /** 1884 * srp_post_send() - send an SRP information unit 1885 * @ch: RDMA channel over which to send the information unit. 1886 * @iu: Information unit to send. 1887 * @len: Length of the information unit excluding immediate data. 1888 */ 1889 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len) 1890 { 1891 struct srp_target_port *target = ch->target; 1892 struct ib_send_wr wr; 1893 1894 if (WARN_ON_ONCE(iu->num_sge > SRP_MAX_SGE)) 1895 return -EINVAL; 1896 1897 iu->sge[0].addr = iu->dma; 1898 iu->sge[0].length = len; 1899 iu->sge[0].lkey = target->lkey; 1900 1901 iu->cqe.done = srp_send_done; 1902 1903 wr.next = NULL; 1904 wr.wr_cqe = &iu->cqe; 1905 wr.sg_list = &iu->sge[0]; 1906 wr.num_sge = iu->num_sge; 1907 wr.opcode = IB_WR_SEND; 1908 wr.send_flags = IB_SEND_SIGNALED; 1909 1910 return ib_post_send(ch->qp, &wr, NULL); 1911 } 1912 1913 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu) 1914 { 1915 struct srp_target_port *target = ch->target; 1916 struct ib_recv_wr wr; 1917 struct ib_sge list; 1918 1919 list.addr = iu->dma; 1920 list.length = iu->size; 1921 list.lkey = target->lkey; 1922 1923 iu->cqe.done = srp_recv_done; 1924 1925 wr.next = NULL; 1926 wr.wr_cqe = &iu->cqe; 1927 wr.sg_list = &list; 1928 wr.num_sge = 1; 1929 1930 return ib_post_recv(ch->qp, &wr, NULL); 1931 } 1932 1933 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) 1934 { 1935 struct srp_target_port *target = ch->target; 1936 struct srp_request *req; 1937 struct scsi_cmnd *scmnd; 1938 unsigned long flags; 1939 1940 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { 1941 spin_lock_irqsave(&ch->lock, flags); 1942 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1943 if (rsp->tag == ch->tsk_mgmt_tag) { 1944 ch->tsk_mgmt_status = -1; 1945 if (be32_to_cpu(rsp->resp_data_len) >= 4) 1946 ch->tsk_mgmt_status = rsp->data[3]; 1947 complete(&ch->tsk_mgmt_done); 1948 } else { 1949 shost_printk(KERN_ERR, target->scsi_host, 1950 "Received tsk mgmt response too late for tag %#llx\n", 1951 rsp->tag); 1952 } 1953 spin_unlock_irqrestore(&ch->lock, flags); 1954 } else { 1955 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag); 1956 if (scmnd) { 1957 req = scsi_cmd_priv(scmnd); 1958 scmnd = srp_claim_req(ch, req, NULL, scmnd); 1959 } 1960 if (!scmnd) { 1961 shost_printk(KERN_ERR, target->scsi_host, 1962 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", 1963 rsp->tag, ch - target->ch, ch->qp->qp_num); 1964 1965 spin_lock_irqsave(&ch->lock, flags); 1966 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1967 spin_unlock_irqrestore(&ch->lock, flags); 1968 1969 return; 1970 } 1971 scmnd->result = rsp->status; 1972 1973 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { 1974 memcpy(scmnd->sense_buffer, rsp->data + 1975 be32_to_cpu(rsp->resp_data_len), 1976 min_t(int, be32_to_cpu(rsp->sense_data_len), 1977 SCSI_SENSE_BUFFERSIZE)); 1978 } 1979 1980 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER)) 1981 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt)); 1982 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER)) 1983 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt)); 1984 1985 srp_free_req(ch, req, scmnd, 1986 be32_to_cpu(rsp->req_lim_delta)); 1987 1988 scsi_done(scmnd); 1989 } 1990 } 1991 1992 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, 1993 void *rsp, int len) 1994 { 1995 struct srp_target_port *target = ch->target; 1996 struct ib_device *dev = target->srp_host->srp_dev->dev; 1997 unsigned long flags; 1998 struct srp_iu *iu; 1999 int err; 2000 2001 spin_lock_irqsave(&ch->lock, flags); 2002 ch->req_lim += req_delta; 2003 iu = __srp_get_tx_iu(ch, SRP_IU_RSP); 2004 spin_unlock_irqrestore(&ch->lock, flags); 2005 2006 if (!iu) { 2007 shost_printk(KERN_ERR, target->scsi_host, PFX 2008 "no IU available to send response\n"); 2009 return 1; 2010 } 2011 2012 iu->num_sge = 1; 2013 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); 2014 memcpy(iu->buf, rsp, len); 2015 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); 2016 2017 err = srp_post_send(ch, iu, len); 2018 if (err) { 2019 shost_printk(KERN_ERR, target->scsi_host, PFX 2020 "unable to post response: %d\n", err); 2021 srp_put_tx_iu(ch, iu, SRP_IU_RSP); 2022 } 2023 2024 return err; 2025 } 2026 2027 static void srp_process_cred_req(struct srp_rdma_ch *ch, 2028 struct srp_cred_req *req) 2029 { 2030 struct srp_cred_rsp rsp = { 2031 .opcode = SRP_CRED_RSP, 2032 .tag = req->tag, 2033 }; 2034 s32 delta = be32_to_cpu(req->req_lim_delta); 2035 2036 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2037 shost_printk(KERN_ERR, ch->target->scsi_host, PFX 2038 "problems processing SRP_CRED_REQ\n"); 2039 } 2040 2041 static void srp_process_aer_req(struct srp_rdma_ch *ch, 2042 struct srp_aer_req *req) 2043 { 2044 struct srp_target_port *target = ch->target; 2045 struct srp_aer_rsp rsp = { 2046 .opcode = SRP_AER_RSP, 2047 .tag = req->tag, 2048 }; 2049 s32 delta = be32_to_cpu(req->req_lim_delta); 2050 2051 shost_printk(KERN_ERR, target->scsi_host, PFX 2052 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); 2053 2054 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2055 shost_printk(KERN_ERR, target->scsi_host, PFX 2056 "problems processing SRP_AER_REQ\n"); 2057 } 2058 2059 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) 2060 { 2061 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 2062 struct srp_rdma_ch *ch = cq->cq_context; 2063 struct srp_target_port *target = ch->target; 2064 struct ib_device *dev = target->srp_host->srp_dev->dev; 2065 int res; 2066 u8 opcode; 2067 2068 if (unlikely(wc->status != IB_WC_SUCCESS)) { 2069 srp_handle_qp_err(cq, wc, "RECV"); 2070 return; 2071 } 2072 2073 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len, 2074 DMA_FROM_DEVICE); 2075 2076 opcode = *(u8 *) iu->buf; 2077 2078 if (0) { 2079 shost_printk(KERN_ERR, target->scsi_host, 2080 PFX "recv completion, opcode 0x%02x\n", opcode); 2081 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1, 2082 iu->buf, wc->byte_len, true); 2083 } 2084 2085 switch (opcode) { 2086 case SRP_RSP: 2087 srp_process_rsp(ch, iu->buf); 2088 break; 2089 2090 case SRP_CRED_REQ: 2091 srp_process_cred_req(ch, iu->buf); 2092 break; 2093 2094 case SRP_AER_REQ: 2095 srp_process_aer_req(ch, iu->buf); 2096 break; 2097 2098 case SRP_T_LOGOUT: 2099 /* XXX Handle target logout */ 2100 shost_printk(KERN_WARNING, target->scsi_host, 2101 PFX "Got target logout request\n"); 2102 break; 2103 2104 default: 2105 shost_printk(KERN_WARNING, target->scsi_host, 2106 PFX "Unhandled SRP opcode 0x%02x\n", opcode); 2107 break; 2108 } 2109 2110 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len, 2111 DMA_FROM_DEVICE); 2112 2113 res = srp_post_recv(ch, iu); 2114 if (res != 0) 2115 shost_printk(KERN_ERR, target->scsi_host, 2116 PFX "Recv failed with error code %d\n", res); 2117 } 2118 2119 /** 2120 * srp_tl_err_work() - handle a transport layer error 2121 * @work: Work structure embedded in an SRP target port. 2122 * 2123 * Note: This function may get invoked before the rport has been created, 2124 * hence the target->rport test. 2125 */ 2126 static void srp_tl_err_work(struct work_struct *work) 2127 { 2128 struct srp_target_port *target; 2129 2130 target = container_of(work, struct srp_target_port, tl_err_work); 2131 if (target->rport) 2132 srp_start_tl_fail_timers(target->rport); 2133 } 2134 2135 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 2136 const char *opname) 2137 { 2138 struct srp_rdma_ch *ch = cq->cq_context; 2139 struct srp_target_port *target = ch->target; 2140 2141 if (ch->connected && !target->qp_in_error) { 2142 shost_printk(KERN_ERR, target->scsi_host, 2143 PFX "failed %s status %s (%d) for CQE %p\n", 2144 opname, ib_wc_status_msg(wc->status), wc->status, 2145 wc->wr_cqe); 2146 queue_work(system_long_wq, &target->tl_err_work); 2147 } 2148 target->qp_in_error = true; 2149 } 2150 2151 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) 2152 { 2153 struct request *rq = scsi_cmd_to_rq(scmnd); 2154 struct srp_target_port *target = host_to_target(shost); 2155 struct srp_rdma_ch *ch; 2156 struct srp_request *req = scsi_cmd_priv(scmnd); 2157 struct srp_iu *iu; 2158 struct srp_cmd *cmd; 2159 struct ib_device *dev; 2160 unsigned long flags; 2161 u32 tag; 2162 int len, ret; 2163 2164 scmnd->result = srp_chkready(target->rport); 2165 if (unlikely(scmnd->result)) 2166 goto err; 2167 2168 WARN_ON_ONCE(rq->tag < 0); 2169 tag = blk_mq_unique_tag(rq); 2170 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 2171 2172 spin_lock_irqsave(&ch->lock, flags); 2173 iu = __srp_get_tx_iu(ch, SRP_IU_CMD); 2174 spin_unlock_irqrestore(&ch->lock, flags); 2175 2176 if (!iu) 2177 goto err; 2178 2179 dev = target->srp_host->srp_dev->dev; 2180 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len, 2181 DMA_TO_DEVICE); 2182 2183 cmd = iu->buf; 2184 memset(cmd, 0, sizeof *cmd); 2185 2186 cmd->opcode = SRP_CMD; 2187 int_to_scsilun(scmnd->device->lun, &cmd->lun); 2188 cmd->tag = tag; 2189 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len); 2190 if (unlikely(scmnd->cmd_len > sizeof(cmd->cdb))) { 2191 cmd->add_cdb_len = round_up(scmnd->cmd_len - sizeof(cmd->cdb), 2192 4); 2193 if (WARN_ON_ONCE(cmd->add_cdb_len > SRP_MAX_ADD_CDB_LEN)) 2194 goto err_iu; 2195 } 2196 2197 req->scmnd = scmnd; 2198 req->cmd = iu; 2199 2200 len = srp_map_data(scmnd, ch, req); 2201 if (len < 0) { 2202 shost_printk(KERN_ERR, target->scsi_host, 2203 PFX "Failed to map data (%d)\n", len); 2204 /* 2205 * If we ran out of memory descriptors (-ENOMEM) because an 2206 * application is queuing many requests with more than 2207 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer 2208 * to reduce queue depth temporarily. 2209 */ 2210 scmnd->result = len == -ENOMEM ? 2211 DID_OK << 16 | SAM_STAT_TASK_SET_FULL : DID_ERROR << 16; 2212 goto err_iu; 2213 } 2214 2215 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len, 2216 DMA_TO_DEVICE); 2217 2218 if (srp_post_send(ch, iu, len)) { 2219 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); 2220 scmnd->result = DID_ERROR << 16; 2221 goto err_unmap; 2222 } 2223 2224 return 0; 2225 2226 err_unmap: 2227 srp_unmap_data(scmnd, ch, req); 2228 2229 err_iu: 2230 srp_put_tx_iu(ch, iu, SRP_IU_CMD); 2231 2232 /* 2233 * Avoid that the loops that iterate over the request ring can 2234 * encounter a dangling SCSI command pointer. 2235 */ 2236 req->scmnd = NULL; 2237 2238 err: 2239 if (scmnd->result) { 2240 scsi_done(scmnd); 2241 ret = 0; 2242 } else { 2243 ret = SCSI_MLQUEUE_HOST_BUSY; 2244 } 2245 2246 return ret; 2247 } 2248 2249 /* 2250 * Note: the resources allocated in this function are freed in 2251 * srp_free_ch_ib(). 2252 */ 2253 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch) 2254 { 2255 struct srp_target_port *target = ch->target; 2256 int i; 2257 2258 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring), 2259 GFP_KERNEL); 2260 if (!ch->rx_ring) 2261 goto err_no_ring; 2262 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring), 2263 GFP_KERNEL); 2264 if (!ch->tx_ring) 2265 goto err_no_ring; 2266 2267 for (i = 0; i < target->queue_size; ++i) { 2268 ch->rx_ring[i] = srp_alloc_iu(target->srp_host, 2269 ch->max_ti_iu_len, 2270 GFP_KERNEL, DMA_FROM_DEVICE); 2271 if (!ch->rx_ring[i]) 2272 goto err; 2273 } 2274 2275 for (i = 0; i < target->queue_size; ++i) { 2276 ch->tx_ring[i] = srp_alloc_iu(target->srp_host, 2277 ch->max_it_iu_len, 2278 GFP_KERNEL, DMA_TO_DEVICE); 2279 if (!ch->tx_ring[i]) 2280 goto err; 2281 2282 list_add(&ch->tx_ring[i]->list, &ch->free_tx); 2283 } 2284 2285 return 0; 2286 2287 err: 2288 for (i = 0; i < target->queue_size; ++i) { 2289 srp_free_iu(target->srp_host, ch->rx_ring[i]); 2290 srp_free_iu(target->srp_host, ch->tx_ring[i]); 2291 } 2292 2293 2294 err_no_ring: 2295 kfree(ch->tx_ring); 2296 ch->tx_ring = NULL; 2297 kfree(ch->rx_ring); 2298 ch->rx_ring = NULL; 2299 2300 return -ENOMEM; 2301 } 2302 2303 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) 2304 { 2305 uint64_t T_tr_ns, max_compl_time_ms; 2306 uint32_t rq_tmo_jiffies; 2307 2308 /* 2309 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair, 2310 * table 91), both the QP timeout and the retry count have to be set 2311 * for RC QP's during the RTR to RTS transition. 2312 */ 2313 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) != 2314 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)); 2315 2316 /* 2317 * Set target->rq_tmo_jiffies to one second more than the largest time 2318 * it can take before an error completion is generated. See also 2319 * C9-140..142 in the IBTA spec for more information about how to 2320 * convert the QP Local ACK Timeout value to nanoseconds. 2321 */ 2322 T_tr_ns = 4096 * (1ULL << qp_attr->timeout); 2323 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns; 2324 do_div(max_compl_time_ms, NSEC_PER_MSEC); 2325 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000); 2326 2327 return rq_tmo_jiffies; 2328 } 2329 2330 static void srp_cm_rep_handler(struct ib_cm_id *cm_id, 2331 const struct srp_login_rsp *lrsp, 2332 struct srp_rdma_ch *ch) 2333 { 2334 struct srp_target_port *target = ch->target; 2335 struct ib_qp_attr *qp_attr = NULL; 2336 int attr_mask = 0; 2337 int ret = 0; 2338 int i; 2339 2340 if (lrsp->opcode == SRP_LOGIN_RSP) { 2341 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); 2342 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); 2343 ch->use_imm_data = srp_use_imm_data && 2344 (lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP); 2345 ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 2346 ch->use_imm_data, 2347 target->max_it_iu_size); 2348 WARN_ON_ONCE(ch->max_it_iu_len > 2349 be32_to_cpu(lrsp->max_it_iu_len)); 2350 2351 if (ch->use_imm_data) 2352 shost_printk(KERN_DEBUG, target->scsi_host, 2353 PFX "using immediate data\n"); 2354 2355 /* 2356 * Reserve credits for task management so we don't 2357 * bounce requests back to the SCSI mid-layer. 2358 */ 2359 target->scsi_host->can_queue 2360 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE, 2361 target->scsi_host->can_queue); 2362 target->scsi_host->cmd_per_lun 2363 = min_t(int, target->scsi_host->can_queue, 2364 target->scsi_host->cmd_per_lun); 2365 } else { 2366 shost_printk(KERN_WARNING, target->scsi_host, 2367 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode); 2368 ret = -ECONNRESET; 2369 goto error; 2370 } 2371 2372 if (!ch->rx_ring) { 2373 ret = srp_alloc_iu_bufs(ch); 2374 if (ret) 2375 goto error; 2376 } 2377 2378 for (i = 0; i < target->queue_size; i++) { 2379 struct srp_iu *iu = ch->rx_ring[i]; 2380 2381 ret = srp_post_recv(ch, iu); 2382 if (ret) 2383 goto error; 2384 } 2385 2386 if (!target->using_rdma_cm) { 2387 ret = -ENOMEM; 2388 qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL); 2389 if (!qp_attr) 2390 goto error; 2391 2392 qp_attr->qp_state = IB_QPS_RTR; 2393 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2394 if (ret) 2395 goto error_free; 2396 2397 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2398 if (ret) 2399 goto error_free; 2400 2401 qp_attr->qp_state = IB_QPS_RTS; 2402 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2403 if (ret) 2404 goto error_free; 2405 2406 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); 2407 2408 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2409 if (ret) 2410 goto error_free; 2411 2412 ret = ib_send_cm_rtu(cm_id, NULL, 0); 2413 } 2414 2415 error_free: 2416 kfree(qp_attr); 2417 2418 error: 2419 ch->status = ret; 2420 } 2421 2422 static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id, 2423 const struct ib_cm_event *event, 2424 struct srp_rdma_ch *ch) 2425 { 2426 struct srp_target_port *target = ch->target; 2427 struct Scsi_Host *shost = target->scsi_host; 2428 struct ib_class_port_info *cpi; 2429 int opcode; 2430 u16 dlid; 2431 2432 switch (event->param.rej_rcvd.reason) { 2433 case IB_CM_REJ_PORT_CM_REDIRECT: 2434 cpi = event->param.rej_rcvd.ari; 2435 dlid = be16_to_cpu(cpi->redirect_lid); 2436 sa_path_set_dlid(&ch->ib_cm.path, dlid); 2437 ch->ib_cm.path.pkey = cpi->redirect_pkey; 2438 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; 2439 memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16); 2440 2441 ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; 2442 break; 2443 2444 case IB_CM_REJ_PORT_REDIRECT: 2445 if (srp_target_is_topspin(target)) { 2446 union ib_gid *dgid = &ch->ib_cm.path.dgid; 2447 2448 /* 2449 * Topspin/Cisco SRP gateways incorrectly send 2450 * reject reason code 25 when they mean 24 2451 * (port redirect). 2452 */ 2453 memcpy(dgid->raw, event->param.rej_rcvd.ari, 16); 2454 2455 shost_printk(KERN_DEBUG, shost, 2456 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", 2457 be64_to_cpu(dgid->global.subnet_prefix), 2458 be64_to_cpu(dgid->global.interface_id)); 2459 2460 ch->status = SRP_PORT_REDIRECT; 2461 } else { 2462 shost_printk(KERN_WARNING, shost, 2463 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); 2464 ch->status = -ECONNRESET; 2465 } 2466 break; 2467 2468 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2469 shost_printk(KERN_WARNING, shost, 2470 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2471 ch->status = -ECONNRESET; 2472 break; 2473 2474 case IB_CM_REJ_CONSUMER_DEFINED: 2475 opcode = *(u8 *) event->private_data; 2476 if (opcode == SRP_LOGIN_REJ) { 2477 struct srp_login_rej *rej = event->private_data; 2478 u32 reason = be32_to_cpu(rej->reason); 2479 2480 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2481 shost_printk(KERN_WARNING, shost, 2482 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2483 else 2484 shost_printk(KERN_WARNING, shost, PFX 2485 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n", 2486 target->sgid.raw, 2487 target->ib_cm.orig_dgid.raw, 2488 reason); 2489 } else 2490 shost_printk(KERN_WARNING, shost, 2491 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED," 2492 " opcode 0x%02x\n", opcode); 2493 ch->status = -ECONNRESET; 2494 break; 2495 2496 case IB_CM_REJ_STALE_CONN: 2497 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); 2498 ch->status = SRP_STALE_CONN; 2499 break; 2500 2501 default: 2502 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2503 event->param.rej_rcvd.reason); 2504 ch->status = -ECONNRESET; 2505 } 2506 } 2507 2508 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 2509 const struct ib_cm_event *event) 2510 { 2511 struct srp_rdma_ch *ch = cm_id->context; 2512 struct srp_target_port *target = ch->target; 2513 int comp = 0; 2514 2515 switch (event->event) { 2516 case IB_CM_REQ_ERROR: 2517 shost_printk(KERN_DEBUG, target->scsi_host, 2518 PFX "Sending CM REQ failed\n"); 2519 comp = 1; 2520 ch->status = -ECONNRESET; 2521 break; 2522 2523 case IB_CM_REP_RECEIVED: 2524 comp = 1; 2525 srp_cm_rep_handler(cm_id, event->private_data, ch); 2526 break; 2527 2528 case IB_CM_REJ_RECEIVED: 2529 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2530 comp = 1; 2531 2532 srp_ib_cm_rej_handler(cm_id, event, ch); 2533 break; 2534 2535 case IB_CM_DREQ_RECEIVED: 2536 shost_printk(KERN_WARNING, target->scsi_host, 2537 PFX "DREQ received - connection closed\n"); 2538 ch->connected = false; 2539 if (ib_send_cm_drep(cm_id, NULL, 0)) 2540 shost_printk(KERN_ERR, target->scsi_host, 2541 PFX "Sending CM DREP failed\n"); 2542 queue_work(system_long_wq, &target->tl_err_work); 2543 break; 2544 2545 case IB_CM_TIMEWAIT_EXIT: 2546 shost_printk(KERN_ERR, target->scsi_host, 2547 PFX "connection closed\n"); 2548 comp = 1; 2549 2550 ch->status = 0; 2551 break; 2552 2553 case IB_CM_MRA_RECEIVED: 2554 case IB_CM_DREQ_ERROR: 2555 case IB_CM_DREP_RECEIVED: 2556 break; 2557 2558 default: 2559 shost_printk(KERN_WARNING, target->scsi_host, 2560 PFX "Unhandled CM event %d\n", event->event); 2561 break; 2562 } 2563 2564 if (comp) 2565 complete(&ch->done); 2566 2567 return 0; 2568 } 2569 2570 static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch, 2571 struct rdma_cm_event *event) 2572 { 2573 struct srp_target_port *target = ch->target; 2574 struct Scsi_Host *shost = target->scsi_host; 2575 int opcode; 2576 2577 switch (event->status) { 2578 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2579 shost_printk(KERN_WARNING, shost, 2580 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2581 ch->status = -ECONNRESET; 2582 break; 2583 2584 case IB_CM_REJ_CONSUMER_DEFINED: 2585 opcode = *(u8 *) event->param.conn.private_data; 2586 if (opcode == SRP_LOGIN_REJ) { 2587 struct srp_login_rej *rej = 2588 (struct srp_login_rej *) 2589 event->param.conn.private_data; 2590 u32 reason = be32_to_cpu(rej->reason); 2591 2592 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2593 shost_printk(KERN_WARNING, shost, 2594 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2595 else 2596 shost_printk(KERN_WARNING, shost, 2597 PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason); 2598 } else { 2599 shost_printk(KERN_WARNING, shost, 2600 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n", 2601 opcode); 2602 } 2603 ch->status = -ECONNRESET; 2604 break; 2605 2606 case IB_CM_REJ_STALE_CONN: 2607 shost_printk(KERN_WARNING, shost, 2608 " REJ reason: stale connection\n"); 2609 ch->status = SRP_STALE_CONN; 2610 break; 2611 2612 default: 2613 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2614 event->status); 2615 ch->status = -ECONNRESET; 2616 break; 2617 } 2618 } 2619 2620 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 2621 struct rdma_cm_event *event) 2622 { 2623 struct srp_rdma_ch *ch = cm_id->context; 2624 struct srp_target_port *target = ch->target; 2625 int comp = 0; 2626 2627 switch (event->event) { 2628 case RDMA_CM_EVENT_ADDR_RESOLVED: 2629 ch->status = 0; 2630 comp = 1; 2631 break; 2632 2633 case RDMA_CM_EVENT_ADDR_ERROR: 2634 ch->status = -ENXIO; 2635 comp = 1; 2636 break; 2637 2638 case RDMA_CM_EVENT_ROUTE_RESOLVED: 2639 ch->status = 0; 2640 comp = 1; 2641 break; 2642 2643 case RDMA_CM_EVENT_ROUTE_ERROR: 2644 case RDMA_CM_EVENT_UNREACHABLE: 2645 ch->status = -EHOSTUNREACH; 2646 comp = 1; 2647 break; 2648 2649 case RDMA_CM_EVENT_CONNECT_ERROR: 2650 shost_printk(KERN_DEBUG, target->scsi_host, 2651 PFX "Sending CM REQ failed\n"); 2652 comp = 1; 2653 ch->status = -ECONNRESET; 2654 break; 2655 2656 case RDMA_CM_EVENT_ESTABLISHED: 2657 comp = 1; 2658 srp_cm_rep_handler(NULL, event->param.conn.private_data, ch); 2659 break; 2660 2661 case RDMA_CM_EVENT_REJECTED: 2662 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2663 comp = 1; 2664 2665 srp_rdma_cm_rej_handler(ch, event); 2666 break; 2667 2668 case RDMA_CM_EVENT_DISCONNECTED: 2669 if (ch->connected) { 2670 shost_printk(KERN_WARNING, target->scsi_host, 2671 PFX "received DREQ\n"); 2672 rdma_disconnect(ch->rdma_cm.cm_id); 2673 comp = 1; 2674 ch->status = 0; 2675 queue_work(system_long_wq, &target->tl_err_work); 2676 } 2677 break; 2678 2679 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 2680 shost_printk(KERN_ERR, target->scsi_host, 2681 PFX "connection closed\n"); 2682 2683 comp = 1; 2684 ch->status = 0; 2685 break; 2686 2687 default: 2688 shost_printk(KERN_WARNING, target->scsi_host, 2689 PFX "Unhandled CM event %d\n", event->event); 2690 break; 2691 } 2692 2693 if (comp) 2694 complete(&ch->done); 2695 2696 return 0; 2697 } 2698 2699 /** 2700 * srp_change_queue_depth - setting device queue depth 2701 * @sdev: scsi device struct 2702 * @qdepth: requested queue depth 2703 * 2704 * Returns queue depth. 2705 */ 2706 static int 2707 srp_change_queue_depth(struct scsi_device *sdev, int qdepth) 2708 { 2709 if (!sdev->tagged_supported) 2710 qdepth = 1; 2711 return scsi_change_queue_depth(sdev, qdepth); 2712 } 2713 2714 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, 2715 u8 func, u8 *status) 2716 { 2717 struct srp_target_port *target = ch->target; 2718 struct srp_rport *rport = target->rport; 2719 struct ib_device *dev = target->srp_host->srp_dev->dev; 2720 struct srp_iu *iu; 2721 struct srp_tsk_mgmt *tsk_mgmt; 2722 int res; 2723 2724 if (!ch->connected || target->qp_in_error) 2725 return -1; 2726 2727 /* 2728 * Lock the rport mutex to avoid that srp_create_ch_ib() is 2729 * invoked while a task management function is being sent. 2730 */ 2731 mutex_lock(&rport->mutex); 2732 spin_lock_irq(&ch->lock); 2733 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT); 2734 spin_unlock_irq(&ch->lock); 2735 2736 if (!iu) { 2737 mutex_unlock(&rport->mutex); 2738 2739 return -1; 2740 } 2741 2742 iu->num_sge = 1; 2743 2744 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, 2745 DMA_TO_DEVICE); 2746 tsk_mgmt = iu->buf; 2747 memset(tsk_mgmt, 0, sizeof *tsk_mgmt); 2748 2749 tsk_mgmt->opcode = SRP_TSK_MGMT; 2750 int_to_scsilun(lun, &tsk_mgmt->lun); 2751 tsk_mgmt->tsk_mgmt_func = func; 2752 tsk_mgmt->task_tag = req_tag; 2753 2754 spin_lock_irq(&ch->lock); 2755 ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT; 2756 tsk_mgmt->tag = ch->tsk_mgmt_tag; 2757 spin_unlock_irq(&ch->lock); 2758 2759 init_completion(&ch->tsk_mgmt_done); 2760 2761 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, 2762 DMA_TO_DEVICE); 2763 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) { 2764 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT); 2765 mutex_unlock(&rport->mutex); 2766 2767 return -1; 2768 } 2769 res = wait_for_completion_timeout(&ch->tsk_mgmt_done, 2770 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)); 2771 if (res > 0 && status) 2772 *status = ch->tsk_mgmt_status; 2773 mutex_unlock(&rport->mutex); 2774 2775 WARN_ON_ONCE(res < 0); 2776 2777 return res > 0 ? 0 : -1; 2778 } 2779 2780 static int srp_abort(struct scsi_cmnd *scmnd) 2781 { 2782 struct srp_target_port *target = host_to_target(scmnd->device->host); 2783 struct srp_request *req = scsi_cmd_priv(scmnd); 2784 u32 tag; 2785 u16 ch_idx; 2786 struct srp_rdma_ch *ch; 2787 2788 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); 2789 2790 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); 2791 ch_idx = blk_mq_unique_tag_to_hwq(tag); 2792 if (WARN_ON_ONCE(ch_idx >= target->ch_count)) 2793 return SUCCESS; 2794 ch = &target->ch[ch_idx]; 2795 if (!srp_claim_req(ch, req, NULL, scmnd)) 2796 return SUCCESS; 2797 shost_printk(KERN_ERR, target->scsi_host, 2798 "Sending SRP abort for tag %#x\n", tag); 2799 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun, 2800 SRP_TSK_ABORT_TASK, NULL) == 0) { 2801 srp_free_req(ch, req, scmnd, 0); 2802 return SUCCESS; 2803 } 2804 if (target->rport->state == SRP_RPORT_LOST) 2805 return FAST_IO_FAIL; 2806 2807 return FAILED; 2808 } 2809 2810 static int srp_reset_device(struct scsi_cmnd *scmnd) 2811 { 2812 struct srp_target_port *target = host_to_target(scmnd->device->host); 2813 struct srp_rdma_ch *ch; 2814 u8 status; 2815 2816 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); 2817 2818 ch = &target->ch[0]; 2819 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun, 2820 SRP_TSK_LUN_RESET, &status)) 2821 return FAILED; 2822 if (status) 2823 return FAILED; 2824 2825 return SUCCESS; 2826 } 2827 2828 static int srp_reset_host(struct scsi_cmnd *scmnd) 2829 { 2830 struct srp_target_port *target = host_to_target(scmnd->device->host); 2831 2832 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n"); 2833 2834 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED; 2835 } 2836 2837 static int srp_target_alloc(struct scsi_target *starget) 2838 { 2839 struct Scsi_Host *shost = dev_to_shost(starget->dev.parent); 2840 struct srp_target_port *target = host_to_target(shost); 2841 2842 if (target->target_can_queue) 2843 starget->can_queue = target->target_can_queue; 2844 return 0; 2845 } 2846 2847 static int srp_slave_configure(struct scsi_device *sdev) 2848 { 2849 struct Scsi_Host *shost = sdev->host; 2850 struct srp_target_port *target = host_to_target(shost); 2851 struct request_queue *q = sdev->request_queue; 2852 unsigned long timeout; 2853 2854 if (sdev->type == TYPE_DISK) { 2855 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies); 2856 blk_queue_rq_timeout(q, timeout); 2857 } 2858 2859 return 0; 2860 } 2861 2862 static ssize_t id_ext_show(struct device *dev, struct device_attribute *attr, 2863 char *buf) 2864 { 2865 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2866 2867 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->id_ext)); 2868 } 2869 2870 static DEVICE_ATTR_RO(id_ext); 2871 2872 static ssize_t ioc_guid_show(struct device *dev, struct device_attribute *attr, 2873 char *buf) 2874 { 2875 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2876 2877 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid)); 2878 } 2879 2880 static DEVICE_ATTR_RO(ioc_guid); 2881 2882 static ssize_t service_id_show(struct device *dev, 2883 struct device_attribute *attr, char *buf) 2884 { 2885 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2886 2887 if (target->using_rdma_cm) 2888 return -ENOENT; 2889 return sysfs_emit(buf, "0x%016llx\n", 2890 be64_to_cpu(target->ib_cm.service_id)); 2891 } 2892 2893 static DEVICE_ATTR_RO(service_id); 2894 2895 static ssize_t pkey_show(struct device *dev, struct device_attribute *attr, 2896 char *buf) 2897 { 2898 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2899 2900 if (target->using_rdma_cm) 2901 return -ENOENT; 2902 2903 return sysfs_emit(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey)); 2904 } 2905 2906 static DEVICE_ATTR_RO(pkey); 2907 2908 static ssize_t sgid_show(struct device *dev, struct device_attribute *attr, 2909 char *buf) 2910 { 2911 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2912 2913 return sysfs_emit(buf, "%pI6\n", target->sgid.raw); 2914 } 2915 2916 static DEVICE_ATTR_RO(sgid); 2917 2918 static ssize_t dgid_show(struct device *dev, struct device_attribute *attr, 2919 char *buf) 2920 { 2921 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2922 struct srp_rdma_ch *ch = &target->ch[0]; 2923 2924 if (target->using_rdma_cm) 2925 return -ENOENT; 2926 2927 return sysfs_emit(buf, "%pI6\n", ch->ib_cm.path.dgid.raw); 2928 } 2929 2930 static DEVICE_ATTR_RO(dgid); 2931 2932 static ssize_t orig_dgid_show(struct device *dev, struct device_attribute *attr, 2933 char *buf) 2934 { 2935 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2936 2937 if (target->using_rdma_cm) 2938 return -ENOENT; 2939 2940 return sysfs_emit(buf, "%pI6\n", target->ib_cm.orig_dgid.raw); 2941 } 2942 2943 static DEVICE_ATTR_RO(orig_dgid); 2944 2945 static ssize_t req_lim_show(struct device *dev, struct device_attribute *attr, 2946 char *buf) 2947 { 2948 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2949 struct srp_rdma_ch *ch; 2950 int i, req_lim = INT_MAX; 2951 2952 for (i = 0; i < target->ch_count; i++) { 2953 ch = &target->ch[i]; 2954 req_lim = min(req_lim, ch->req_lim); 2955 } 2956 2957 return sysfs_emit(buf, "%d\n", req_lim); 2958 } 2959 2960 static DEVICE_ATTR_RO(req_lim); 2961 2962 static ssize_t zero_req_lim_show(struct device *dev, 2963 struct device_attribute *attr, char *buf) 2964 { 2965 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2966 2967 return sysfs_emit(buf, "%d\n", target->zero_req_lim); 2968 } 2969 2970 static DEVICE_ATTR_RO(zero_req_lim); 2971 2972 static ssize_t local_ib_port_show(struct device *dev, 2973 struct device_attribute *attr, char *buf) 2974 { 2975 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2976 2977 return sysfs_emit(buf, "%u\n", target->srp_host->port); 2978 } 2979 2980 static DEVICE_ATTR_RO(local_ib_port); 2981 2982 static ssize_t local_ib_device_show(struct device *dev, 2983 struct device_attribute *attr, char *buf) 2984 { 2985 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2986 2987 return sysfs_emit(buf, "%s\n", 2988 dev_name(&target->srp_host->srp_dev->dev->dev)); 2989 } 2990 2991 static DEVICE_ATTR_RO(local_ib_device); 2992 2993 static ssize_t ch_count_show(struct device *dev, struct device_attribute *attr, 2994 char *buf) 2995 { 2996 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2997 2998 return sysfs_emit(buf, "%d\n", target->ch_count); 2999 } 3000 3001 static DEVICE_ATTR_RO(ch_count); 3002 3003 static ssize_t comp_vector_show(struct device *dev, 3004 struct device_attribute *attr, char *buf) 3005 { 3006 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3007 3008 return sysfs_emit(buf, "%d\n", target->comp_vector); 3009 } 3010 3011 static DEVICE_ATTR_RO(comp_vector); 3012 3013 static ssize_t tl_retry_count_show(struct device *dev, 3014 struct device_attribute *attr, char *buf) 3015 { 3016 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3017 3018 return sysfs_emit(buf, "%d\n", target->tl_retry_count); 3019 } 3020 3021 static DEVICE_ATTR_RO(tl_retry_count); 3022 3023 static ssize_t cmd_sg_entries_show(struct device *dev, 3024 struct device_attribute *attr, char *buf) 3025 { 3026 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3027 3028 return sysfs_emit(buf, "%u\n", target->cmd_sg_cnt); 3029 } 3030 3031 static DEVICE_ATTR_RO(cmd_sg_entries); 3032 3033 static ssize_t allow_ext_sg_show(struct device *dev, 3034 struct device_attribute *attr, char *buf) 3035 { 3036 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3037 3038 return sysfs_emit(buf, "%s\n", target->allow_ext_sg ? "true" : "false"); 3039 } 3040 3041 static DEVICE_ATTR_RO(allow_ext_sg); 3042 3043 static struct attribute *srp_host_attrs[] = { 3044 &dev_attr_id_ext.attr, 3045 &dev_attr_ioc_guid.attr, 3046 &dev_attr_service_id.attr, 3047 &dev_attr_pkey.attr, 3048 &dev_attr_sgid.attr, 3049 &dev_attr_dgid.attr, 3050 &dev_attr_orig_dgid.attr, 3051 &dev_attr_req_lim.attr, 3052 &dev_attr_zero_req_lim.attr, 3053 &dev_attr_local_ib_port.attr, 3054 &dev_attr_local_ib_device.attr, 3055 &dev_attr_ch_count.attr, 3056 &dev_attr_comp_vector.attr, 3057 &dev_attr_tl_retry_count.attr, 3058 &dev_attr_cmd_sg_entries.attr, 3059 &dev_attr_allow_ext_sg.attr, 3060 NULL 3061 }; 3062 3063 ATTRIBUTE_GROUPS(srp_host); 3064 3065 static const struct scsi_host_template srp_template = { 3066 .module = THIS_MODULE, 3067 .name = "InfiniBand SRP initiator", 3068 .proc_name = DRV_NAME, 3069 .target_alloc = srp_target_alloc, 3070 .slave_configure = srp_slave_configure, 3071 .info = srp_target_info, 3072 .init_cmd_priv = srp_init_cmd_priv, 3073 .exit_cmd_priv = srp_exit_cmd_priv, 3074 .queuecommand = srp_queuecommand, 3075 .change_queue_depth = srp_change_queue_depth, 3076 .eh_timed_out = srp_timed_out, 3077 .eh_abort_handler = srp_abort, 3078 .eh_device_reset_handler = srp_reset_device, 3079 .eh_host_reset_handler = srp_reset_host, 3080 .skip_settle_delay = true, 3081 .sg_tablesize = SRP_DEF_SG_TABLESIZE, 3082 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE, 3083 .this_id = -1, 3084 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE, 3085 .shost_groups = srp_host_groups, 3086 .track_queue_depth = 1, 3087 .cmd_size = sizeof(struct srp_request), 3088 }; 3089 3090 static int srp_sdev_count(struct Scsi_Host *host) 3091 { 3092 struct scsi_device *sdev; 3093 int c = 0; 3094 3095 shost_for_each_device(sdev, host) 3096 c++; 3097 3098 return c; 3099 } 3100 3101 /* 3102 * Return values: 3103 * < 0 upon failure. Caller is responsible for SRP target port cleanup. 3104 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port 3105 * removal has been scheduled. 3106 * 0 and target->state != SRP_TARGET_REMOVED upon success. 3107 */ 3108 static int srp_add_target(struct srp_host *host, struct srp_target_port *target) 3109 { 3110 struct srp_rport_identifiers ids; 3111 struct srp_rport *rport; 3112 3113 target->state = SRP_TARGET_SCANNING; 3114 sprintf(target->target_name, "SRP.T10:%016llX", 3115 be64_to_cpu(target->id_ext)); 3116 3117 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent)) 3118 return -ENODEV; 3119 3120 memcpy(ids.port_id, &target->id_ext, 8); 3121 memcpy(ids.port_id + 8, &target->ioc_guid, 8); 3122 ids.roles = SRP_RPORT_ROLE_TARGET; 3123 rport = srp_rport_add(target->scsi_host, &ids); 3124 if (IS_ERR(rport)) { 3125 scsi_remove_host(target->scsi_host); 3126 return PTR_ERR(rport); 3127 } 3128 3129 rport->lld_data = target; 3130 target->rport = rport; 3131 3132 spin_lock(&host->target_lock); 3133 list_add_tail(&target->list, &host->target_list); 3134 spin_unlock(&host->target_lock); 3135 3136 scsi_scan_target(&target->scsi_host->shost_gendev, 3137 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); 3138 3139 if (srp_connected_ch(target) < target->ch_count || 3140 target->qp_in_error) { 3141 shost_printk(KERN_INFO, target->scsi_host, 3142 PFX "SCSI scan failed - removing SCSI host\n"); 3143 srp_queue_remove_work(target); 3144 goto out; 3145 } 3146 3147 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n", 3148 dev_name(&target->scsi_host->shost_gendev), 3149 srp_sdev_count(target->scsi_host)); 3150 3151 spin_lock_irq(&target->lock); 3152 if (target->state == SRP_TARGET_SCANNING) 3153 target->state = SRP_TARGET_LIVE; 3154 spin_unlock_irq(&target->lock); 3155 3156 out: 3157 return 0; 3158 } 3159 3160 static void srp_release_dev(struct device *dev) 3161 { 3162 struct srp_host *host = 3163 container_of(dev, struct srp_host, dev); 3164 3165 kfree(host); 3166 } 3167 3168 static struct attribute *srp_class_attrs[]; 3169 3170 ATTRIBUTE_GROUPS(srp_class); 3171 3172 static struct class srp_class = { 3173 .name = "infiniband_srp", 3174 .dev_groups = srp_class_groups, 3175 .dev_release = srp_release_dev 3176 }; 3177 3178 /** 3179 * srp_conn_unique() - check whether the connection to a target is unique 3180 * @host: SRP host. 3181 * @target: SRP target port. 3182 */ 3183 static bool srp_conn_unique(struct srp_host *host, 3184 struct srp_target_port *target) 3185 { 3186 struct srp_target_port *t; 3187 bool ret = false; 3188 3189 if (target->state == SRP_TARGET_REMOVED) 3190 goto out; 3191 3192 ret = true; 3193 3194 spin_lock(&host->target_lock); 3195 list_for_each_entry(t, &host->target_list, list) { 3196 if (t != target && 3197 target->id_ext == t->id_ext && 3198 target->ioc_guid == t->ioc_guid && 3199 target->initiator_ext == t->initiator_ext) { 3200 ret = false; 3201 break; 3202 } 3203 } 3204 spin_unlock(&host->target_lock); 3205 3206 out: 3207 return ret; 3208 } 3209 3210 /* 3211 * Target ports are added by writing 3212 * 3213 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>, 3214 * pkey=<P_Key>,service_id=<service ID> 3215 * or 3216 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>, 3217 * [src=<IPv4 address>,]dest=<IPv4 address>:<port number> 3218 * 3219 * to the add_target sysfs attribute. 3220 */ 3221 enum { 3222 SRP_OPT_ERR = 0, 3223 SRP_OPT_ID_EXT = 1 << 0, 3224 SRP_OPT_IOC_GUID = 1 << 1, 3225 SRP_OPT_DGID = 1 << 2, 3226 SRP_OPT_PKEY = 1 << 3, 3227 SRP_OPT_SERVICE_ID = 1 << 4, 3228 SRP_OPT_MAX_SECT = 1 << 5, 3229 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, 3230 SRP_OPT_IO_CLASS = 1 << 7, 3231 SRP_OPT_INITIATOR_EXT = 1 << 8, 3232 SRP_OPT_CMD_SG_ENTRIES = 1 << 9, 3233 SRP_OPT_ALLOW_EXT_SG = 1 << 10, 3234 SRP_OPT_SG_TABLESIZE = 1 << 11, 3235 SRP_OPT_COMP_VECTOR = 1 << 12, 3236 SRP_OPT_TL_RETRY_COUNT = 1 << 13, 3237 SRP_OPT_QUEUE_SIZE = 1 << 14, 3238 SRP_OPT_IP_SRC = 1 << 15, 3239 SRP_OPT_IP_DEST = 1 << 16, 3240 SRP_OPT_TARGET_CAN_QUEUE= 1 << 17, 3241 SRP_OPT_MAX_IT_IU_SIZE = 1 << 18, 3242 SRP_OPT_CH_COUNT = 1 << 19, 3243 }; 3244 3245 static unsigned int srp_opt_mandatory[] = { 3246 SRP_OPT_ID_EXT | 3247 SRP_OPT_IOC_GUID | 3248 SRP_OPT_DGID | 3249 SRP_OPT_PKEY | 3250 SRP_OPT_SERVICE_ID, 3251 SRP_OPT_ID_EXT | 3252 SRP_OPT_IOC_GUID | 3253 SRP_OPT_IP_DEST, 3254 }; 3255 3256 static const match_table_t srp_opt_tokens = { 3257 { SRP_OPT_ID_EXT, "id_ext=%s" }, 3258 { SRP_OPT_IOC_GUID, "ioc_guid=%s" }, 3259 { SRP_OPT_DGID, "dgid=%s" }, 3260 { SRP_OPT_PKEY, "pkey=%x" }, 3261 { SRP_OPT_SERVICE_ID, "service_id=%s" }, 3262 { SRP_OPT_MAX_SECT, "max_sect=%d" }, 3263 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, 3264 { SRP_OPT_TARGET_CAN_QUEUE, "target_can_queue=%d" }, 3265 { SRP_OPT_IO_CLASS, "io_class=%x" }, 3266 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, 3267 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, 3268 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, 3269 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, 3270 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" }, 3271 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" }, 3272 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" }, 3273 { SRP_OPT_IP_SRC, "src=%s" }, 3274 { SRP_OPT_IP_DEST, "dest=%s" }, 3275 { SRP_OPT_MAX_IT_IU_SIZE, "max_it_iu_size=%d" }, 3276 { SRP_OPT_CH_COUNT, "ch_count=%u", }, 3277 { SRP_OPT_ERR, NULL } 3278 }; 3279 3280 /** 3281 * srp_parse_in - parse an IP address and port number combination 3282 * @net: [in] Network namespace. 3283 * @sa: [out] Address family, IP address and port number. 3284 * @addr_port_str: [in] IP address and port number. 3285 * @has_port: [out] Whether or not @addr_port_str includes a port number. 3286 * 3287 * Parse the following address formats: 3288 * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5. 3289 * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5. 3290 */ 3291 static int srp_parse_in(struct net *net, struct sockaddr_storage *sa, 3292 const char *addr_port_str, bool *has_port) 3293 { 3294 char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL); 3295 char *port_str; 3296 int ret; 3297 3298 if (!addr) 3299 return -ENOMEM; 3300 port_str = strrchr(addr, ':'); 3301 if (port_str && strchr(port_str, ']')) 3302 port_str = NULL; 3303 if (port_str) 3304 *port_str++ = '\0'; 3305 if (has_port) 3306 *has_port = port_str != NULL; 3307 ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa); 3308 if (ret && addr[0]) { 3309 addr_end = addr + strlen(addr) - 1; 3310 if (addr[0] == '[' && *addr_end == ']') { 3311 *addr_end = '\0'; 3312 ret = inet_pton_with_scope(net, AF_INET6, addr + 1, 3313 port_str, sa); 3314 } 3315 } 3316 kfree(addr); 3317 pr_debug("%s -> %pISpfsc\n", addr_port_str, sa); 3318 return ret; 3319 } 3320 3321 static int srp_parse_options(struct net *net, const char *buf, 3322 struct srp_target_port *target) 3323 { 3324 char *options, *sep_opt; 3325 char *p; 3326 substring_t args[MAX_OPT_ARGS]; 3327 unsigned long long ull; 3328 bool has_port; 3329 int opt_mask = 0; 3330 int token; 3331 int ret = -EINVAL; 3332 int i; 3333 3334 options = kstrdup(buf, GFP_KERNEL); 3335 if (!options) 3336 return -ENOMEM; 3337 3338 sep_opt = options; 3339 while ((p = strsep(&sep_opt, ",\n")) != NULL) { 3340 if (!*p) 3341 continue; 3342 3343 token = match_token(p, srp_opt_tokens, args); 3344 opt_mask |= token; 3345 3346 switch (token) { 3347 case SRP_OPT_ID_EXT: 3348 p = match_strdup(args); 3349 if (!p) { 3350 ret = -ENOMEM; 3351 goto out; 3352 } 3353 ret = kstrtoull(p, 16, &ull); 3354 if (ret) { 3355 pr_warn("invalid id_ext parameter '%s'\n", p); 3356 kfree(p); 3357 goto out; 3358 } 3359 target->id_ext = cpu_to_be64(ull); 3360 kfree(p); 3361 break; 3362 3363 case SRP_OPT_IOC_GUID: 3364 p = match_strdup(args); 3365 if (!p) { 3366 ret = -ENOMEM; 3367 goto out; 3368 } 3369 ret = kstrtoull(p, 16, &ull); 3370 if (ret) { 3371 pr_warn("invalid ioc_guid parameter '%s'\n", p); 3372 kfree(p); 3373 goto out; 3374 } 3375 target->ioc_guid = cpu_to_be64(ull); 3376 kfree(p); 3377 break; 3378 3379 case SRP_OPT_DGID: 3380 p = match_strdup(args); 3381 if (!p) { 3382 ret = -ENOMEM; 3383 goto out; 3384 } 3385 if (strlen(p) != 32) { 3386 pr_warn("bad dest GID parameter '%s'\n", p); 3387 kfree(p); 3388 goto out; 3389 } 3390 3391 ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16); 3392 kfree(p); 3393 if (ret < 0) 3394 goto out; 3395 break; 3396 3397 case SRP_OPT_PKEY: 3398 ret = match_hex(args, &token); 3399 if (ret) { 3400 pr_warn("bad P_Key parameter '%s'\n", p); 3401 goto out; 3402 } 3403 target->ib_cm.pkey = cpu_to_be16(token); 3404 break; 3405 3406 case SRP_OPT_SERVICE_ID: 3407 p = match_strdup(args); 3408 if (!p) { 3409 ret = -ENOMEM; 3410 goto out; 3411 } 3412 ret = kstrtoull(p, 16, &ull); 3413 if (ret) { 3414 pr_warn("bad service_id parameter '%s'\n", p); 3415 kfree(p); 3416 goto out; 3417 } 3418 target->ib_cm.service_id = cpu_to_be64(ull); 3419 kfree(p); 3420 break; 3421 3422 case SRP_OPT_IP_SRC: 3423 p = match_strdup(args); 3424 if (!p) { 3425 ret = -ENOMEM; 3426 goto out; 3427 } 3428 ret = srp_parse_in(net, &target->rdma_cm.src.ss, p, 3429 NULL); 3430 if (ret < 0) { 3431 pr_warn("bad source parameter '%s'\n", p); 3432 kfree(p); 3433 goto out; 3434 } 3435 target->rdma_cm.src_specified = true; 3436 kfree(p); 3437 break; 3438 3439 case SRP_OPT_IP_DEST: 3440 p = match_strdup(args); 3441 if (!p) { 3442 ret = -ENOMEM; 3443 goto out; 3444 } 3445 ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p, 3446 &has_port); 3447 if (!has_port) 3448 ret = -EINVAL; 3449 if (ret < 0) { 3450 pr_warn("bad dest parameter '%s'\n", p); 3451 kfree(p); 3452 goto out; 3453 } 3454 target->using_rdma_cm = true; 3455 kfree(p); 3456 break; 3457 3458 case SRP_OPT_MAX_SECT: 3459 ret = match_int(args, &token); 3460 if (ret) { 3461 pr_warn("bad max sect parameter '%s'\n", p); 3462 goto out; 3463 } 3464 target->scsi_host->max_sectors = token; 3465 break; 3466 3467 case SRP_OPT_QUEUE_SIZE: 3468 ret = match_int(args, &token); 3469 if (ret) { 3470 pr_warn("match_int() failed for queue_size parameter '%s', Error %d\n", 3471 p, ret); 3472 goto out; 3473 } 3474 if (token < 1) { 3475 pr_warn("bad queue_size parameter '%s'\n", p); 3476 ret = -EINVAL; 3477 goto out; 3478 } 3479 target->scsi_host->can_queue = token; 3480 target->queue_size = token + SRP_RSP_SQ_SIZE + 3481 SRP_TSK_MGMT_SQ_SIZE; 3482 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3483 target->scsi_host->cmd_per_lun = token; 3484 break; 3485 3486 case SRP_OPT_MAX_CMD_PER_LUN: 3487 ret = match_int(args, &token); 3488 if (ret) { 3489 pr_warn("match_int() failed for max cmd_per_lun parameter '%s', Error %d\n", 3490 p, ret); 3491 goto out; 3492 } 3493 if (token < 1) { 3494 pr_warn("bad max cmd_per_lun parameter '%s'\n", 3495 p); 3496 ret = -EINVAL; 3497 goto out; 3498 } 3499 target->scsi_host->cmd_per_lun = token; 3500 break; 3501 3502 case SRP_OPT_TARGET_CAN_QUEUE: 3503 ret = match_int(args, &token); 3504 if (ret) { 3505 pr_warn("match_int() failed for max target_can_queue parameter '%s', Error %d\n", 3506 p, ret); 3507 goto out; 3508 } 3509 if (token < 1) { 3510 pr_warn("bad max target_can_queue parameter '%s'\n", 3511 p); 3512 ret = -EINVAL; 3513 goto out; 3514 } 3515 target->target_can_queue = token; 3516 break; 3517 3518 case SRP_OPT_IO_CLASS: 3519 ret = match_hex(args, &token); 3520 if (ret) { 3521 pr_warn("bad IO class parameter '%s'\n", p); 3522 goto out; 3523 } 3524 if (token != SRP_REV10_IB_IO_CLASS && 3525 token != SRP_REV16A_IB_IO_CLASS) { 3526 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n", 3527 token, SRP_REV10_IB_IO_CLASS, 3528 SRP_REV16A_IB_IO_CLASS); 3529 ret = -EINVAL; 3530 goto out; 3531 } 3532 target->io_class = token; 3533 break; 3534 3535 case SRP_OPT_INITIATOR_EXT: 3536 p = match_strdup(args); 3537 if (!p) { 3538 ret = -ENOMEM; 3539 goto out; 3540 } 3541 ret = kstrtoull(p, 16, &ull); 3542 if (ret) { 3543 pr_warn("bad initiator_ext value '%s'\n", p); 3544 kfree(p); 3545 goto out; 3546 } 3547 target->initiator_ext = cpu_to_be64(ull); 3548 kfree(p); 3549 break; 3550 3551 case SRP_OPT_CMD_SG_ENTRIES: 3552 ret = match_int(args, &token); 3553 if (ret) { 3554 pr_warn("match_int() failed for max cmd_sg_entries parameter '%s', Error %d\n", 3555 p, ret); 3556 goto out; 3557 } 3558 if (token < 1 || token > 255) { 3559 pr_warn("bad max cmd_sg_entries parameter '%s'\n", 3560 p); 3561 ret = -EINVAL; 3562 goto out; 3563 } 3564 target->cmd_sg_cnt = token; 3565 break; 3566 3567 case SRP_OPT_ALLOW_EXT_SG: 3568 ret = match_int(args, &token); 3569 if (ret) { 3570 pr_warn("bad allow_ext_sg parameter '%s'\n", p); 3571 goto out; 3572 } 3573 target->allow_ext_sg = !!token; 3574 break; 3575 3576 case SRP_OPT_SG_TABLESIZE: 3577 ret = match_int(args, &token); 3578 if (ret) { 3579 pr_warn("match_int() failed for max sg_tablesize parameter '%s', Error %d\n", 3580 p, ret); 3581 goto out; 3582 } 3583 if (token < 1 || token > SG_MAX_SEGMENTS) { 3584 pr_warn("bad max sg_tablesize parameter '%s'\n", 3585 p); 3586 ret = -EINVAL; 3587 goto out; 3588 } 3589 target->sg_tablesize = token; 3590 break; 3591 3592 case SRP_OPT_COMP_VECTOR: 3593 ret = match_int(args, &token); 3594 if (ret) { 3595 pr_warn("match_int() failed for comp_vector parameter '%s', Error %d\n", 3596 p, ret); 3597 goto out; 3598 } 3599 if (token < 0) { 3600 pr_warn("bad comp_vector parameter '%s'\n", p); 3601 ret = -EINVAL; 3602 goto out; 3603 } 3604 target->comp_vector = token; 3605 break; 3606 3607 case SRP_OPT_TL_RETRY_COUNT: 3608 ret = match_int(args, &token); 3609 if (ret) { 3610 pr_warn("match_int() failed for tl_retry_count parameter '%s', Error %d\n", 3611 p, ret); 3612 goto out; 3613 } 3614 if (token < 2 || token > 7) { 3615 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n", 3616 p); 3617 ret = -EINVAL; 3618 goto out; 3619 } 3620 target->tl_retry_count = token; 3621 break; 3622 3623 case SRP_OPT_MAX_IT_IU_SIZE: 3624 ret = match_int(args, &token); 3625 if (ret) { 3626 pr_warn("match_int() failed for max it_iu_size parameter '%s', Error %d\n", 3627 p, ret); 3628 goto out; 3629 } 3630 if (token < 0) { 3631 pr_warn("bad maximum initiator to target IU size '%s'\n", p); 3632 ret = -EINVAL; 3633 goto out; 3634 } 3635 target->max_it_iu_size = token; 3636 break; 3637 3638 case SRP_OPT_CH_COUNT: 3639 ret = match_int(args, &token); 3640 if (ret) { 3641 pr_warn("match_int() failed for channel count parameter '%s', Error %d\n", 3642 p, ret); 3643 goto out; 3644 } 3645 if (token < 1) { 3646 pr_warn("bad channel count %s\n", p); 3647 ret = -EINVAL; 3648 goto out; 3649 } 3650 target->ch_count = token; 3651 break; 3652 3653 default: 3654 pr_warn("unknown parameter or missing value '%s' in target creation request\n", 3655 p); 3656 ret = -EINVAL; 3657 goto out; 3658 } 3659 } 3660 3661 for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) { 3662 if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) { 3663 ret = 0; 3664 break; 3665 } 3666 } 3667 if (ret) 3668 pr_warn("target creation request is missing one or more parameters\n"); 3669 3670 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue 3671 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3672 pr_warn("cmd_per_lun = %d > queue_size = %d\n", 3673 target->scsi_host->cmd_per_lun, 3674 target->scsi_host->can_queue); 3675 3676 out: 3677 kfree(options); 3678 return ret; 3679 } 3680 3681 static ssize_t add_target_store(struct device *dev, 3682 struct device_attribute *attr, const char *buf, 3683 size_t count) 3684 { 3685 struct srp_host *host = 3686 container_of(dev, struct srp_host, dev); 3687 struct Scsi_Host *target_host; 3688 struct srp_target_port *target; 3689 struct srp_rdma_ch *ch; 3690 struct srp_device *srp_dev = host->srp_dev; 3691 struct ib_device *ibdev = srp_dev->dev; 3692 int ret, i, ch_idx; 3693 unsigned int max_sectors_per_mr, mr_per_cmd = 0; 3694 bool multich = false; 3695 uint32_t max_iu_len; 3696 3697 target_host = scsi_host_alloc(&srp_template, 3698 sizeof (struct srp_target_port)); 3699 if (!target_host) 3700 return -ENOMEM; 3701 3702 target_host->transportt = ib_srp_transport_template; 3703 target_host->max_channel = 0; 3704 target_host->max_id = 1; 3705 target_host->max_lun = -1LL; 3706 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; 3707 target_host->max_segment_size = ib_dma_max_seg_size(ibdev); 3708 3709 if (!(ibdev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)) 3710 target_host->virt_boundary_mask = ~srp_dev->mr_page_mask; 3711 3712 target = host_to_target(target_host); 3713 3714 target->net = kobj_ns_grab_current(KOBJ_NS_TYPE_NET); 3715 target->io_class = SRP_REV16A_IB_IO_CLASS; 3716 target->scsi_host = target_host; 3717 target->srp_host = host; 3718 target->lkey = host->srp_dev->pd->local_dma_lkey; 3719 target->global_rkey = host->srp_dev->global_rkey; 3720 target->cmd_sg_cnt = cmd_sg_entries; 3721 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries; 3722 target->allow_ext_sg = allow_ext_sg; 3723 target->tl_retry_count = 7; 3724 target->queue_size = SRP_DEFAULT_QUEUE_SIZE; 3725 3726 /* 3727 * Avoid that the SCSI host can be removed by srp_remove_target() 3728 * before this function returns. 3729 */ 3730 scsi_host_get(target->scsi_host); 3731 3732 ret = mutex_lock_interruptible(&host->add_target_mutex); 3733 if (ret < 0) 3734 goto put; 3735 3736 ret = srp_parse_options(target->net, buf, target); 3737 if (ret) 3738 goto out; 3739 3740 if (!srp_conn_unique(target->srp_host, target)) { 3741 if (target->using_rdma_cm) { 3742 shost_printk(KERN_INFO, target->scsi_host, 3743 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n", 3744 be64_to_cpu(target->id_ext), 3745 be64_to_cpu(target->ioc_guid), 3746 &target->rdma_cm.dst); 3747 } else { 3748 shost_printk(KERN_INFO, target->scsi_host, 3749 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n", 3750 be64_to_cpu(target->id_ext), 3751 be64_to_cpu(target->ioc_guid), 3752 be64_to_cpu(target->initiator_ext)); 3753 } 3754 ret = -EEXIST; 3755 goto out; 3756 } 3757 3758 if (!srp_dev->has_fr && !target->allow_ext_sg && 3759 target->cmd_sg_cnt < target->sg_tablesize) { 3760 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n"); 3761 target->sg_tablesize = target->cmd_sg_cnt; 3762 } 3763 3764 if (srp_dev->use_fast_reg) { 3765 bool gaps_reg = ibdev->attrs.kernel_cap_flags & 3766 IBK_SG_GAPS_REG; 3767 3768 max_sectors_per_mr = srp_dev->max_pages_per_mr << 3769 (ilog2(srp_dev->mr_page_size) - 9); 3770 if (!gaps_reg) { 3771 /* 3772 * FR can only map one HCA page per entry. If the start 3773 * address is not aligned on a HCA page boundary two 3774 * entries will be used for the head and the tail 3775 * although these two entries combined contain at most 3776 * one HCA page of data. Hence the "+ 1" in the 3777 * calculation below. 3778 * 3779 * The indirect data buffer descriptor is contiguous 3780 * so the memory for that buffer will only be 3781 * registered if register_always is true. Hence add 3782 * one to mr_per_cmd if register_always has been set. 3783 */ 3784 mr_per_cmd = register_always + 3785 (target->scsi_host->max_sectors + 1 + 3786 max_sectors_per_mr - 1) / max_sectors_per_mr; 3787 } else { 3788 mr_per_cmd = register_always + 3789 (target->sg_tablesize + 3790 srp_dev->max_pages_per_mr - 1) / 3791 srp_dev->max_pages_per_mr; 3792 } 3793 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n", 3794 target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size, 3795 max_sectors_per_mr, mr_per_cmd); 3796 } 3797 3798 target_host->sg_tablesize = target->sg_tablesize; 3799 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd; 3800 target->mr_per_cmd = mr_per_cmd; 3801 target->indirect_size = target->sg_tablesize * 3802 sizeof (struct srp_direct_buf); 3803 max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 3804 srp_use_imm_data, 3805 target->max_it_iu_size); 3806 3807 INIT_WORK(&target->tl_err_work, srp_tl_err_work); 3808 INIT_WORK(&target->remove_work, srp_remove_work); 3809 spin_lock_init(&target->lock); 3810 ret = rdma_query_gid(ibdev, host->port, 0, &target->sgid); 3811 if (ret) 3812 goto out; 3813 3814 ret = -ENOMEM; 3815 if (target->ch_count == 0) { 3816 target->ch_count = 3817 min(ch_count ?: 3818 max(4 * num_online_nodes(), 3819 ibdev->num_comp_vectors), 3820 num_online_cpus()); 3821 } 3822 3823 target->ch = kcalloc(target->ch_count, sizeof(*target->ch), 3824 GFP_KERNEL); 3825 if (!target->ch) 3826 goto out; 3827 3828 for (ch_idx = 0; ch_idx < target->ch_count; ++ch_idx) { 3829 ch = &target->ch[ch_idx]; 3830 ch->target = target; 3831 ch->comp_vector = ch_idx % ibdev->num_comp_vectors; 3832 spin_lock_init(&ch->lock); 3833 INIT_LIST_HEAD(&ch->free_tx); 3834 ret = srp_new_cm_id(ch); 3835 if (ret) 3836 goto err_disconnect; 3837 3838 ret = srp_create_ch_ib(ch); 3839 if (ret) 3840 goto err_disconnect; 3841 3842 ret = srp_connect_ch(ch, max_iu_len, multich); 3843 if (ret) { 3844 char dst[64]; 3845 3846 if (target->using_rdma_cm) 3847 snprintf(dst, sizeof(dst), "%pIS", 3848 &target->rdma_cm.dst); 3849 else 3850 snprintf(dst, sizeof(dst), "%pI6", 3851 target->ib_cm.orig_dgid.raw); 3852 shost_printk(KERN_ERR, target->scsi_host, 3853 PFX "Connection %d/%d to %s failed\n", 3854 ch_idx, 3855 target->ch_count, dst); 3856 if (ch_idx == 0) { 3857 goto free_ch; 3858 } else { 3859 srp_free_ch_ib(target, ch); 3860 target->ch_count = ch - target->ch; 3861 goto connected; 3862 } 3863 } 3864 multich = true; 3865 } 3866 3867 connected: 3868 target->scsi_host->nr_hw_queues = target->ch_count; 3869 3870 ret = srp_add_target(host, target); 3871 if (ret) 3872 goto err_disconnect; 3873 3874 if (target->state != SRP_TARGET_REMOVED) { 3875 if (target->using_rdma_cm) { 3876 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3877 "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n", 3878 be64_to_cpu(target->id_ext), 3879 be64_to_cpu(target->ioc_guid), 3880 target->sgid.raw, &target->rdma_cm.dst); 3881 } else { 3882 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3883 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n", 3884 be64_to_cpu(target->id_ext), 3885 be64_to_cpu(target->ioc_guid), 3886 be16_to_cpu(target->ib_cm.pkey), 3887 be64_to_cpu(target->ib_cm.service_id), 3888 target->sgid.raw, 3889 target->ib_cm.orig_dgid.raw); 3890 } 3891 } 3892 3893 ret = count; 3894 3895 out: 3896 mutex_unlock(&host->add_target_mutex); 3897 3898 put: 3899 scsi_host_put(target->scsi_host); 3900 if (ret < 0) { 3901 /* 3902 * If a call to srp_remove_target() has not been scheduled, 3903 * drop the network namespace reference now that was obtained 3904 * earlier in this function. 3905 */ 3906 if (target->state != SRP_TARGET_REMOVED) 3907 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 3908 scsi_host_put(target->scsi_host); 3909 } 3910 3911 return ret; 3912 3913 err_disconnect: 3914 srp_disconnect_target(target); 3915 3916 free_ch: 3917 for (i = 0; i < target->ch_count; i++) { 3918 ch = &target->ch[i]; 3919 srp_free_ch_ib(target, ch); 3920 } 3921 3922 kfree(target->ch); 3923 goto out; 3924 } 3925 3926 static DEVICE_ATTR_WO(add_target); 3927 3928 static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr, 3929 char *buf) 3930 { 3931 struct srp_host *host = container_of(dev, struct srp_host, dev); 3932 3933 return sysfs_emit(buf, "%s\n", dev_name(&host->srp_dev->dev->dev)); 3934 } 3935 3936 static DEVICE_ATTR_RO(ibdev); 3937 3938 static ssize_t port_show(struct device *dev, struct device_attribute *attr, 3939 char *buf) 3940 { 3941 struct srp_host *host = container_of(dev, struct srp_host, dev); 3942 3943 return sysfs_emit(buf, "%u\n", host->port); 3944 } 3945 3946 static DEVICE_ATTR_RO(port); 3947 3948 static struct attribute *srp_class_attrs[] = { 3949 &dev_attr_add_target.attr, 3950 &dev_attr_ibdev.attr, 3951 &dev_attr_port.attr, 3952 NULL 3953 }; 3954 3955 static struct srp_host *srp_add_port(struct srp_device *device, u32 port) 3956 { 3957 struct srp_host *host; 3958 3959 host = kzalloc(sizeof *host, GFP_KERNEL); 3960 if (!host) 3961 return NULL; 3962 3963 INIT_LIST_HEAD(&host->target_list); 3964 spin_lock_init(&host->target_lock); 3965 mutex_init(&host->add_target_mutex); 3966 host->srp_dev = device; 3967 host->port = port; 3968 3969 device_initialize(&host->dev); 3970 host->dev.class = &srp_class; 3971 host->dev.parent = device->dev->dev.parent; 3972 if (dev_set_name(&host->dev, "srp-%s-%u", dev_name(&device->dev->dev), 3973 port)) 3974 goto put_host; 3975 if (device_add(&host->dev)) 3976 goto put_host; 3977 3978 return host; 3979 3980 put_host: 3981 device_del(&host->dev); 3982 put_device(&host->dev); 3983 return NULL; 3984 } 3985 3986 static void srp_rename_dev(struct ib_device *device, void *client_data) 3987 { 3988 struct srp_device *srp_dev = client_data; 3989 struct srp_host *host, *tmp_host; 3990 3991 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 3992 char name[IB_DEVICE_NAME_MAX + 8]; 3993 3994 snprintf(name, sizeof(name), "srp-%s-%u", 3995 dev_name(&device->dev), host->port); 3996 device_rename(&host->dev, name); 3997 } 3998 } 3999 4000 static int srp_add_one(struct ib_device *device) 4001 { 4002 struct srp_device *srp_dev; 4003 struct ib_device_attr *attr = &device->attrs; 4004 struct srp_host *host; 4005 int mr_page_shift; 4006 u32 p; 4007 u64 max_pages_per_mr; 4008 unsigned int flags = 0; 4009 4010 srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL); 4011 if (!srp_dev) 4012 return -ENOMEM; 4013 4014 /* 4015 * Use the smallest page size supported by the HCA, down to a 4016 * minimum of 4096 bytes. We're unlikely to build large sglists 4017 * out of smaller entries. 4018 */ 4019 mr_page_shift = max(12, ffs(attr->page_size_cap) - 1); 4020 srp_dev->mr_page_size = 1 << mr_page_shift; 4021 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1); 4022 max_pages_per_mr = attr->max_mr_size; 4023 do_div(max_pages_per_mr, srp_dev->mr_page_size); 4024 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__, 4025 attr->max_mr_size, srp_dev->mr_page_size, 4026 max_pages_per_mr, SRP_MAX_PAGES_PER_MR); 4027 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR, 4028 max_pages_per_mr); 4029 4030 srp_dev->has_fr = (attr->device_cap_flags & 4031 IB_DEVICE_MEM_MGT_EXTENSIONS); 4032 if (!never_register && !srp_dev->has_fr) 4033 dev_warn(&device->dev, "FR is not supported\n"); 4034 else if (!never_register && 4035 attr->max_mr_size >= 2 * srp_dev->mr_page_size) 4036 srp_dev->use_fast_reg = srp_dev->has_fr; 4037 4038 if (never_register || !register_always || !srp_dev->has_fr) 4039 flags |= IB_PD_UNSAFE_GLOBAL_RKEY; 4040 4041 if (srp_dev->use_fast_reg) { 4042 srp_dev->max_pages_per_mr = 4043 min_t(u32, srp_dev->max_pages_per_mr, 4044 attr->max_fast_reg_page_list_len); 4045 } 4046 srp_dev->mr_max_size = srp_dev->mr_page_size * 4047 srp_dev->max_pages_per_mr; 4048 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n", 4049 dev_name(&device->dev), mr_page_shift, attr->max_mr_size, 4050 attr->max_fast_reg_page_list_len, 4051 srp_dev->max_pages_per_mr, srp_dev->mr_max_size); 4052 4053 INIT_LIST_HEAD(&srp_dev->dev_list); 4054 4055 srp_dev->dev = device; 4056 srp_dev->pd = ib_alloc_pd(device, flags); 4057 if (IS_ERR(srp_dev->pd)) { 4058 int ret = PTR_ERR(srp_dev->pd); 4059 4060 kfree(srp_dev); 4061 return ret; 4062 } 4063 4064 if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) { 4065 srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey; 4066 WARN_ON_ONCE(srp_dev->global_rkey == 0); 4067 } 4068 4069 rdma_for_each_port (device, p) { 4070 host = srp_add_port(srp_dev, p); 4071 if (host) 4072 list_add_tail(&host->list, &srp_dev->dev_list); 4073 } 4074 4075 ib_set_client_data(device, &srp_client, srp_dev); 4076 return 0; 4077 } 4078 4079 static void srp_remove_one(struct ib_device *device, void *client_data) 4080 { 4081 struct srp_device *srp_dev; 4082 struct srp_host *host, *tmp_host; 4083 struct srp_target_port *target; 4084 4085 srp_dev = client_data; 4086 4087 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 4088 /* 4089 * Remove the add_target sysfs entry so that no new target ports 4090 * can be created. 4091 */ 4092 device_del(&host->dev); 4093 4094 /* 4095 * Remove all target ports. 4096 */ 4097 spin_lock(&host->target_lock); 4098 list_for_each_entry(target, &host->target_list, list) 4099 srp_queue_remove_work(target); 4100 spin_unlock(&host->target_lock); 4101 4102 /* 4103 * srp_queue_remove_work() queues a call to 4104 * srp_remove_target(). The latter function cancels 4105 * target->tl_err_work so waiting for the remove works to 4106 * finish is sufficient. 4107 */ 4108 flush_workqueue(srp_remove_wq); 4109 4110 put_device(&host->dev); 4111 } 4112 4113 ib_dealloc_pd(srp_dev->pd); 4114 4115 kfree(srp_dev); 4116 } 4117 4118 static struct srp_function_template ib_srp_transport_functions = { 4119 .has_rport_state = true, 4120 .reset_timer_if_blocked = true, 4121 .reconnect_delay = &srp_reconnect_delay, 4122 .fast_io_fail_tmo = &srp_fast_io_fail_tmo, 4123 .dev_loss_tmo = &srp_dev_loss_tmo, 4124 .reconnect = srp_rport_reconnect, 4125 .rport_delete = srp_rport_delete, 4126 .terminate_rport_io = srp_terminate_io, 4127 }; 4128 4129 static int __init srp_init_module(void) 4130 { 4131 int ret; 4132 4133 BUILD_BUG_ON(sizeof(struct srp_aer_req) != 36); 4134 BUILD_BUG_ON(sizeof(struct srp_cmd) != 48); 4135 BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 4); 4136 BUILD_BUG_ON(sizeof(struct srp_indirect_buf) != 20); 4137 BUILD_BUG_ON(sizeof(struct srp_login_req) != 64); 4138 BUILD_BUG_ON(sizeof(struct srp_login_req_rdma) != 56); 4139 BUILD_BUG_ON(sizeof(struct srp_rsp) != 36); 4140 4141 if (srp_sg_tablesize) { 4142 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); 4143 if (!cmd_sg_entries) 4144 cmd_sg_entries = srp_sg_tablesize; 4145 } 4146 4147 if (!cmd_sg_entries) 4148 cmd_sg_entries = SRP_DEF_SG_TABLESIZE; 4149 4150 if (cmd_sg_entries > 255) { 4151 pr_warn("Clamping cmd_sg_entries to 255\n"); 4152 cmd_sg_entries = 255; 4153 } 4154 4155 if (!indirect_sg_entries) 4156 indirect_sg_entries = cmd_sg_entries; 4157 else if (indirect_sg_entries < cmd_sg_entries) { 4158 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n", 4159 cmd_sg_entries); 4160 indirect_sg_entries = cmd_sg_entries; 4161 } 4162 4163 if (indirect_sg_entries > SG_MAX_SEGMENTS) { 4164 pr_warn("Clamping indirect_sg_entries to %u\n", 4165 SG_MAX_SEGMENTS); 4166 indirect_sg_entries = SG_MAX_SEGMENTS; 4167 } 4168 4169 srp_remove_wq = create_workqueue("srp_remove"); 4170 if (!srp_remove_wq) { 4171 ret = -ENOMEM; 4172 goto out; 4173 } 4174 4175 ret = -ENOMEM; 4176 ib_srp_transport_template = 4177 srp_attach_transport(&ib_srp_transport_functions); 4178 if (!ib_srp_transport_template) 4179 goto destroy_wq; 4180 4181 ret = class_register(&srp_class); 4182 if (ret) { 4183 pr_err("couldn't register class infiniband_srp\n"); 4184 goto release_tr; 4185 } 4186 4187 ib_sa_register_client(&srp_sa_client); 4188 4189 ret = ib_register_client(&srp_client); 4190 if (ret) { 4191 pr_err("couldn't register IB client\n"); 4192 goto unreg_sa; 4193 } 4194 4195 out: 4196 return ret; 4197 4198 unreg_sa: 4199 ib_sa_unregister_client(&srp_sa_client); 4200 class_unregister(&srp_class); 4201 4202 release_tr: 4203 srp_release_transport(ib_srp_transport_template); 4204 4205 destroy_wq: 4206 destroy_workqueue(srp_remove_wq); 4207 goto out; 4208 } 4209 4210 static void __exit srp_cleanup_module(void) 4211 { 4212 ib_unregister_client(&srp_client); 4213 ib_sa_unregister_client(&srp_sa_client); 4214 class_unregister(&srp_class); 4215 srp_release_transport(ib_srp_transport_template); 4216 destroy_workqueue(srp_remove_wq); 4217 } 4218 4219 module_init(srp_init_module); 4220 module_exit(srp_cleanup_module); 4221