1 /* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/module.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/err.h> 39 #include <linux/string.h> 40 #include <linux/parser.h> 41 #include <linux/random.h> 42 #include <linux/jiffies.h> 43 #include <linux/lockdep.h> 44 #include <linux/inet.h> 45 #include <rdma/ib_cache.h> 46 47 #include <linux/atomic.h> 48 49 #include <scsi/scsi.h> 50 #include <scsi/scsi_device.h> 51 #include <scsi/scsi_dbg.h> 52 #include <scsi/scsi_tcq.h> 53 #include <scsi/srp.h> 54 #include <scsi/scsi_transport_srp.h> 55 56 #include "ib_srp.h" 57 58 #define DRV_NAME "ib_srp" 59 #define PFX DRV_NAME ": " 60 61 MODULE_AUTHOR("Roland Dreier"); 62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); 63 MODULE_LICENSE("Dual BSD/GPL"); 64 65 #if !defined(CONFIG_DYNAMIC_DEBUG) 66 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) 67 #define DYNAMIC_DEBUG_BRANCH(descriptor) false 68 #endif 69 70 static unsigned int srp_sg_tablesize; 71 static unsigned int cmd_sg_entries; 72 static unsigned int indirect_sg_entries; 73 static bool allow_ext_sg; 74 static bool register_always = true; 75 static bool never_register; 76 static int topspin_workarounds = 1; 77 78 module_param(srp_sg_tablesize, uint, 0444); 79 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); 80 81 module_param(cmd_sg_entries, uint, 0444); 82 MODULE_PARM_DESC(cmd_sg_entries, 83 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); 84 85 module_param(indirect_sg_entries, uint, 0444); 86 MODULE_PARM_DESC(indirect_sg_entries, 87 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")"); 88 89 module_param(allow_ext_sg, bool, 0444); 90 MODULE_PARM_DESC(allow_ext_sg, 91 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)"); 92 93 module_param(topspin_workarounds, int, 0444); 94 MODULE_PARM_DESC(topspin_workarounds, 95 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0"); 96 97 module_param(register_always, bool, 0444); 98 MODULE_PARM_DESC(register_always, 99 "Use memory registration even for contiguous memory regions"); 100 101 module_param(never_register, bool, 0444); 102 MODULE_PARM_DESC(never_register, "Never register memory"); 103 104 static const struct kernel_param_ops srp_tmo_ops; 105 106 static int srp_reconnect_delay = 10; 107 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 108 S_IRUGO | S_IWUSR); 109 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts"); 110 111 static int srp_fast_io_fail_tmo = 15; 112 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo, 113 S_IRUGO | S_IWUSR); 114 MODULE_PARM_DESC(fast_io_fail_tmo, 115 "Number of seconds between the observation of a transport" 116 " layer error and failing all I/O. \"off\" means that this" 117 " functionality is disabled."); 118 119 static int srp_dev_loss_tmo = 600; 120 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo, 121 S_IRUGO | S_IWUSR); 122 MODULE_PARM_DESC(dev_loss_tmo, 123 "Maximum number of seconds that the SRP transport should" 124 " insulate transport layer errors. After this time has been" 125 " exceeded the SCSI host is removed. Should be" 126 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT) 127 " if fast_io_fail_tmo has not been set. \"off\" means that" 128 " this functionality is disabled."); 129 130 static bool srp_use_imm_data = true; 131 module_param_named(use_imm_data, srp_use_imm_data, bool, 0644); 132 MODULE_PARM_DESC(use_imm_data, 133 "Whether or not to request permission to use immediate data during SRP login."); 134 135 static unsigned int srp_max_imm_data = 8 * 1024; 136 module_param_named(max_imm_data, srp_max_imm_data, uint, 0644); 137 MODULE_PARM_DESC(max_imm_data, "Maximum immediate data size."); 138 139 static unsigned ch_count; 140 module_param(ch_count, uint, 0444); 141 MODULE_PARM_DESC(ch_count, 142 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA."); 143 144 static int srp_add_one(struct ib_device *device); 145 static void srp_remove_one(struct ib_device *device, void *client_data); 146 static void srp_rename_dev(struct ib_device *device, void *client_data); 147 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc); 148 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 149 const char *opname); 150 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 151 const struct ib_cm_event *event); 152 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 153 struct rdma_cm_event *event); 154 155 static struct scsi_transport_template *ib_srp_transport_template; 156 static struct workqueue_struct *srp_remove_wq; 157 158 static struct ib_client srp_client = { 159 .name = "srp", 160 .add = srp_add_one, 161 .remove = srp_remove_one, 162 .rename = srp_rename_dev 163 }; 164 165 static struct ib_sa_client srp_sa_client; 166 167 static int srp_tmo_get(char *buffer, const struct kernel_param *kp) 168 { 169 int tmo = *(int *)kp->arg; 170 171 if (tmo >= 0) 172 return sysfs_emit(buffer, "%d\n", tmo); 173 else 174 return sysfs_emit(buffer, "off\n"); 175 } 176 177 static int srp_tmo_set(const char *val, const struct kernel_param *kp) 178 { 179 int tmo, res; 180 181 res = srp_parse_tmo(&tmo, val); 182 if (res) 183 goto out; 184 185 if (kp->arg == &srp_reconnect_delay) 186 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo, 187 srp_dev_loss_tmo); 188 else if (kp->arg == &srp_fast_io_fail_tmo) 189 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo); 190 else 191 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo, 192 tmo); 193 if (res) 194 goto out; 195 *(int *)kp->arg = tmo; 196 197 out: 198 return res; 199 } 200 201 static const struct kernel_param_ops srp_tmo_ops = { 202 .get = srp_tmo_get, 203 .set = srp_tmo_set, 204 }; 205 206 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) 207 { 208 return (struct srp_target_port *) host->hostdata; 209 } 210 211 static const char *srp_target_info(struct Scsi_Host *host) 212 { 213 return host_to_target(host)->target_name; 214 } 215 216 static int srp_target_is_topspin(struct srp_target_port *target) 217 { 218 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad }; 219 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d }; 220 221 return topspin_workarounds && 222 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) || 223 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui)); 224 } 225 226 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size, 227 gfp_t gfp_mask, 228 enum dma_data_direction direction) 229 { 230 struct srp_iu *iu; 231 232 iu = kmalloc(sizeof *iu, gfp_mask); 233 if (!iu) 234 goto out; 235 236 iu->buf = kzalloc(size, gfp_mask); 237 if (!iu->buf) 238 goto out_free_iu; 239 240 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size, 241 direction); 242 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma)) 243 goto out_free_buf; 244 245 iu->size = size; 246 iu->direction = direction; 247 248 return iu; 249 250 out_free_buf: 251 kfree(iu->buf); 252 out_free_iu: 253 kfree(iu); 254 out: 255 return NULL; 256 } 257 258 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu) 259 { 260 if (!iu) 261 return; 262 263 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size, 264 iu->direction); 265 kfree(iu->buf); 266 kfree(iu); 267 } 268 269 static void srp_qp_event(struct ib_event *event, void *context) 270 { 271 pr_debug("QP event %s (%d)\n", 272 ib_event_msg(event->event), event->event); 273 } 274 275 static int srp_init_ib_qp(struct srp_target_port *target, 276 struct ib_qp *qp) 277 { 278 struct ib_qp_attr *attr; 279 int ret; 280 281 attr = kmalloc(sizeof *attr, GFP_KERNEL); 282 if (!attr) 283 return -ENOMEM; 284 285 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 286 target->srp_host->port, 287 be16_to_cpu(target->ib_cm.pkey), 288 &attr->pkey_index); 289 if (ret) 290 goto out; 291 292 attr->qp_state = IB_QPS_INIT; 293 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ | 294 IB_ACCESS_REMOTE_WRITE); 295 attr->port_num = target->srp_host->port; 296 297 ret = ib_modify_qp(qp, attr, 298 IB_QP_STATE | 299 IB_QP_PKEY_INDEX | 300 IB_QP_ACCESS_FLAGS | 301 IB_QP_PORT); 302 303 out: 304 kfree(attr); 305 return ret; 306 } 307 308 static int srp_new_ib_cm_id(struct srp_rdma_ch *ch) 309 { 310 struct srp_target_port *target = ch->target; 311 struct ib_cm_id *new_cm_id; 312 313 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, 314 srp_ib_cm_handler, ch); 315 if (IS_ERR(new_cm_id)) 316 return PTR_ERR(new_cm_id); 317 318 if (ch->ib_cm.cm_id) 319 ib_destroy_cm_id(ch->ib_cm.cm_id); 320 ch->ib_cm.cm_id = new_cm_id; 321 if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev, 322 target->srp_host->port)) 323 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA; 324 else 325 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB; 326 ch->ib_cm.path.sgid = target->sgid; 327 ch->ib_cm.path.dgid = target->ib_cm.orig_dgid; 328 ch->ib_cm.path.pkey = target->ib_cm.pkey; 329 ch->ib_cm.path.service_id = target->ib_cm.service_id; 330 331 return 0; 332 } 333 334 static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch) 335 { 336 struct srp_target_port *target = ch->target; 337 struct rdma_cm_id *new_cm_id; 338 int ret; 339 340 new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch, 341 RDMA_PS_TCP, IB_QPT_RC); 342 if (IS_ERR(new_cm_id)) { 343 ret = PTR_ERR(new_cm_id); 344 new_cm_id = NULL; 345 goto out; 346 } 347 348 init_completion(&ch->done); 349 ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ? 350 &target->rdma_cm.src.sa : NULL, 351 &target->rdma_cm.dst.sa, 352 SRP_PATH_REC_TIMEOUT_MS); 353 if (ret) { 354 pr_err("No route available from %pISpsc to %pISpsc (%d)\n", 355 &target->rdma_cm.src, &target->rdma_cm.dst, ret); 356 goto out; 357 } 358 ret = wait_for_completion_interruptible(&ch->done); 359 if (ret < 0) 360 goto out; 361 362 ret = ch->status; 363 if (ret) { 364 pr_err("Resolving address %pISpsc failed (%d)\n", 365 &target->rdma_cm.dst, ret); 366 goto out; 367 } 368 369 swap(ch->rdma_cm.cm_id, new_cm_id); 370 371 out: 372 if (new_cm_id) 373 rdma_destroy_id(new_cm_id); 374 375 return ret; 376 } 377 378 static int srp_new_cm_id(struct srp_rdma_ch *ch) 379 { 380 struct srp_target_port *target = ch->target; 381 382 return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) : 383 srp_new_ib_cm_id(ch); 384 } 385 386 /** 387 * srp_destroy_fr_pool() - free the resources owned by a pool 388 * @pool: Fast registration pool to be destroyed. 389 */ 390 static void srp_destroy_fr_pool(struct srp_fr_pool *pool) 391 { 392 int i; 393 struct srp_fr_desc *d; 394 395 if (!pool) 396 return; 397 398 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 399 if (d->mr) 400 ib_dereg_mr(d->mr); 401 } 402 kfree(pool); 403 } 404 405 /** 406 * srp_create_fr_pool() - allocate and initialize a pool for fast registration 407 * @device: IB device to allocate fast registration descriptors for. 408 * @pd: Protection domain associated with the FR descriptors. 409 * @pool_size: Number of descriptors to allocate. 410 * @max_page_list_len: Maximum fast registration work request page list length. 411 */ 412 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, 413 struct ib_pd *pd, int pool_size, 414 int max_page_list_len) 415 { 416 struct srp_fr_pool *pool; 417 struct srp_fr_desc *d; 418 struct ib_mr *mr; 419 int i, ret = -EINVAL; 420 enum ib_mr_type mr_type; 421 422 if (pool_size <= 0) 423 goto err; 424 ret = -ENOMEM; 425 pool = kzalloc(struct_size(pool, desc, pool_size), GFP_KERNEL); 426 if (!pool) 427 goto err; 428 pool->size = pool_size; 429 pool->max_page_list_len = max_page_list_len; 430 spin_lock_init(&pool->lock); 431 INIT_LIST_HEAD(&pool->free_list); 432 433 if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG) 434 mr_type = IB_MR_TYPE_SG_GAPS; 435 else 436 mr_type = IB_MR_TYPE_MEM_REG; 437 438 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 439 mr = ib_alloc_mr(pd, mr_type, max_page_list_len); 440 if (IS_ERR(mr)) { 441 ret = PTR_ERR(mr); 442 if (ret == -ENOMEM) 443 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n", 444 dev_name(&device->dev)); 445 goto destroy_pool; 446 } 447 d->mr = mr; 448 list_add_tail(&d->entry, &pool->free_list); 449 } 450 451 out: 452 return pool; 453 454 destroy_pool: 455 srp_destroy_fr_pool(pool); 456 457 err: 458 pool = ERR_PTR(ret); 459 goto out; 460 } 461 462 /** 463 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration 464 * @pool: Pool to obtain descriptor from. 465 */ 466 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool) 467 { 468 struct srp_fr_desc *d = NULL; 469 unsigned long flags; 470 471 spin_lock_irqsave(&pool->lock, flags); 472 if (!list_empty(&pool->free_list)) { 473 d = list_first_entry(&pool->free_list, typeof(*d), entry); 474 list_del(&d->entry); 475 } 476 spin_unlock_irqrestore(&pool->lock, flags); 477 478 return d; 479 } 480 481 /** 482 * srp_fr_pool_put() - put an FR descriptor back in the free list 483 * @pool: Pool the descriptor was allocated from. 484 * @desc: Pointer to an array of fast registration descriptor pointers. 485 * @n: Number of descriptors to put back. 486 * 487 * Note: The caller must already have queued an invalidation request for 488 * desc->mr->rkey before calling this function. 489 */ 490 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc, 491 int n) 492 { 493 unsigned long flags; 494 int i; 495 496 spin_lock_irqsave(&pool->lock, flags); 497 for (i = 0; i < n; i++) 498 list_add(&desc[i]->entry, &pool->free_list); 499 spin_unlock_irqrestore(&pool->lock, flags); 500 } 501 502 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) 503 { 504 struct srp_device *dev = target->srp_host->srp_dev; 505 506 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size, 507 dev->max_pages_per_mr); 508 } 509 510 /** 511 * srp_destroy_qp() - destroy an RDMA queue pair 512 * @ch: SRP RDMA channel. 513 * 514 * Drain the qp before destroying it. This avoids that the receive 515 * completion handler can access the queue pair while it is 516 * being destroyed. 517 */ 518 static void srp_destroy_qp(struct srp_rdma_ch *ch) 519 { 520 spin_lock_irq(&ch->lock); 521 ib_process_cq_direct(ch->send_cq, -1); 522 spin_unlock_irq(&ch->lock); 523 524 ib_drain_qp(ch->qp); 525 ib_destroy_qp(ch->qp); 526 } 527 528 static int srp_create_ch_ib(struct srp_rdma_ch *ch) 529 { 530 struct srp_target_port *target = ch->target; 531 struct srp_device *dev = target->srp_host->srp_dev; 532 const struct ib_device_attr *attr = &dev->dev->attrs; 533 struct ib_qp_init_attr *init_attr; 534 struct ib_cq *recv_cq, *send_cq; 535 struct ib_qp *qp; 536 struct srp_fr_pool *fr_pool = NULL; 537 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2; 538 int ret; 539 540 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL); 541 if (!init_attr) 542 return -ENOMEM; 543 544 /* queue_size + 1 for ib_drain_rq() */ 545 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1, 546 ch->comp_vector, IB_POLL_SOFTIRQ); 547 if (IS_ERR(recv_cq)) { 548 ret = PTR_ERR(recv_cq); 549 goto err; 550 } 551 552 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size, 553 ch->comp_vector, IB_POLL_DIRECT); 554 if (IS_ERR(send_cq)) { 555 ret = PTR_ERR(send_cq); 556 goto err_recv_cq; 557 } 558 559 init_attr->event_handler = srp_qp_event; 560 init_attr->cap.max_send_wr = m * target->queue_size; 561 init_attr->cap.max_recv_wr = target->queue_size + 1; 562 init_attr->cap.max_recv_sge = 1; 563 init_attr->cap.max_send_sge = min(SRP_MAX_SGE, attr->max_send_sge); 564 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 565 init_attr->qp_type = IB_QPT_RC; 566 init_attr->send_cq = send_cq; 567 init_attr->recv_cq = recv_cq; 568 569 ch->max_imm_sge = min(init_attr->cap.max_send_sge - 1U, 255U); 570 571 if (target->using_rdma_cm) { 572 ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr); 573 qp = ch->rdma_cm.cm_id->qp; 574 } else { 575 qp = ib_create_qp(dev->pd, init_attr); 576 if (!IS_ERR(qp)) { 577 ret = srp_init_ib_qp(target, qp); 578 if (ret) 579 ib_destroy_qp(qp); 580 } else { 581 ret = PTR_ERR(qp); 582 } 583 } 584 if (ret) { 585 pr_err("QP creation failed for dev %s: %d\n", 586 dev_name(&dev->dev->dev), ret); 587 goto err_send_cq; 588 } 589 590 if (dev->use_fast_reg) { 591 fr_pool = srp_alloc_fr_pool(target); 592 if (IS_ERR(fr_pool)) { 593 ret = PTR_ERR(fr_pool); 594 shost_printk(KERN_WARNING, target->scsi_host, PFX 595 "FR pool allocation failed (%d)\n", ret); 596 goto err_qp; 597 } 598 } 599 600 if (ch->qp) 601 srp_destroy_qp(ch); 602 if (ch->recv_cq) 603 ib_free_cq(ch->recv_cq); 604 if (ch->send_cq) 605 ib_free_cq(ch->send_cq); 606 607 ch->qp = qp; 608 ch->recv_cq = recv_cq; 609 ch->send_cq = send_cq; 610 611 if (dev->use_fast_reg) { 612 if (ch->fr_pool) 613 srp_destroy_fr_pool(ch->fr_pool); 614 ch->fr_pool = fr_pool; 615 } 616 617 kfree(init_attr); 618 return 0; 619 620 err_qp: 621 if (target->using_rdma_cm) 622 rdma_destroy_qp(ch->rdma_cm.cm_id); 623 else 624 ib_destroy_qp(qp); 625 626 err_send_cq: 627 ib_free_cq(send_cq); 628 629 err_recv_cq: 630 ib_free_cq(recv_cq); 631 632 err: 633 kfree(init_attr); 634 return ret; 635 } 636 637 /* 638 * Note: this function may be called without srp_alloc_iu_bufs() having been 639 * invoked. Hence the ch->[rt]x_ring checks. 640 */ 641 static void srp_free_ch_ib(struct srp_target_port *target, 642 struct srp_rdma_ch *ch) 643 { 644 struct srp_device *dev = target->srp_host->srp_dev; 645 int i; 646 647 if (!ch->target) 648 return; 649 650 if (target->using_rdma_cm) { 651 if (ch->rdma_cm.cm_id) { 652 rdma_destroy_id(ch->rdma_cm.cm_id); 653 ch->rdma_cm.cm_id = NULL; 654 } 655 } else { 656 if (ch->ib_cm.cm_id) { 657 ib_destroy_cm_id(ch->ib_cm.cm_id); 658 ch->ib_cm.cm_id = NULL; 659 } 660 } 661 662 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */ 663 if (!ch->qp) 664 return; 665 666 if (dev->use_fast_reg) { 667 if (ch->fr_pool) 668 srp_destroy_fr_pool(ch->fr_pool); 669 } 670 671 srp_destroy_qp(ch); 672 ib_free_cq(ch->send_cq); 673 ib_free_cq(ch->recv_cq); 674 675 /* 676 * Avoid that the SCSI error handler tries to use this channel after 677 * it has been freed. The SCSI error handler can namely continue 678 * trying to perform recovery actions after scsi_remove_host() 679 * returned. 680 */ 681 ch->target = NULL; 682 683 ch->qp = NULL; 684 ch->send_cq = ch->recv_cq = NULL; 685 686 if (ch->rx_ring) { 687 for (i = 0; i < target->queue_size; ++i) 688 srp_free_iu(target->srp_host, ch->rx_ring[i]); 689 kfree(ch->rx_ring); 690 ch->rx_ring = NULL; 691 } 692 if (ch->tx_ring) { 693 for (i = 0; i < target->queue_size; ++i) 694 srp_free_iu(target->srp_host, ch->tx_ring[i]); 695 kfree(ch->tx_ring); 696 ch->tx_ring = NULL; 697 } 698 } 699 700 static void srp_path_rec_completion(int status, 701 struct sa_path_rec *pathrec, 702 void *ch_ptr) 703 { 704 struct srp_rdma_ch *ch = ch_ptr; 705 struct srp_target_port *target = ch->target; 706 707 ch->status = status; 708 if (status) 709 shost_printk(KERN_ERR, target->scsi_host, 710 PFX "Got failed path rec status %d\n", status); 711 else 712 ch->ib_cm.path = *pathrec; 713 complete(&ch->done); 714 } 715 716 static int srp_ib_lookup_path(struct srp_rdma_ch *ch) 717 { 718 struct srp_target_port *target = ch->target; 719 int ret; 720 721 ch->ib_cm.path.numb_path = 1; 722 723 init_completion(&ch->done); 724 725 ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client, 726 target->srp_host->srp_dev->dev, 727 target->srp_host->port, 728 &ch->ib_cm.path, 729 IB_SA_PATH_REC_SERVICE_ID | 730 IB_SA_PATH_REC_DGID | 731 IB_SA_PATH_REC_SGID | 732 IB_SA_PATH_REC_NUMB_PATH | 733 IB_SA_PATH_REC_PKEY, 734 SRP_PATH_REC_TIMEOUT_MS, 735 GFP_KERNEL, 736 srp_path_rec_completion, 737 ch, &ch->ib_cm.path_query); 738 if (ch->ib_cm.path_query_id < 0) 739 return ch->ib_cm.path_query_id; 740 741 ret = wait_for_completion_interruptible(&ch->done); 742 if (ret < 0) 743 return ret; 744 745 if (ch->status < 0) 746 shost_printk(KERN_WARNING, target->scsi_host, 747 PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n", 748 ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw, 749 be16_to_cpu(target->ib_cm.pkey), 750 be64_to_cpu(target->ib_cm.service_id)); 751 752 return ch->status; 753 } 754 755 static int srp_rdma_lookup_path(struct srp_rdma_ch *ch) 756 { 757 struct srp_target_port *target = ch->target; 758 int ret; 759 760 init_completion(&ch->done); 761 762 ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS); 763 if (ret) 764 return ret; 765 766 wait_for_completion_interruptible(&ch->done); 767 768 if (ch->status != 0) 769 shost_printk(KERN_WARNING, target->scsi_host, 770 PFX "Path resolution failed\n"); 771 772 return ch->status; 773 } 774 775 static int srp_lookup_path(struct srp_rdma_ch *ch) 776 { 777 struct srp_target_port *target = ch->target; 778 779 return target->using_rdma_cm ? srp_rdma_lookup_path(ch) : 780 srp_ib_lookup_path(ch); 781 } 782 783 static u8 srp_get_subnet_timeout(struct srp_host *host) 784 { 785 struct ib_port_attr attr; 786 int ret; 787 u8 subnet_timeout = 18; 788 789 ret = ib_query_port(host->srp_dev->dev, host->port, &attr); 790 if (ret == 0) 791 subnet_timeout = attr.subnet_timeout; 792 793 if (unlikely(subnet_timeout < 15)) 794 pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n", 795 dev_name(&host->srp_dev->dev->dev), subnet_timeout); 796 797 return subnet_timeout; 798 } 799 800 static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len, 801 bool multich) 802 { 803 struct srp_target_port *target = ch->target; 804 struct { 805 struct rdma_conn_param rdma_param; 806 struct srp_login_req_rdma rdma_req; 807 struct ib_cm_req_param ib_param; 808 struct srp_login_req ib_req; 809 } *req = NULL; 810 char *ipi, *tpi; 811 int status; 812 813 req = kzalloc(sizeof *req, GFP_KERNEL); 814 if (!req) 815 return -ENOMEM; 816 817 req->ib_param.flow_control = 1; 818 req->ib_param.retry_count = target->tl_retry_count; 819 820 /* 821 * Pick some arbitrary defaults here; we could make these 822 * module parameters if anyone cared about setting them. 823 */ 824 req->ib_param.responder_resources = 4; 825 req->ib_param.rnr_retry_count = 7; 826 req->ib_param.max_cm_retries = 15; 827 828 req->ib_req.opcode = SRP_LOGIN_REQ; 829 req->ib_req.tag = 0; 830 req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len); 831 req->ib_req.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | 832 SRP_BUF_FORMAT_INDIRECT); 833 req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI : 834 SRP_MULTICHAN_SINGLE); 835 if (srp_use_imm_data) { 836 req->ib_req.req_flags |= SRP_IMMED_REQUESTED; 837 req->ib_req.imm_data_offset = cpu_to_be16(SRP_IMM_DATA_OFFSET); 838 } 839 840 if (target->using_rdma_cm) { 841 req->rdma_param.flow_control = req->ib_param.flow_control; 842 req->rdma_param.responder_resources = 843 req->ib_param.responder_resources; 844 req->rdma_param.initiator_depth = req->ib_param.initiator_depth; 845 req->rdma_param.retry_count = req->ib_param.retry_count; 846 req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count; 847 req->rdma_param.private_data = &req->rdma_req; 848 req->rdma_param.private_data_len = sizeof(req->rdma_req); 849 850 req->rdma_req.opcode = req->ib_req.opcode; 851 req->rdma_req.tag = req->ib_req.tag; 852 req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len; 853 req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt; 854 req->rdma_req.req_flags = req->ib_req.req_flags; 855 req->rdma_req.imm_data_offset = req->ib_req.imm_data_offset; 856 857 ipi = req->rdma_req.initiator_port_id; 858 tpi = req->rdma_req.target_port_id; 859 } else { 860 u8 subnet_timeout; 861 862 subnet_timeout = srp_get_subnet_timeout(target->srp_host); 863 864 req->ib_param.primary_path = &ch->ib_cm.path; 865 req->ib_param.alternate_path = NULL; 866 req->ib_param.service_id = target->ib_cm.service_id; 867 get_random_bytes(&req->ib_param.starting_psn, 4); 868 req->ib_param.starting_psn &= 0xffffff; 869 req->ib_param.qp_num = ch->qp->qp_num; 870 req->ib_param.qp_type = ch->qp->qp_type; 871 req->ib_param.local_cm_response_timeout = subnet_timeout + 2; 872 req->ib_param.remote_cm_response_timeout = subnet_timeout + 2; 873 req->ib_param.private_data = &req->ib_req; 874 req->ib_param.private_data_len = sizeof(req->ib_req); 875 876 ipi = req->ib_req.initiator_port_id; 877 tpi = req->ib_req.target_port_id; 878 } 879 880 /* 881 * In the published SRP specification (draft rev. 16a), the 882 * port identifier format is 8 bytes of ID extension followed 883 * by 8 bytes of GUID. Older drafts put the two halves in the 884 * opposite order, so that the GUID comes first. 885 * 886 * Targets conforming to these obsolete drafts can be 887 * recognized by the I/O Class they report. 888 */ 889 if (target->io_class == SRP_REV10_IB_IO_CLASS) { 890 memcpy(ipi, &target->sgid.global.interface_id, 8); 891 memcpy(ipi + 8, &target->initiator_ext, 8); 892 memcpy(tpi, &target->ioc_guid, 8); 893 memcpy(tpi + 8, &target->id_ext, 8); 894 } else { 895 memcpy(ipi, &target->initiator_ext, 8); 896 memcpy(ipi + 8, &target->sgid.global.interface_id, 8); 897 memcpy(tpi, &target->id_ext, 8); 898 memcpy(tpi + 8, &target->ioc_guid, 8); 899 } 900 901 /* 902 * Topspin/Cisco SRP targets will reject our login unless we 903 * zero out the first 8 bytes of our initiator port ID and set 904 * the second 8 bytes to the local node GUID. 905 */ 906 if (srp_target_is_topspin(target)) { 907 shost_printk(KERN_DEBUG, target->scsi_host, 908 PFX "Topspin/Cisco initiator port ID workaround " 909 "activated for target GUID %016llx\n", 910 be64_to_cpu(target->ioc_guid)); 911 memset(ipi, 0, 8); 912 memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8); 913 } 914 915 if (target->using_rdma_cm) 916 status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param); 917 else 918 status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param); 919 920 kfree(req); 921 922 return status; 923 } 924 925 static bool srp_queue_remove_work(struct srp_target_port *target) 926 { 927 bool changed = false; 928 929 spin_lock_irq(&target->lock); 930 if (target->state != SRP_TARGET_REMOVED) { 931 target->state = SRP_TARGET_REMOVED; 932 changed = true; 933 } 934 spin_unlock_irq(&target->lock); 935 936 if (changed) 937 queue_work(srp_remove_wq, &target->remove_work); 938 939 return changed; 940 } 941 942 static void srp_disconnect_target(struct srp_target_port *target) 943 { 944 struct srp_rdma_ch *ch; 945 int i, ret; 946 947 /* XXX should send SRP_I_LOGOUT request */ 948 949 for (i = 0; i < target->ch_count; i++) { 950 ch = &target->ch[i]; 951 ch->connected = false; 952 ret = 0; 953 if (target->using_rdma_cm) { 954 if (ch->rdma_cm.cm_id) 955 rdma_disconnect(ch->rdma_cm.cm_id); 956 } else { 957 if (ch->ib_cm.cm_id) 958 ret = ib_send_cm_dreq(ch->ib_cm.cm_id, 959 NULL, 0); 960 } 961 if (ret < 0) { 962 shost_printk(KERN_DEBUG, target->scsi_host, 963 PFX "Sending CM DREQ failed\n"); 964 } 965 } 966 } 967 968 static void srp_free_req_data(struct srp_target_port *target, 969 struct srp_rdma_ch *ch) 970 { 971 struct srp_device *dev = target->srp_host->srp_dev; 972 struct ib_device *ibdev = dev->dev; 973 struct srp_request *req; 974 int i; 975 976 if (!ch->req_ring) 977 return; 978 979 for (i = 0; i < target->req_ring_size; ++i) { 980 req = &ch->req_ring[i]; 981 if (dev->use_fast_reg) 982 kfree(req->fr_list); 983 if (req->indirect_dma_addr) { 984 ib_dma_unmap_single(ibdev, req->indirect_dma_addr, 985 target->indirect_size, 986 DMA_TO_DEVICE); 987 } 988 kfree(req->indirect_desc); 989 } 990 991 kfree(ch->req_ring); 992 ch->req_ring = NULL; 993 } 994 995 static int srp_alloc_req_data(struct srp_rdma_ch *ch) 996 { 997 struct srp_target_port *target = ch->target; 998 struct srp_device *srp_dev = target->srp_host->srp_dev; 999 struct ib_device *ibdev = srp_dev->dev; 1000 struct srp_request *req; 1001 void *mr_list; 1002 dma_addr_t dma_addr; 1003 int i, ret = -ENOMEM; 1004 1005 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring), 1006 GFP_KERNEL); 1007 if (!ch->req_ring) 1008 goto out; 1009 1010 for (i = 0; i < target->req_ring_size; ++i) { 1011 req = &ch->req_ring[i]; 1012 mr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *), 1013 GFP_KERNEL); 1014 if (!mr_list) 1015 goto out; 1016 if (srp_dev->use_fast_reg) 1017 req->fr_list = mr_list; 1018 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL); 1019 if (!req->indirect_desc) 1020 goto out; 1021 1022 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc, 1023 target->indirect_size, 1024 DMA_TO_DEVICE); 1025 if (ib_dma_mapping_error(ibdev, dma_addr)) 1026 goto out; 1027 1028 req->indirect_dma_addr = dma_addr; 1029 } 1030 ret = 0; 1031 1032 out: 1033 return ret; 1034 } 1035 1036 /** 1037 * srp_del_scsi_host_attr() - Remove attributes defined in the host template. 1038 * @shost: SCSI host whose attributes to remove from sysfs. 1039 * 1040 * Note: Any attributes defined in the host template and that did not exist 1041 * before invocation of this function will be ignored. 1042 */ 1043 static void srp_del_scsi_host_attr(struct Scsi_Host *shost) 1044 { 1045 struct device_attribute **attr; 1046 1047 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr) 1048 device_remove_file(&shost->shost_dev, *attr); 1049 } 1050 1051 static void srp_remove_target(struct srp_target_port *target) 1052 { 1053 struct srp_rdma_ch *ch; 1054 int i; 1055 1056 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1057 1058 srp_del_scsi_host_attr(target->scsi_host); 1059 srp_rport_get(target->rport); 1060 srp_remove_host(target->scsi_host); 1061 scsi_remove_host(target->scsi_host); 1062 srp_stop_rport_timers(target->rport); 1063 srp_disconnect_target(target); 1064 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 1065 for (i = 0; i < target->ch_count; i++) { 1066 ch = &target->ch[i]; 1067 srp_free_ch_ib(target, ch); 1068 } 1069 cancel_work_sync(&target->tl_err_work); 1070 srp_rport_put(target->rport); 1071 for (i = 0; i < target->ch_count; i++) { 1072 ch = &target->ch[i]; 1073 srp_free_req_data(target, ch); 1074 } 1075 kfree(target->ch); 1076 target->ch = NULL; 1077 1078 spin_lock(&target->srp_host->target_lock); 1079 list_del(&target->list); 1080 spin_unlock(&target->srp_host->target_lock); 1081 1082 scsi_host_put(target->scsi_host); 1083 } 1084 1085 static void srp_remove_work(struct work_struct *work) 1086 { 1087 struct srp_target_port *target = 1088 container_of(work, struct srp_target_port, remove_work); 1089 1090 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1091 1092 srp_remove_target(target); 1093 } 1094 1095 static void srp_rport_delete(struct srp_rport *rport) 1096 { 1097 struct srp_target_port *target = rport->lld_data; 1098 1099 srp_queue_remove_work(target); 1100 } 1101 1102 /** 1103 * srp_connected_ch() - number of connected channels 1104 * @target: SRP target port. 1105 */ 1106 static int srp_connected_ch(struct srp_target_port *target) 1107 { 1108 int i, c = 0; 1109 1110 for (i = 0; i < target->ch_count; i++) 1111 c += target->ch[i].connected; 1112 1113 return c; 1114 } 1115 1116 static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len, 1117 bool multich) 1118 { 1119 struct srp_target_port *target = ch->target; 1120 int ret; 1121 1122 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0); 1123 1124 ret = srp_lookup_path(ch); 1125 if (ret) 1126 goto out; 1127 1128 while (1) { 1129 init_completion(&ch->done); 1130 ret = srp_send_req(ch, max_iu_len, multich); 1131 if (ret) 1132 goto out; 1133 ret = wait_for_completion_interruptible(&ch->done); 1134 if (ret < 0) 1135 goto out; 1136 1137 /* 1138 * The CM event handling code will set status to 1139 * SRP_PORT_REDIRECT if we get a port redirect REJ 1140 * back, or SRP_DLID_REDIRECT if we get a lid/qp 1141 * redirect REJ back. 1142 */ 1143 ret = ch->status; 1144 switch (ret) { 1145 case 0: 1146 ch->connected = true; 1147 goto out; 1148 1149 case SRP_PORT_REDIRECT: 1150 ret = srp_lookup_path(ch); 1151 if (ret) 1152 goto out; 1153 break; 1154 1155 case SRP_DLID_REDIRECT: 1156 break; 1157 1158 case SRP_STALE_CONN: 1159 shost_printk(KERN_ERR, target->scsi_host, PFX 1160 "giving up on stale connection\n"); 1161 ret = -ECONNRESET; 1162 goto out; 1163 1164 default: 1165 goto out; 1166 } 1167 } 1168 1169 out: 1170 return ret <= 0 ? ret : -ENODEV; 1171 } 1172 1173 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc) 1174 { 1175 srp_handle_qp_err(cq, wc, "INV RKEY"); 1176 } 1177 1178 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch, 1179 u32 rkey) 1180 { 1181 struct ib_send_wr wr = { 1182 .opcode = IB_WR_LOCAL_INV, 1183 .next = NULL, 1184 .num_sge = 0, 1185 .send_flags = 0, 1186 .ex.invalidate_rkey = rkey, 1187 }; 1188 1189 wr.wr_cqe = &req->reg_cqe; 1190 req->reg_cqe.done = srp_inv_rkey_err_done; 1191 return ib_post_send(ch->qp, &wr, NULL); 1192 } 1193 1194 static void srp_unmap_data(struct scsi_cmnd *scmnd, 1195 struct srp_rdma_ch *ch, 1196 struct srp_request *req) 1197 { 1198 struct srp_target_port *target = ch->target; 1199 struct srp_device *dev = target->srp_host->srp_dev; 1200 struct ib_device *ibdev = dev->dev; 1201 int i, res; 1202 1203 if (!scsi_sglist(scmnd) || 1204 (scmnd->sc_data_direction != DMA_TO_DEVICE && 1205 scmnd->sc_data_direction != DMA_FROM_DEVICE)) 1206 return; 1207 1208 if (dev->use_fast_reg) { 1209 struct srp_fr_desc **pfr; 1210 1211 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { 1212 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey); 1213 if (res < 0) { 1214 shost_printk(KERN_ERR, target->scsi_host, PFX 1215 "Queueing INV WR for rkey %#x failed (%d)\n", 1216 (*pfr)->mr->rkey, res); 1217 queue_work(system_long_wq, 1218 &target->tl_err_work); 1219 } 1220 } 1221 if (req->nmdesc) 1222 srp_fr_pool_put(ch->fr_pool, req->fr_list, 1223 req->nmdesc); 1224 } 1225 1226 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd), 1227 scmnd->sc_data_direction); 1228 } 1229 1230 /** 1231 * srp_claim_req - Take ownership of the scmnd associated with a request. 1232 * @ch: SRP RDMA channel. 1233 * @req: SRP request. 1234 * @sdev: If not NULL, only take ownership for this SCSI device. 1235 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take 1236 * ownership of @req->scmnd if it equals @scmnd. 1237 * 1238 * Return value: 1239 * Either NULL or a pointer to the SCSI command the caller became owner of. 1240 */ 1241 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch, 1242 struct srp_request *req, 1243 struct scsi_device *sdev, 1244 struct scsi_cmnd *scmnd) 1245 { 1246 unsigned long flags; 1247 1248 spin_lock_irqsave(&ch->lock, flags); 1249 if (req->scmnd && 1250 (!sdev || req->scmnd->device == sdev) && 1251 (!scmnd || req->scmnd == scmnd)) { 1252 scmnd = req->scmnd; 1253 req->scmnd = NULL; 1254 } else { 1255 scmnd = NULL; 1256 } 1257 spin_unlock_irqrestore(&ch->lock, flags); 1258 1259 return scmnd; 1260 } 1261 1262 /** 1263 * srp_free_req() - Unmap data and adjust ch->req_lim. 1264 * @ch: SRP RDMA channel. 1265 * @req: Request to be freed. 1266 * @scmnd: SCSI command associated with @req. 1267 * @req_lim_delta: Amount to be added to @target->req_lim. 1268 */ 1269 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req, 1270 struct scsi_cmnd *scmnd, s32 req_lim_delta) 1271 { 1272 unsigned long flags; 1273 1274 srp_unmap_data(scmnd, ch, req); 1275 1276 spin_lock_irqsave(&ch->lock, flags); 1277 ch->req_lim += req_lim_delta; 1278 spin_unlock_irqrestore(&ch->lock, flags); 1279 } 1280 1281 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req, 1282 struct scsi_device *sdev, int result) 1283 { 1284 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL); 1285 1286 if (scmnd) { 1287 srp_free_req(ch, req, scmnd, 0); 1288 scmnd->result = result; 1289 scmnd->scsi_done(scmnd); 1290 } 1291 } 1292 1293 static void srp_terminate_io(struct srp_rport *rport) 1294 { 1295 struct srp_target_port *target = rport->lld_data; 1296 struct srp_rdma_ch *ch; 1297 int i, j; 1298 1299 for (i = 0; i < target->ch_count; i++) { 1300 ch = &target->ch[i]; 1301 1302 for (j = 0; j < target->req_ring_size; ++j) { 1303 struct srp_request *req = &ch->req_ring[j]; 1304 1305 srp_finish_req(ch, req, NULL, 1306 DID_TRANSPORT_FAILFAST << 16); 1307 } 1308 } 1309 } 1310 1311 /* Calculate maximum initiator to target information unit length. */ 1312 static uint32_t srp_max_it_iu_len(int cmd_sg_cnt, bool use_imm_data, 1313 uint32_t max_it_iu_size) 1314 { 1315 uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN + 1316 sizeof(struct srp_indirect_buf) + 1317 cmd_sg_cnt * sizeof(struct srp_direct_buf); 1318 1319 if (use_imm_data) 1320 max_iu_len = max(max_iu_len, SRP_IMM_DATA_OFFSET + 1321 srp_max_imm_data); 1322 1323 if (max_it_iu_size) 1324 max_iu_len = min(max_iu_len, max_it_iu_size); 1325 1326 pr_debug("max_iu_len = %d\n", max_iu_len); 1327 1328 return max_iu_len; 1329 } 1330 1331 /* 1332 * It is up to the caller to ensure that srp_rport_reconnect() calls are 1333 * serialized and that no concurrent srp_queuecommand(), srp_abort(), 1334 * srp_reset_device() or srp_reset_host() calls will occur while this function 1335 * is in progress. One way to realize that is not to call this function 1336 * directly but to call srp_reconnect_rport() instead since that last function 1337 * serializes calls of this function via rport->mutex and also blocks 1338 * srp_queuecommand() calls before invoking this function. 1339 */ 1340 static int srp_rport_reconnect(struct srp_rport *rport) 1341 { 1342 struct srp_target_port *target = rport->lld_data; 1343 struct srp_rdma_ch *ch; 1344 uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 1345 srp_use_imm_data, 1346 target->max_it_iu_size); 1347 int i, j, ret = 0; 1348 bool multich = false; 1349 1350 srp_disconnect_target(target); 1351 1352 if (target->state == SRP_TARGET_SCANNING) 1353 return -ENODEV; 1354 1355 /* 1356 * Now get a new local CM ID so that we avoid confusing the target in 1357 * case things are really fouled up. Doing so also ensures that all CM 1358 * callbacks will have finished before a new QP is allocated. 1359 */ 1360 for (i = 0; i < target->ch_count; i++) { 1361 ch = &target->ch[i]; 1362 ret += srp_new_cm_id(ch); 1363 } 1364 for (i = 0; i < target->ch_count; i++) { 1365 ch = &target->ch[i]; 1366 for (j = 0; j < target->req_ring_size; ++j) { 1367 struct srp_request *req = &ch->req_ring[j]; 1368 1369 srp_finish_req(ch, req, NULL, DID_RESET << 16); 1370 } 1371 } 1372 for (i = 0; i < target->ch_count; i++) { 1373 ch = &target->ch[i]; 1374 /* 1375 * Whether or not creating a new CM ID succeeded, create a new 1376 * QP. This guarantees that all completion callback function 1377 * invocations have finished before request resetting starts. 1378 */ 1379 ret += srp_create_ch_ib(ch); 1380 1381 INIT_LIST_HEAD(&ch->free_tx); 1382 for (j = 0; j < target->queue_size; ++j) 1383 list_add(&ch->tx_ring[j]->list, &ch->free_tx); 1384 } 1385 1386 target->qp_in_error = false; 1387 1388 for (i = 0; i < target->ch_count; i++) { 1389 ch = &target->ch[i]; 1390 if (ret) 1391 break; 1392 ret = srp_connect_ch(ch, max_iu_len, multich); 1393 multich = true; 1394 } 1395 1396 if (ret == 0) 1397 shost_printk(KERN_INFO, target->scsi_host, 1398 PFX "reconnect succeeded\n"); 1399 1400 return ret; 1401 } 1402 1403 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, 1404 unsigned int dma_len, u32 rkey) 1405 { 1406 struct srp_direct_buf *desc = state->desc; 1407 1408 WARN_ON_ONCE(!dma_len); 1409 1410 desc->va = cpu_to_be64(dma_addr); 1411 desc->key = cpu_to_be32(rkey); 1412 desc->len = cpu_to_be32(dma_len); 1413 1414 state->total_len += dma_len; 1415 state->desc++; 1416 state->ndesc++; 1417 } 1418 1419 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc) 1420 { 1421 srp_handle_qp_err(cq, wc, "FAST REG"); 1422 } 1423 1424 /* 1425 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset 1426 * where to start in the first element. If sg_offset_p != NULL then 1427 * *sg_offset_p is updated to the offset in state->sg[retval] of the first 1428 * byte that has not yet been mapped. 1429 */ 1430 static int srp_map_finish_fr(struct srp_map_state *state, 1431 struct srp_request *req, 1432 struct srp_rdma_ch *ch, int sg_nents, 1433 unsigned int *sg_offset_p) 1434 { 1435 struct srp_target_port *target = ch->target; 1436 struct srp_device *dev = target->srp_host->srp_dev; 1437 struct ib_reg_wr wr; 1438 struct srp_fr_desc *desc; 1439 u32 rkey; 1440 int n, err; 1441 1442 if (state->fr.next >= state->fr.end) { 1443 shost_printk(KERN_ERR, ch->target->scsi_host, 1444 PFX "Out of MRs (mr_per_cmd = %d)\n", 1445 ch->target->mr_per_cmd); 1446 return -ENOMEM; 1447 } 1448 1449 WARN_ON_ONCE(!dev->use_fast_reg); 1450 1451 if (sg_nents == 1 && target->global_rkey) { 1452 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1453 1454 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset, 1455 sg_dma_len(state->sg) - sg_offset, 1456 target->global_rkey); 1457 if (sg_offset_p) 1458 *sg_offset_p = 0; 1459 return 1; 1460 } 1461 1462 desc = srp_fr_pool_get(ch->fr_pool); 1463 if (!desc) 1464 return -ENOMEM; 1465 1466 rkey = ib_inc_rkey(desc->mr->rkey); 1467 ib_update_fast_reg_key(desc->mr, rkey); 1468 1469 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p, 1470 dev->mr_page_size); 1471 if (unlikely(n < 0)) { 1472 srp_fr_pool_put(ch->fr_pool, &desc, 1); 1473 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n", 1474 dev_name(&req->scmnd->device->sdev_gendev), sg_nents, 1475 sg_offset_p ? *sg_offset_p : -1, n); 1476 return n; 1477 } 1478 1479 WARN_ON_ONCE(desc->mr->length == 0); 1480 1481 req->reg_cqe.done = srp_reg_mr_err_done; 1482 1483 wr.wr.next = NULL; 1484 wr.wr.opcode = IB_WR_REG_MR; 1485 wr.wr.wr_cqe = &req->reg_cqe; 1486 wr.wr.num_sge = 0; 1487 wr.wr.send_flags = 0; 1488 wr.mr = desc->mr; 1489 wr.key = desc->mr->rkey; 1490 wr.access = (IB_ACCESS_LOCAL_WRITE | 1491 IB_ACCESS_REMOTE_READ | 1492 IB_ACCESS_REMOTE_WRITE); 1493 1494 *state->fr.next++ = desc; 1495 state->nmdesc++; 1496 1497 srp_map_desc(state, desc->mr->iova, 1498 desc->mr->length, desc->mr->rkey); 1499 1500 err = ib_post_send(ch->qp, &wr.wr, NULL); 1501 if (unlikely(err)) { 1502 WARN_ON_ONCE(err == -ENOMEM); 1503 return err; 1504 } 1505 1506 return n; 1507 } 1508 1509 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1510 struct srp_request *req, struct scatterlist *scat, 1511 int count) 1512 { 1513 unsigned int sg_offset = 0; 1514 1515 state->fr.next = req->fr_list; 1516 state->fr.end = req->fr_list + ch->target->mr_per_cmd; 1517 state->sg = scat; 1518 1519 if (count == 0) 1520 return 0; 1521 1522 while (count) { 1523 int i, n; 1524 1525 n = srp_map_finish_fr(state, req, ch, count, &sg_offset); 1526 if (unlikely(n < 0)) 1527 return n; 1528 1529 count -= n; 1530 for (i = 0; i < n; i++) 1531 state->sg = sg_next(state->sg); 1532 } 1533 1534 return 0; 1535 } 1536 1537 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch, 1538 struct srp_request *req, struct scatterlist *scat, 1539 int count) 1540 { 1541 struct srp_target_port *target = ch->target; 1542 struct scatterlist *sg; 1543 int i; 1544 1545 for_each_sg(scat, sg, count, i) { 1546 srp_map_desc(state, sg_dma_address(sg), sg_dma_len(sg), 1547 target->global_rkey); 1548 } 1549 1550 return 0; 1551 } 1552 1553 /* 1554 * Register the indirect data buffer descriptor with the HCA. 1555 * 1556 * Note: since the indirect data buffer descriptor has been allocated with 1557 * kmalloc() it is guaranteed that this buffer is a physically contiguous 1558 * memory buffer. 1559 */ 1560 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req, 1561 void **next_mr, void **end_mr, u32 idb_len, 1562 __be32 *idb_rkey) 1563 { 1564 struct srp_target_port *target = ch->target; 1565 struct srp_device *dev = target->srp_host->srp_dev; 1566 struct srp_map_state state; 1567 struct srp_direct_buf idb_desc; 1568 struct scatterlist idb_sg[1]; 1569 int ret; 1570 1571 memset(&state, 0, sizeof(state)); 1572 memset(&idb_desc, 0, sizeof(idb_desc)); 1573 state.gen.next = next_mr; 1574 state.gen.end = end_mr; 1575 state.desc = &idb_desc; 1576 state.base_dma_addr = req->indirect_dma_addr; 1577 state.dma_len = idb_len; 1578 1579 if (dev->use_fast_reg) { 1580 state.sg = idb_sg; 1581 sg_init_one(idb_sg, req->indirect_desc, idb_len); 1582 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */ 1583 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1584 idb_sg->dma_length = idb_sg->length; /* hack^2 */ 1585 #endif 1586 ret = srp_map_finish_fr(&state, req, ch, 1, NULL); 1587 if (ret < 0) 1588 return ret; 1589 WARN_ON_ONCE(ret < 1); 1590 } else { 1591 return -EINVAL; 1592 } 1593 1594 *idb_rkey = idb_desc.key; 1595 1596 return 0; 1597 } 1598 1599 static void srp_check_mapping(struct srp_map_state *state, 1600 struct srp_rdma_ch *ch, struct srp_request *req, 1601 struct scatterlist *scat, int count) 1602 { 1603 struct srp_device *dev = ch->target->srp_host->srp_dev; 1604 struct srp_fr_desc **pfr; 1605 u64 desc_len = 0, mr_len = 0; 1606 int i; 1607 1608 for (i = 0; i < state->ndesc; i++) 1609 desc_len += be32_to_cpu(req->indirect_desc[i].len); 1610 if (dev->use_fast_reg) 1611 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++) 1612 mr_len += (*pfr)->mr->length; 1613 if (desc_len != scsi_bufflen(req->scmnd) || 1614 mr_len > scsi_bufflen(req->scmnd)) 1615 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n", 1616 scsi_bufflen(req->scmnd), desc_len, mr_len, 1617 state->ndesc, state->nmdesc); 1618 } 1619 1620 /** 1621 * srp_map_data() - map SCSI data buffer onto an SRP request 1622 * @scmnd: SCSI command to map 1623 * @ch: SRP RDMA channel 1624 * @req: SRP request 1625 * 1626 * Returns the length in bytes of the SRP_CMD IU or a negative value if 1627 * mapping failed. The size of any immediate data is not included in the 1628 * return value. 1629 */ 1630 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch, 1631 struct srp_request *req) 1632 { 1633 struct srp_target_port *target = ch->target; 1634 struct scatterlist *scat, *sg; 1635 struct srp_cmd *cmd = req->cmd->buf; 1636 int i, len, nents, count, ret; 1637 struct srp_device *dev; 1638 struct ib_device *ibdev; 1639 struct srp_map_state state; 1640 struct srp_indirect_buf *indirect_hdr; 1641 u64 data_len; 1642 u32 idb_len, table_len; 1643 __be32 idb_rkey; 1644 u8 fmt; 1645 1646 req->cmd->num_sge = 1; 1647 1648 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE) 1649 return sizeof(struct srp_cmd) + cmd->add_cdb_len; 1650 1651 if (scmnd->sc_data_direction != DMA_FROM_DEVICE && 1652 scmnd->sc_data_direction != DMA_TO_DEVICE) { 1653 shost_printk(KERN_WARNING, target->scsi_host, 1654 PFX "Unhandled data direction %d\n", 1655 scmnd->sc_data_direction); 1656 return -EINVAL; 1657 } 1658 1659 nents = scsi_sg_count(scmnd); 1660 scat = scsi_sglist(scmnd); 1661 data_len = scsi_bufflen(scmnd); 1662 1663 dev = target->srp_host->srp_dev; 1664 ibdev = dev->dev; 1665 1666 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction); 1667 if (unlikely(count == 0)) 1668 return -EIO; 1669 1670 if (ch->use_imm_data && 1671 count <= ch->max_imm_sge && 1672 SRP_IMM_DATA_OFFSET + data_len <= ch->max_it_iu_len && 1673 scmnd->sc_data_direction == DMA_TO_DEVICE) { 1674 struct srp_imm_buf *buf; 1675 struct ib_sge *sge = &req->cmd->sge[1]; 1676 1677 fmt = SRP_DATA_DESC_IMM; 1678 len = SRP_IMM_DATA_OFFSET; 1679 req->nmdesc = 0; 1680 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1681 buf->len = cpu_to_be32(data_len); 1682 WARN_ON_ONCE((void *)(buf + 1) > (void *)cmd + len); 1683 for_each_sg(scat, sg, count, i) { 1684 sge[i].addr = sg_dma_address(sg); 1685 sge[i].length = sg_dma_len(sg); 1686 sge[i].lkey = target->lkey; 1687 } 1688 req->cmd->num_sge += count; 1689 goto map_complete; 1690 } 1691 1692 fmt = SRP_DATA_DESC_DIRECT; 1693 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1694 sizeof(struct srp_direct_buf); 1695 1696 if (count == 1 && target->global_rkey) { 1697 /* 1698 * The midlayer only generated a single gather/scatter 1699 * entry, or DMA mapping coalesced everything to a 1700 * single entry. So a direct descriptor along with 1701 * the DMA MR suffices. 1702 */ 1703 struct srp_direct_buf *buf; 1704 1705 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1706 buf->va = cpu_to_be64(sg_dma_address(scat)); 1707 buf->key = cpu_to_be32(target->global_rkey); 1708 buf->len = cpu_to_be32(sg_dma_len(scat)); 1709 1710 req->nmdesc = 0; 1711 goto map_complete; 1712 } 1713 1714 /* 1715 * We have more than one scatter/gather entry, so build our indirect 1716 * descriptor table, trying to merge as many entries as we can. 1717 */ 1718 indirect_hdr = (void *)cmd->add_data + cmd->add_cdb_len; 1719 1720 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr, 1721 target->indirect_size, DMA_TO_DEVICE); 1722 1723 memset(&state, 0, sizeof(state)); 1724 state.desc = req->indirect_desc; 1725 if (dev->use_fast_reg) 1726 ret = srp_map_sg_fr(&state, ch, req, scat, count); 1727 else 1728 ret = srp_map_sg_dma(&state, ch, req, scat, count); 1729 req->nmdesc = state.nmdesc; 1730 if (ret < 0) 1731 goto unmap; 1732 1733 { 1734 DEFINE_DYNAMIC_DEBUG_METADATA(ddm, 1735 "Memory mapping consistency check"); 1736 if (DYNAMIC_DEBUG_BRANCH(ddm)) 1737 srp_check_mapping(&state, ch, req, scat, count); 1738 } 1739 1740 /* We've mapped the request, now pull as much of the indirect 1741 * descriptor table as we can into the command buffer. If this 1742 * target is not using an external indirect table, we are 1743 * guaranteed to fit into the command, as the SCSI layer won't 1744 * give us more S/G entries than we allow. 1745 */ 1746 if (state.ndesc == 1) { 1747 /* 1748 * Memory registration collapsed the sg-list into one entry, 1749 * so use a direct descriptor. 1750 */ 1751 struct srp_direct_buf *buf; 1752 1753 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1754 *buf = req->indirect_desc[0]; 1755 goto map_complete; 1756 } 1757 1758 if (unlikely(target->cmd_sg_cnt < state.ndesc && 1759 !target->allow_ext_sg)) { 1760 shost_printk(KERN_ERR, target->scsi_host, 1761 "Could not fit S/G list into SRP_CMD\n"); 1762 ret = -EIO; 1763 goto unmap; 1764 } 1765 1766 count = min(state.ndesc, target->cmd_sg_cnt); 1767 table_len = state.ndesc * sizeof (struct srp_direct_buf); 1768 idb_len = sizeof(struct srp_indirect_buf) + table_len; 1769 1770 fmt = SRP_DATA_DESC_INDIRECT; 1771 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1772 sizeof(struct srp_indirect_buf); 1773 len += count * sizeof (struct srp_direct_buf); 1774 1775 memcpy(indirect_hdr->desc_list, req->indirect_desc, 1776 count * sizeof (struct srp_direct_buf)); 1777 1778 if (!target->global_rkey) { 1779 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end, 1780 idb_len, &idb_rkey); 1781 if (ret < 0) 1782 goto unmap; 1783 req->nmdesc++; 1784 } else { 1785 idb_rkey = cpu_to_be32(target->global_rkey); 1786 } 1787 1788 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr); 1789 indirect_hdr->table_desc.key = idb_rkey; 1790 indirect_hdr->table_desc.len = cpu_to_be32(table_len); 1791 indirect_hdr->len = cpu_to_be32(state.total_len); 1792 1793 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1794 cmd->data_out_desc_cnt = count; 1795 else 1796 cmd->data_in_desc_cnt = count; 1797 1798 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len, 1799 DMA_TO_DEVICE); 1800 1801 map_complete: 1802 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1803 cmd->buf_fmt = fmt << 4; 1804 else 1805 cmd->buf_fmt = fmt; 1806 1807 return len; 1808 1809 unmap: 1810 srp_unmap_data(scmnd, ch, req); 1811 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size) 1812 ret = -E2BIG; 1813 return ret; 1814 } 1815 1816 /* 1817 * Return an IU and possible credit to the free pool 1818 */ 1819 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu, 1820 enum srp_iu_type iu_type) 1821 { 1822 unsigned long flags; 1823 1824 spin_lock_irqsave(&ch->lock, flags); 1825 list_add(&iu->list, &ch->free_tx); 1826 if (iu_type != SRP_IU_RSP) 1827 ++ch->req_lim; 1828 spin_unlock_irqrestore(&ch->lock, flags); 1829 } 1830 1831 /* 1832 * Must be called with ch->lock held to protect req_lim and free_tx. 1833 * If IU is not sent, it must be returned using srp_put_tx_iu(). 1834 * 1835 * Note: 1836 * An upper limit for the number of allocated information units for each 1837 * request type is: 1838 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues 1839 * more than Scsi_Host.can_queue requests. 1840 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE. 1841 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than 1842 * one unanswered SRP request to an initiator. 1843 */ 1844 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch, 1845 enum srp_iu_type iu_type) 1846 { 1847 struct srp_target_port *target = ch->target; 1848 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; 1849 struct srp_iu *iu; 1850 1851 lockdep_assert_held(&ch->lock); 1852 1853 ib_process_cq_direct(ch->send_cq, -1); 1854 1855 if (list_empty(&ch->free_tx)) 1856 return NULL; 1857 1858 /* Initiator responses to target requests do not consume credits */ 1859 if (iu_type != SRP_IU_RSP) { 1860 if (ch->req_lim <= rsv) { 1861 ++target->zero_req_lim; 1862 return NULL; 1863 } 1864 1865 --ch->req_lim; 1866 } 1867 1868 iu = list_first_entry(&ch->free_tx, struct srp_iu, list); 1869 list_del(&iu->list); 1870 return iu; 1871 } 1872 1873 /* 1874 * Note: if this function is called from inside ib_drain_sq() then it will 1875 * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE 1876 * with status IB_WC_SUCCESS then that's a bug. 1877 */ 1878 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc) 1879 { 1880 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1881 struct srp_rdma_ch *ch = cq->cq_context; 1882 1883 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1884 srp_handle_qp_err(cq, wc, "SEND"); 1885 return; 1886 } 1887 1888 lockdep_assert_held(&ch->lock); 1889 1890 list_add(&iu->list, &ch->free_tx); 1891 } 1892 1893 /** 1894 * srp_post_send() - send an SRP information unit 1895 * @ch: RDMA channel over which to send the information unit. 1896 * @iu: Information unit to send. 1897 * @len: Length of the information unit excluding immediate data. 1898 */ 1899 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len) 1900 { 1901 struct srp_target_port *target = ch->target; 1902 struct ib_send_wr wr; 1903 1904 if (WARN_ON_ONCE(iu->num_sge > SRP_MAX_SGE)) 1905 return -EINVAL; 1906 1907 iu->sge[0].addr = iu->dma; 1908 iu->sge[0].length = len; 1909 iu->sge[0].lkey = target->lkey; 1910 1911 iu->cqe.done = srp_send_done; 1912 1913 wr.next = NULL; 1914 wr.wr_cqe = &iu->cqe; 1915 wr.sg_list = &iu->sge[0]; 1916 wr.num_sge = iu->num_sge; 1917 wr.opcode = IB_WR_SEND; 1918 wr.send_flags = IB_SEND_SIGNALED; 1919 1920 return ib_post_send(ch->qp, &wr, NULL); 1921 } 1922 1923 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu) 1924 { 1925 struct srp_target_port *target = ch->target; 1926 struct ib_recv_wr wr; 1927 struct ib_sge list; 1928 1929 list.addr = iu->dma; 1930 list.length = iu->size; 1931 list.lkey = target->lkey; 1932 1933 iu->cqe.done = srp_recv_done; 1934 1935 wr.next = NULL; 1936 wr.wr_cqe = &iu->cqe; 1937 wr.sg_list = &list; 1938 wr.num_sge = 1; 1939 1940 return ib_post_recv(ch->qp, &wr, NULL); 1941 } 1942 1943 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) 1944 { 1945 struct srp_target_port *target = ch->target; 1946 struct srp_request *req; 1947 struct scsi_cmnd *scmnd; 1948 unsigned long flags; 1949 1950 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { 1951 spin_lock_irqsave(&ch->lock, flags); 1952 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1953 if (rsp->tag == ch->tsk_mgmt_tag) { 1954 ch->tsk_mgmt_status = -1; 1955 if (be32_to_cpu(rsp->resp_data_len) >= 4) 1956 ch->tsk_mgmt_status = rsp->data[3]; 1957 complete(&ch->tsk_mgmt_done); 1958 } else { 1959 shost_printk(KERN_ERR, target->scsi_host, 1960 "Received tsk mgmt response too late for tag %#llx\n", 1961 rsp->tag); 1962 } 1963 spin_unlock_irqrestore(&ch->lock, flags); 1964 } else { 1965 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag); 1966 if (scmnd && scmnd->host_scribble) { 1967 req = (void *)scmnd->host_scribble; 1968 scmnd = srp_claim_req(ch, req, NULL, scmnd); 1969 } else { 1970 scmnd = NULL; 1971 } 1972 if (!scmnd) { 1973 shost_printk(KERN_ERR, target->scsi_host, 1974 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", 1975 rsp->tag, ch - target->ch, ch->qp->qp_num); 1976 1977 spin_lock_irqsave(&ch->lock, flags); 1978 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1979 spin_unlock_irqrestore(&ch->lock, flags); 1980 1981 return; 1982 } 1983 scmnd->result = rsp->status; 1984 1985 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { 1986 memcpy(scmnd->sense_buffer, rsp->data + 1987 be32_to_cpu(rsp->resp_data_len), 1988 min_t(int, be32_to_cpu(rsp->sense_data_len), 1989 SCSI_SENSE_BUFFERSIZE)); 1990 } 1991 1992 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER)) 1993 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt)); 1994 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER)) 1995 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt)); 1996 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER)) 1997 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt)); 1998 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER)) 1999 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt)); 2000 2001 srp_free_req(ch, req, scmnd, 2002 be32_to_cpu(rsp->req_lim_delta)); 2003 2004 scmnd->host_scribble = NULL; 2005 scmnd->scsi_done(scmnd); 2006 } 2007 } 2008 2009 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, 2010 void *rsp, int len) 2011 { 2012 struct srp_target_port *target = ch->target; 2013 struct ib_device *dev = target->srp_host->srp_dev->dev; 2014 unsigned long flags; 2015 struct srp_iu *iu; 2016 int err; 2017 2018 spin_lock_irqsave(&ch->lock, flags); 2019 ch->req_lim += req_delta; 2020 iu = __srp_get_tx_iu(ch, SRP_IU_RSP); 2021 spin_unlock_irqrestore(&ch->lock, flags); 2022 2023 if (!iu) { 2024 shost_printk(KERN_ERR, target->scsi_host, PFX 2025 "no IU available to send response\n"); 2026 return 1; 2027 } 2028 2029 iu->num_sge = 1; 2030 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); 2031 memcpy(iu->buf, rsp, len); 2032 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); 2033 2034 err = srp_post_send(ch, iu, len); 2035 if (err) { 2036 shost_printk(KERN_ERR, target->scsi_host, PFX 2037 "unable to post response: %d\n", err); 2038 srp_put_tx_iu(ch, iu, SRP_IU_RSP); 2039 } 2040 2041 return err; 2042 } 2043 2044 static void srp_process_cred_req(struct srp_rdma_ch *ch, 2045 struct srp_cred_req *req) 2046 { 2047 struct srp_cred_rsp rsp = { 2048 .opcode = SRP_CRED_RSP, 2049 .tag = req->tag, 2050 }; 2051 s32 delta = be32_to_cpu(req->req_lim_delta); 2052 2053 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2054 shost_printk(KERN_ERR, ch->target->scsi_host, PFX 2055 "problems processing SRP_CRED_REQ\n"); 2056 } 2057 2058 static void srp_process_aer_req(struct srp_rdma_ch *ch, 2059 struct srp_aer_req *req) 2060 { 2061 struct srp_target_port *target = ch->target; 2062 struct srp_aer_rsp rsp = { 2063 .opcode = SRP_AER_RSP, 2064 .tag = req->tag, 2065 }; 2066 s32 delta = be32_to_cpu(req->req_lim_delta); 2067 2068 shost_printk(KERN_ERR, target->scsi_host, PFX 2069 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); 2070 2071 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2072 shost_printk(KERN_ERR, target->scsi_host, PFX 2073 "problems processing SRP_AER_REQ\n"); 2074 } 2075 2076 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) 2077 { 2078 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 2079 struct srp_rdma_ch *ch = cq->cq_context; 2080 struct srp_target_port *target = ch->target; 2081 struct ib_device *dev = target->srp_host->srp_dev->dev; 2082 int res; 2083 u8 opcode; 2084 2085 if (unlikely(wc->status != IB_WC_SUCCESS)) { 2086 srp_handle_qp_err(cq, wc, "RECV"); 2087 return; 2088 } 2089 2090 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len, 2091 DMA_FROM_DEVICE); 2092 2093 opcode = *(u8 *) iu->buf; 2094 2095 if (0) { 2096 shost_printk(KERN_ERR, target->scsi_host, 2097 PFX "recv completion, opcode 0x%02x\n", opcode); 2098 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1, 2099 iu->buf, wc->byte_len, true); 2100 } 2101 2102 switch (opcode) { 2103 case SRP_RSP: 2104 srp_process_rsp(ch, iu->buf); 2105 break; 2106 2107 case SRP_CRED_REQ: 2108 srp_process_cred_req(ch, iu->buf); 2109 break; 2110 2111 case SRP_AER_REQ: 2112 srp_process_aer_req(ch, iu->buf); 2113 break; 2114 2115 case SRP_T_LOGOUT: 2116 /* XXX Handle target logout */ 2117 shost_printk(KERN_WARNING, target->scsi_host, 2118 PFX "Got target logout request\n"); 2119 break; 2120 2121 default: 2122 shost_printk(KERN_WARNING, target->scsi_host, 2123 PFX "Unhandled SRP opcode 0x%02x\n", opcode); 2124 break; 2125 } 2126 2127 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len, 2128 DMA_FROM_DEVICE); 2129 2130 res = srp_post_recv(ch, iu); 2131 if (res != 0) 2132 shost_printk(KERN_ERR, target->scsi_host, 2133 PFX "Recv failed with error code %d\n", res); 2134 } 2135 2136 /** 2137 * srp_tl_err_work() - handle a transport layer error 2138 * @work: Work structure embedded in an SRP target port. 2139 * 2140 * Note: This function may get invoked before the rport has been created, 2141 * hence the target->rport test. 2142 */ 2143 static void srp_tl_err_work(struct work_struct *work) 2144 { 2145 struct srp_target_port *target; 2146 2147 target = container_of(work, struct srp_target_port, tl_err_work); 2148 if (target->rport) 2149 srp_start_tl_fail_timers(target->rport); 2150 } 2151 2152 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 2153 const char *opname) 2154 { 2155 struct srp_rdma_ch *ch = cq->cq_context; 2156 struct srp_target_port *target = ch->target; 2157 2158 if (ch->connected && !target->qp_in_error) { 2159 shost_printk(KERN_ERR, target->scsi_host, 2160 PFX "failed %s status %s (%d) for CQE %p\n", 2161 opname, ib_wc_status_msg(wc->status), wc->status, 2162 wc->wr_cqe); 2163 queue_work(system_long_wq, &target->tl_err_work); 2164 } 2165 target->qp_in_error = true; 2166 } 2167 2168 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) 2169 { 2170 struct srp_target_port *target = host_to_target(shost); 2171 struct srp_rdma_ch *ch; 2172 struct srp_request *req; 2173 struct srp_iu *iu; 2174 struct srp_cmd *cmd; 2175 struct ib_device *dev; 2176 unsigned long flags; 2177 u32 tag; 2178 u16 idx; 2179 int len, ret; 2180 2181 scmnd->result = srp_chkready(target->rport); 2182 if (unlikely(scmnd->result)) 2183 goto err; 2184 2185 WARN_ON_ONCE(scmnd->request->tag < 0); 2186 tag = blk_mq_unique_tag(scmnd->request); 2187 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 2188 idx = blk_mq_unique_tag_to_tag(tag); 2189 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n", 2190 dev_name(&shost->shost_gendev), tag, idx, 2191 target->req_ring_size); 2192 2193 spin_lock_irqsave(&ch->lock, flags); 2194 iu = __srp_get_tx_iu(ch, SRP_IU_CMD); 2195 spin_unlock_irqrestore(&ch->lock, flags); 2196 2197 if (!iu) 2198 goto err; 2199 2200 req = &ch->req_ring[idx]; 2201 dev = target->srp_host->srp_dev->dev; 2202 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len, 2203 DMA_TO_DEVICE); 2204 2205 scmnd->host_scribble = (void *) req; 2206 2207 cmd = iu->buf; 2208 memset(cmd, 0, sizeof *cmd); 2209 2210 cmd->opcode = SRP_CMD; 2211 int_to_scsilun(scmnd->device->lun, &cmd->lun); 2212 cmd->tag = tag; 2213 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len); 2214 if (unlikely(scmnd->cmd_len > sizeof(cmd->cdb))) { 2215 cmd->add_cdb_len = round_up(scmnd->cmd_len - sizeof(cmd->cdb), 2216 4); 2217 if (WARN_ON_ONCE(cmd->add_cdb_len > SRP_MAX_ADD_CDB_LEN)) 2218 goto err_iu; 2219 } 2220 2221 req->scmnd = scmnd; 2222 req->cmd = iu; 2223 2224 len = srp_map_data(scmnd, ch, req); 2225 if (len < 0) { 2226 shost_printk(KERN_ERR, target->scsi_host, 2227 PFX "Failed to map data (%d)\n", len); 2228 /* 2229 * If we ran out of memory descriptors (-ENOMEM) because an 2230 * application is queuing many requests with more than 2231 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer 2232 * to reduce queue depth temporarily. 2233 */ 2234 scmnd->result = len == -ENOMEM ? 2235 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16; 2236 goto err_iu; 2237 } 2238 2239 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len, 2240 DMA_TO_DEVICE); 2241 2242 if (srp_post_send(ch, iu, len)) { 2243 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); 2244 scmnd->result = DID_ERROR << 16; 2245 goto err_unmap; 2246 } 2247 2248 return 0; 2249 2250 err_unmap: 2251 srp_unmap_data(scmnd, ch, req); 2252 2253 err_iu: 2254 srp_put_tx_iu(ch, iu, SRP_IU_CMD); 2255 2256 /* 2257 * Avoid that the loops that iterate over the request ring can 2258 * encounter a dangling SCSI command pointer. 2259 */ 2260 req->scmnd = NULL; 2261 2262 err: 2263 if (scmnd->result) { 2264 scmnd->scsi_done(scmnd); 2265 ret = 0; 2266 } else { 2267 ret = SCSI_MLQUEUE_HOST_BUSY; 2268 } 2269 2270 return ret; 2271 } 2272 2273 /* 2274 * Note: the resources allocated in this function are freed in 2275 * srp_free_ch_ib(). 2276 */ 2277 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch) 2278 { 2279 struct srp_target_port *target = ch->target; 2280 int i; 2281 2282 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring), 2283 GFP_KERNEL); 2284 if (!ch->rx_ring) 2285 goto err_no_ring; 2286 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring), 2287 GFP_KERNEL); 2288 if (!ch->tx_ring) 2289 goto err_no_ring; 2290 2291 for (i = 0; i < target->queue_size; ++i) { 2292 ch->rx_ring[i] = srp_alloc_iu(target->srp_host, 2293 ch->max_ti_iu_len, 2294 GFP_KERNEL, DMA_FROM_DEVICE); 2295 if (!ch->rx_ring[i]) 2296 goto err; 2297 } 2298 2299 for (i = 0; i < target->queue_size; ++i) { 2300 ch->tx_ring[i] = srp_alloc_iu(target->srp_host, 2301 ch->max_it_iu_len, 2302 GFP_KERNEL, DMA_TO_DEVICE); 2303 if (!ch->tx_ring[i]) 2304 goto err; 2305 2306 list_add(&ch->tx_ring[i]->list, &ch->free_tx); 2307 } 2308 2309 return 0; 2310 2311 err: 2312 for (i = 0; i < target->queue_size; ++i) { 2313 srp_free_iu(target->srp_host, ch->rx_ring[i]); 2314 srp_free_iu(target->srp_host, ch->tx_ring[i]); 2315 } 2316 2317 2318 err_no_ring: 2319 kfree(ch->tx_ring); 2320 ch->tx_ring = NULL; 2321 kfree(ch->rx_ring); 2322 ch->rx_ring = NULL; 2323 2324 return -ENOMEM; 2325 } 2326 2327 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) 2328 { 2329 uint64_t T_tr_ns, max_compl_time_ms; 2330 uint32_t rq_tmo_jiffies; 2331 2332 /* 2333 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair, 2334 * table 91), both the QP timeout and the retry count have to be set 2335 * for RC QP's during the RTR to RTS transition. 2336 */ 2337 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) != 2338 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)); 2339 2340 /* 2341 * Set target->rq_tmo_jiffies to one second more than the largest time 2342 * it can take before an error completion is generated. See also 2343 * C9-140..142 in the IBTA spec for more information about how to 2344 * convert the QP Local ACK Timeout value to nanoseconds. 2345 */ 2346 T_tr_ns = 4096 * (1ULL << qp_attr->timeout); 2347 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns; 2348 do_div(max_compl_time_ms, NSEC_PER_MSEC); 2349 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000); 2350 2351 return rq_tmo_jiffies; 2352 } 2353 2354 static void srp_cm_rep_handler(struct ib_cm_id *cm_id, 2355 const struct srp_login_rsp *lrsp, 2356 struct srp_rdma_ch *ch) 2357 { 2358 struct srp_target_port *target = ch->target; 2359 struct ib_qp_attr *qp_attr = NULL; 2360 int attr_mask = 0; 2361 int ret = 0; 2362 int i; 2363 2364 if (lrsp->opcode == SRP_LOGIN_RSP) { 2365 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); 2366 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); 2367 ch->use_imm_data = srp_use_imm_data && 2368 (lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP); 2369 ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 2370 ch->use_imm_data, 2371 target->max_it_iu_size); 2372 WARN_ON_ONCE(ch->max_it_iu_len > 2373 be32_to_cpu(lrsp->max_it_iu_len)); 2374 2375 if (ch->use_imm_data) 2376 shost_printk(KERN_DEBUG, target->scsi_host, 2377 PFX "using immediate data\n"); 2378 2379 /* 2380 * Reserve credits for task management so we don't 2381 * bounce requests back to the SCSI mid-layer. 2382 */ 2383 target->scsi_host->can_queue 2384 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE, 2385 target->scsi_host->can_queue); 2386 target->scsi_host->cmd_per_lun 2387 = min_t(int, target->scsi_host->can_queue, 2388 target->scsi_host->cmd_per_lun); 2389 } else { 2390 shost_printk(KERN_WARNING, target->scsi_host, 2391 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode); 2392 ret = -ECONNRESET; 2393 goto error; 2394 } 2395 2396 if (!ch->rx_ring) { 2397 ret = srp_alloc_iu_bufs(ch); 2398 if (ret) 2399 goto error; 2400 } 2401 2402 for (i = 0; i < target->queue_size; i++) { 2403 struct srp_iu *iu = ch->rx_ring[i]; 2404 2405 ret = srp_post_recv(ch, iu); 2406 if (ret) 2407 goto error; 2408 } 2409 2410 if (!target->using_rdma_cm) { 2411 ret = -ENOMEM; 2412 qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL); 2413 if (!qp_attr) 2414 goto error; 2415 2416 qp_attr->qp_state = IB_QPS_RTR; 2417 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2418 if (ret) 2419 goto error_free; 2420 2421 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2422 if (ret) 2423 goto error_free; 2424 2425 qp_attr->qp_state = IB_QPS_RTS; 2426 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2427 if (ret) 2428 goto error_free; 2429 2430 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); 2431 2432 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2433 if (ret) 2434 goto error_free; 2435 2436 ret = ib_send_cm_rtu(cm_id, NULL, 0); 2437 } 2438 2439 error_free: 2440 kfree(qp_attr); 2441 2442 error: 2443 ch->status = ret; 2444 } 2445 2446 static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id, 2447 const struct ib_cm_event *event, 2448 struct srp_rdma_ch *ch) 2449 { 2450 struct srp_target_port *target = ch->target; 2451 struct Scsi_Host *shost = target->scsi_host; 2452 struct ib_class_port_info *cpi; 2453 int opcode; 2454 u16 dlid; 2455 2456 switch (event->param.rej_rcvd.reason) { 2457 case IB_CM_REJ_PORT_CM_REDIRECT: 2458 cpi = event->param.rej_rcvd.ari; 2459 dlid = be16_to_cpu(cpi->redirect_lid); 2460 sa_path_set_dlid(&ch->ib_cm.path, dlid); 2461 ch->ib_cm.path.pkey = cpi->redirect_pkey; 2462 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; 2463 memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16); 2464 2465 ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; 2466 break; 2467 2468 case IB_CM_REJ_PORT_REDIRECT: 2469 if (srp_target_is_topspin(target)) { 2470 union ib_gid *dgid = &ch->ib_cm.path.dgid; 2471 2472 /* 2473 * Topspin/Cisco SRP gateways incorrectly send 2474 * reject reason code 25 when they mean 24 2475 * (port redirect). 2476 */ 2477 memcpy(dgid->raw, event->param.rej_rcvd.ari, 16); 2478 2479 shost_printk(KERN_DEBUG, shost, 2480 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", 2481 be64_to_cpu(dgid->global.subnet_prefix), 2482 be64_to_cpu(dgid->global.interface_id)); 2483 2484 ch->status = SRP_PORT_REDIRECT; 2485 } else { 2486 shost_printk(KERN_WARNING, shost, 2487 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); 2488 ch->status = -ECONNRESET; 2489 } 2490 break; 2491 2492 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2493 shost_printk(KERN_WARNING, shost, 2494 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2495 ch->status = -ECONNRESET; 2496 break; 2497 2498 case IB_CM_REJ_CONSUMER_DEFINED: 2499 opcode = *(u8 *) event->private_data; 2500 if (opcode == SRP_LOGIN_REJ) { 2501 struct srp_login_rej *rej = event->private_data; 2502 u32 reason = be32_to_cpu(rej->reason); 2503 2504 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2505 shost_printk(KERN_WARNING, shost, 2506 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2507 else 2508 shost_printk(KERN_WARNING, shost, PFX 2509 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n", 2510 target->sgid.raw, 2511 target->ib_cm.orig_dgid.raw, 2512 reason); 2513 } else 2514 shost_printk(KERN_WARNING, shost, 2515 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED," 2516 " opcode 0x%02x\n", opcode); 2517 ch->status = -ECONNRESET; 2518 break; 2519 2520 case IB_CM_REJ_STALE_CONN: 2521 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); 2522 ch->status = SRP_STALE_CONN; 2523 break; 2524 2525 default: 2526 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2527 event->param.rej_rcvd.reason); 2528 ch->status = -ECONNRESET; 2529 } 2530 } 2531 2532 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 2533 const struct ib_cm_event *event) 2534 { 2535 struct srp_rdma_ch *ch = cm_id->context; 2536 struct srp_target_port *target = ch->target; 2537 int comp = 0; 2538 2539 switch (event->event) { 2540 case IB_CM_REQ_ERROR: 2541 shost_printk(KERN_DEBUG, target->scsi_host, 2542 PFX "Sending CM REQ failed\n"); 2543 comp = 1; 2544 ch->status = -ECONNRESET; 2545 break; 2546 2547 case IB_CM_REP_RECEIVED: 2548 comp = 1; 2549 srp_cm_rep_handler(cm_id, event->private_data, ch); 2550 break; 2551 2552 case IB_CM_REJ_RECEIVED: 2553 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2554 comp = 1; 2555 2556 srp_ib_cm_rej_handler(cm_id, event, ch); 2557 break; 2558 2559 case IB_CM_DREQ_RECEIVED: 2560 shost_printk(KERN_WARNING, target->scsi_host, 2561 PFX "DREQ received - connection closed\n"); 2562 ch->connected = false; 2563 if (ib_send_cm_drep(cm_id, NULL, 0)) 2564 shost_printk(KERN_ERR, target->scsi_host, 2565 PFX "Sending CM DREP failed\n"); 2566 queue_work(system_long_wq, &target->tl_err_work); 2567 break; 2568 2569 case IB_CM_TIMEWAIT_EXIT: 2570 shost_printk(KERN_ERR, target->scsi_host, 2571 PFX "connection closed\n"); 2572 comp = 1; 2573 2574 ch->status = 0; 2575 break; 2576 2577 case IB_CM_MRA_RECEIVED: 2578 case IB_CM_DREQ_ERROR: 2579 case IB_CM_DREP_RECEIVED: 2580 break; 2581 2582 default: 2583 shost_printk(KERN_WARNING, target->scsi_host, 2584 PFX "Unhandled CM event %d\n", event->event); 2585 break; 2586 } 2587 2588 if (comp) 2589 complete(&ch->done); 2590 2591 return 0; 2592 } 2593 2594 static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch, 2595 struct rdma_cm_event *event) 2596 { 2597 struct srp_target_port *target = ch->target; 2598 struct Scsi_Host *shost = target->scsi_host; 2599 int opcode; 2600 2601 switch (event->status) { 2602 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2603 shost_printk(KERN_WARNING, shost, 2604 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2605 ch->status = -ECONNRESET; 2606 break; 2607 2608 case IB_CM_REJ_CONSUMER_DEFINED: 2609 opcode = *(u8 *) event->param.conn.private_data; 2610 if (opcode == SRP_LOGIN_REJ) { 2611 struct srp_login_rej *rej = 2612 (struct srp_login_rej *) 2613 event->param.conn.private_data; 2614 u32 reason = be32_to_cpu(rej->reason); 2615 2616 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2617 shost_printk(KERN_WARNING, shost, 2618 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2619 else 2620 shost_printk(KERN_WARNING, shost, 2621 PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason); 2622 } else { 2623 shost_printk(KERN_WARNING, shost, 2624 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n", 2625 opcode); 2626 } 2627 ch->status = -ECONNRESET; 2628 break; 2629 2630 case IB_CM_REJ_STALE_CONN: 2631 shost_printk(KERN_WARNING, shost, 2632 " REJ reason: stale connection\n"); 2633 ch->status = SRP_STALE_CONN; 2634 break; 2635 2636 default: 2637 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2638 event->status); 2639 ch->status = -ECONNRESET; 2640 break; 2641 } 2642 } 2643 2644 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 2645 struct rdma_cm_event *event) 2646 { 2647 struct srp_rdma_ch *ch = cm_id->context; 2648 struct srp_target_port *target = ch->target; 2649 int comp = 0; 2650 2651 switch (event->event) { 2652 case RDMA_CM_EVENT_ADDR_RESOLVED: 2653 ch->status = 0; 2654 comp = 1; 2655 break; 2656 2657 case RDMA_CM_EVENT_ADDR_ERROR: 2658 ch->status = -ENXIO; 2659 comp = 1; 2660 break; 2661 2662 case RDMA_CM_EVENT_ROUTE_RESOLVED: 2663 ch->status = 0; 2664 comp = 1; 2665 break; 2666 2667 case RDMA_CM_EVENT_ROUTE_ERROR: 2668 case RDMA_CM_EVENT_UNREACHABLE: 2669 ch->status = -EHOSTUNREACH; 2670 comp = 1; 2671 break; 2672 2673 case RDMA_CM_EVENT_CONNECT_ERROR: 2674 shost_printk(KERN_DEBUG, target->scsi_host, 2675 PFX "Sending CM REQ failed\n"); 2676 comp = 1; 2677 ch->status = -ECONNRESET; 2678 break; 2679 2680 case RDMA_CM_EVENT_ESTABLISHED: 2681 comp = 1; 2682 srp_cm_rep_handler(NULL, event->param.conn.private_data, ch); 2683 break; 2684 2685 case RDMA_CM_EVENT_REJECTED: 2686 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2687 comp = 1; 2688 2689 srp_rdma_cm_rej_handler(ch, event); 2690 break; 2691 2692 case RDMA_CM_EVENT_DISCONNECTED: 2693 if (ch->connected) { 2694 shost_printk(KERN_WARNING, target->scsi_host, 2695 PFX "received DREQ\n"); 2696 rdma_disconnect(ch->rdma_cm.cm_id); 2697 comp = 1; 2698 ch->status = 0; 2699 queue_work(system_long_wq, &target->tl_err_work); 2700 } 2701 break; 2702 2703 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 2704 shost_printk(KERN_ERR, target->scsi_host, 2705 PFX "connection closed\n"); 2706 2707 comp = 1; 2708 ch->status = 0; 2709 break; 2710 2711 default: 2712 shost_printk(KERN_WARNING, target->scsi_host, 2713 PFX "Unhandled CM event %d\n", event->event); 2714 break; 2715 } 2716 2717 if (comp) 2718 complete(&ch->done); 2719 2720 return 0; 2721 } 2722 2723 /** 2724 * srp_change_queue_depth - setting device queue depth 2725 * @sdev: scsi device struct 2726 * @qdepth: requested queue depth 2727 * 2728 * Returns queue depth. 2729 */ 2730 static int 2731 srp_change_queue_depth(struct scsi_device *sdev, int qdepth) 2732 { 2733 if (!sdev->tagged_supported) 2734 qdepth = 1; 2735 return scsi_change_queue_depth(sdev, qdepth); 2736 } 2737 2738 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, 2739 u8 func, u8 *status) 2740 { 2741 struct srp_target_port *target = ch->target; 2742 struct srp_rport *rport = target->rport; 2743 struct ib_device *dev = target->srp_host->srp_dev->dev; 2744 struct srp_iu *iu; 2745 struct srp_tsk_mgmt *tsk_mgmt; 2746 int res; 2747 2748 if (!ch->connected || target->qp_in_error) 2749 return -1; 2750 2751 /* 2752 * Lock the rport mutex to avoid that srp_create_ch_ib() is 2753 * invoked while a task management function is being sent. 2754 */ 2755 mutex_lock(&rport->mutex); 2756 spin_lock_irq(&ch->lock); 2757 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT); 2758 spin_unlock_irq(&ch->lock); 2759 2760 if (!iu) { 2761 mutex_unlock(&rport->mutex); 2762 2763 return -1; 2764 } 2765 2766 iu->num_sge = 1; 2767 2768 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, 2769 DMA_TO_DEVICE); 2770 tsk_mgmt = iu->buf; 2771 memset(tsk_mgmt, 0, sizeof *tsk_mgmt); 2772 2773 tsk_mgmt->opcode = SRP_TSK_MGMT; 2774 int_to_scsilun(lun, &tsk_mgmt->lun); 2775 tsk_mgmt->tsk_mgmt_func = func; 2776 tsk_mgmt->task_tag = req_tag; 2777 2778 spin_lock_irq(&ch->lock); 2779 ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT; 2780 tsk_mgmt->tag = ch->tsk_mgmt_tag; 2781 spin_unlock_irq(&ch->lock); 2782 2783 init_completion(&ch->tsk_mgmt_done); 2784 2785 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, 2786 DMA_TO_DEVICE); 2787 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) { 2788 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT); 2789 mutex_unlock(&rport->mutex); 2790 2791 return -1; 2792 } 2793 res = wait_for_completion_timeout(&ch->tsk_mgmt_done, 2794 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)); 2795 if (res > 0 && status) 2796 *status = ch->tsk_mgmt_status; 2797 mutex_unlock(&rport->mutex); 2798 2799 WARN_ON_ONCE(res < 0); 2800 2801 return res > 0 ? 0 : -1; 2802 } 2803 2804 static int srp_abort(struct scsi_cmnd *scmnd) 2805 { 2806 struct srp_target_port *target = host_to_target(scmnd->device->host); 2807 struct srp_request *req = (struct srp_request *) scmnd->host_scribble; 2808 u32 tag; 2809 u16 ch_idx; 2810 struct srp_rdma_ch *ch; 2811 int ret; 2812 2813 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); 2814 2815 if (!req) 2816 return SUCCESS; 2817 tag = blk_mq_unique_tag(scmnd->request); 2818 ch_idx = blk_mq_unique_tag_to_hwq(tag); 2819 if (WARN_ON_ONCE(ch_idx >= target->ch_count)) 2820 return SUCCESS; 2821 ch = &target->ch[ch_idx]; 2822 if (!srp_claim_req(ch, req, NULL, scmnd)) 2823 return SUCCESS; 2824 shost_printk(KERN_ERR, target->scsi_host, 2825 "Sending SRP abort for tag %#x\n", tag); 2826 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun, 2827 SRP_TSK_ABORT_TASK, NULL) == 0) 2828 ret = SUCCESS; 2829 else if (target->rport->state == SRP_RPORT_LOST) 2830 ret = FAST_IO_FAIL; 2831 else 2832 ret = FAILED; 2833 if (ret == SUCCESS) { 2834 srp_free_req(ch, req, scmnd, 0); 2835 scmnd->result = DID_ABORT << 16; 2836 scmnd->scsi_done(scmnd); 2837 } 2838 2839 return ret; 2840 } 2841 2842 static int srp_reset_device(struct scsi_cmnd *scmnd) 2843 { 2844 struct srp_target_port *target = host_to_target(scmnd->device->host); 2845 struct srp_rdma_ch *ch; 2846 u8 status; 2847 2848 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); 2849 2850 ch = &target->ch[0]; 2851 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun, 2852 SRP_TSK_LUN_RESET, &status)) 2853 return FAILED; 2854 if (status) 2855 return FAILED; 2856 2857 return SUCCESS; 2858 } 2859 2860 static int srp_reset_host(struct scsi_cmnd *scmnd) 2861 { 2862 struct srp_target_port *target = host_to_target(scmnd->device->host); 2863 2864 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n"); 2865 2866 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED; 2867 } 2868 2869 static int srp_target_alloc(struct scsi_target *starget) 2870 { 2871 struct Scsi_Host *shost = dev_to_shost(starget->dev.parent); 2872 struct srp_target_port *target = host_to_target(shost); 2873 2874 if (target->target_can_queue) 2875 starget->can_queue = target->target_can_queue; 2876 return 0; 2877 } 2878 2879 static int srp_slave_configure(struct scsi_device *sdev) 2880 { 2881 struct Scsi_Host *shost = sdev->host; 2882 struct srp_target_port *target = host_to_target(shost); 2883 struct request_queue *q = sdev->request_queue; 2884 unsigned long timeout; 2885 2886 if (sdev->type == TYPE_DISK) { 2887 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies); 2888 blk_queue_rq_timeout(q, timeout); 2889 } 2890 2891 return 0; 2892 } 2893 2894 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr, 2895 char *buf) 2896 { 2897 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2898 2899 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->id_ext)); 2900 } 2901 2902 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr, 2903 char *buf) 2904 { 2905 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2906 2907 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid)); 2908 } 2909 2910 static ssize_t show_service_id(struct device *dev, 2911 struct device_attribute *attr, char *buf) 2912 { 2913 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2914 2915 if (target->using_rdma_cm) 2916 return -ENOENT; 2917 return sysfs_emit(buf, "0x%016llx\n", 2918 be64_to_cpu(target->ib_cm.service_id)); 2919 } 2920 2921 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr, 2922 char *buf) 2923 { 2924 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2925 2926 if (target->using_rdma_cm) 2927 return -ENOENT; 2928 2929 return sysfs_emit(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey)); 2930 } 2931 2932 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr, 2933 char *buf) 2934 { 2935 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2936 2937 return sysfs_emit(buf, "%pI6\n", target->sgid.raw); 2938 } 2939 2940 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr, 2941 char *buf) 2942 { 2943 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2944 struct srp_rdma_ch *ch = &target->ch[0]; 2945 2946 if (target->using_rdma_cm) 2947 return -ENOENT; 2948 2949 return sysfs_emit(buf, "%pI6\n", ch->ib_cm.path.dgid.raw); 2950 } 2951 2952 static ssize_t show_orig_dgid(struct device *dev, 2953 struct device_attribute *attr, char *buf) 2954 { 2955 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2956 2957 if (target->using_rdma_cm) 2958 return -ENOENT; 2959 2960 return sysfs_emit(buf, "%pI6\n", target->ib_cm.orig_dgid.raw); 2961 } 2962 2963 static ssize_t show_req_lim(struct device *dev, 2964 struct device_attribute *attr, char *buf) 2965 { 2966 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2967 struct srp_rdma_ch *ch; 2968 int i, req_lim = INT_MAX; 2969 2970 for (i = 0; i < target->ch_count; i++) { 2971 ch = &target->ch[i]; 2972 req_lim = min(req_lim, ch->req_lim); 2973 } 2974 2975 return sysfs_emit(buf, "%d\n", req_lim); 2976 } 2977 2978 static ssize_t show_zero_req_lim(struct device *dev, 2979 struct device_attribute *attr, char *buf) 2980 { 2981 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2982 2983 return sysfs_emit(buf, "%d\n", target->zero_req_lim); 2984 } 2985 2986 static ssize_t show_local_ib_port(struct device *dev, 2987 struct device_attribute *attr, char *buf) 2988 { 2989 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2990 2991 return sysfs_emit(buf, "%d\n", target->srp_host->port); 2992 } 2993 2994 static ssize_t show_local_ib_device(struct device *dev, 2995 struct device_attribute *attr, char *buf) 2996 { 2997 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2998 2999 return sysfs_emit(buf, "%s\n", 3000 dev_name(&target->srp_host->srp_dev->dev->dev)); 3001 } 3002 3003 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr, 3004 char *buf) 3005 { 3006 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3007 3008 return sysfs_emit(buf, "%d\n", target->ch_count); 3009 } 3010 3011 static ssize_t show_comp_vector(struct device *dev, 3012 struct device_attribute *attr, char *buf) 3013 { 3014 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3015 3016 return sysfs_emit(buf, "%d\n", target->comp_vector); 3017 } 3018 3019 static ssize_t show_tl_retry_count(struct device *dev, 3020 struct device_attribute *attr, char *buf) 3021 { 3022 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3023 3024 return sysfs_emit(buf, "%d\n", target->tl_retry_count); 3025 } 3026 3027 static ssize_t show_cmd_sg_entries(struct device *dev, 3028 struct device_attribute *attr, char *buf) 3029 { 3030 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3031 3032 return sysfs_emit(buf, "%u\n", target->cmd_sg_cnt); 3033 } 3034 3035 static ssize_t show_allow_ext_sg(struct device *dev, 3036 struct device_attribute *attr, char *buf) 3037 { 3038 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3039 3040 return sysfs_emit(buf, "%s\n", target->allow_ext_sg ? "true" : "false"); 3041 } 3042 3043 static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL); 3044 static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL); 3045 static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL); 3046 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL); 3047 static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL); 3048 static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL); 3049 static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL); 3050 static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL); 3051 static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL); 3052 static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL); 3053 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL); 3054 static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL); 3055 static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL); 3056 static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL); 3057 static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL); 3058 static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL); 3059 3060 static struct device_attribute *srp_host_attrs[] = { 3061 &dev_attr_id_ext, 3062 &dev_attr_ioc_guid, 3063 &dev_attr_service_id, 3064 &dev_attr_pkey, 3065 &dev_attr_sgid, 3066 &dev_attr_dgid, 3067 &dev_attr_orig_dgid, 3068 &dev_attr_req_lim, 3069 &dev_attr_zero_req_lim, 3070 &dev_attr_local_ib_port, 3071 &dev_attr_local_ib_device, 3072 &dev_attr_ch_count, 3073 &dev_attr_comp_vector, 3074 &dev_attr_tl_retry_count, 3075 &dev_attr_cmd_sg_entries, 3076 &dev_attr_allow_ext_sg, 3077 NULL 3078 }; 3079 3080 static struct scsi_host_template srp_template = { 3081 .module = THIS_MODULE, 3082 .name = "InfiniBand SRP initiator", 3083 .proc_name = DRV_NAME, 3084 .target_alloc = srp_target_alloc, 3085 .slave_configure = srp_slave_configure, 3086 .info = srp_target_info, 3087 .queuecommand = srp_queuecommand, 3088 .change_queue_depth = srp_change_queue_depth, 3089 .eh_timed_out = srp_timed_out, 3090 .eh_abort_handler = srp_abort, 3091 .eh_device_reset_handler = srp_reset_device, 3092 .eh_host_reset_handler = srp_reset_host, 3093 .skip_settle_delay = true, 3094 .sg_tablesize = SRP_DEF_SG_TABLESIZE, 3095 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE, 3096 .this_id = -1, 3097 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE, 3098 .shost_attrs = srp_host_attrs, 3099 .track_queue_depth = 1, 3100 }; 3101 3102 static int srp_sdev_count(struct Scsi_Host *host) 3103 { 3104 struct scsi_device *sdev; 3105 int c = 0; 3106 3107 shost_for_each_device(sdev, host) 3108 c++; 3109 3110 return c; 3111 } 3112 3113 /* 3114 * Return values: 3115 * < 0 upon failure. Caller is responsible for SRP target port cleanup. 3116 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port 3117 * removal has been scheduled. 3118 * 0 and target->state != SRP_TARGET_REMOVED upon success. 3119 */ 3120 static int srp_add_target(struct srp_host *host, struct srp_target_port *target) 3121 { 3122 struct srp_rport_identifiers ids; 3123 struct srp_rport *rport; 3124 3125 target->state = SRP_TARGET_SCANNING; 3126 sprintf(target->target_name, "SRP.T10:%016llX", 3127 be64_to_cpu(target->id_ext)); 3128 3129 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent)) 3130 return -ENODEV; 3131 3132 memcpy(ids.port_id, &target->id_ext, 8); 3133 memcpy(ids.port_id + 8, &target->ioc_guid, 8); 3134 ids.roles = SRP_RPORT_ROLE_TARGET; 3135 rport = srp_rport_add(target->scsi_host, &ids); 3136 if (IS_ERR(rport)) { 3137 scsi_remove_host(target->scsi_host); 3138 return PTR_ERR(rport); 3139 } 3140 3141 rport->lld_data = target; 3142 target->rport = rport; 3143 3144 spin_lock(&host->target_lock); 3145 list_add_tail(&target->list, &host->target_list); 3146 spin_unlock(&host->target_lock); 3147 3148 scsi_scan_target(&target->scsi_host->shost_gendev, 3149 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); 3150 3151 if (srp_connected_ch(target) < target->ch_count || 3152 target->qp_in_error) { 3153 shost_printk(KERN_INFO, target->scsi_host, 3154 PFX "SCSI scan failed - removing SCSI host\n"); 3155 srp_queue_remove_work(target); 3156 goto out; 3157 } 3158 3159 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n", 3160 dev_name(&target->scsi_host->shost_gendev), 3161 srp_sdev_count(target->scsi_host)); 3162 3163 spin_lock_irq(&target->lock); 3164 if (target->state == SRP_TARGET_SCANNING) 3165 target->state = SRP_TARGET_LIVE; 3166 spin_unlock_irq(&target->lock); 3167 3168 out: 3169 return 0; 3170 } 3171 3172 static void srp_release_dev(struct device *dev) 3173 { 3174 struct srp_host *host = 3175 container_of(dev, struct srp_host, dev); 3176 3177 complete(&host->released); 3178 } 3179 3180 static struct class srp_class = { 3181 .name = "infiniband_srp", 3182 .dev_release = srp_release_dev 3183 }; 3184 3185 /** 3186 * srp_conn_unique() - check whether the connection to a target is unique 3187 * @host: SRP host. 3188 * @target: SRP target port. 3189 */ 3190 static bool srp_conn_unique(struct srp_host *host, 3191 struct srp_target_port *target) 3192 { 3193 struct srp_target_port *t; 3194 bool ret = false; 3195 3196 if (target->state == SRP_TARGET_REMOVED) 3197 goto out; 3198 3199 ret = true; 3200 3201 spin_lock(&host->target_lock); 3202 list_for_each_entry(t, &host->target_list, list) { 3203 if (t != target && 3204 target->id_ext == t->id_ext && 3205 target->ioc_guid == t->ioc_guid && 3206 target->initiator_ext == t->initiator_ext) { 3207 ret = false; 3208 break; 3209 } 3210 } 3211 spin_unlock(&host->target_lock); 3212 3213 out: 3214 return ret; 3215 } 3216 3217 /* 3218 * Target ports are added by writing 3219 * 3220 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>, 3221 * pkey=<P_Key>,service_id=<service ID> 3222 * or 3223 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>, 3224 * [src=<IPv4 address>,]dest=<IPv4 address>:<port number> 3225 * 3226 * to the add_target sysfs attribute. 3227 */ 3228 enum { 3229 SRP_OPT_ERR = 0, 3230 SRP_OPT_ID_EXT = 1 << 0, 3231 SRP_OPT_IOC_GUID = 1 << 1, 3232 SRP_OPT_DGID = 1 << 2, 3233 SRP_OPT_PKEY = 1 << 3, 3234 SRP_OPT_SERVICE_ID = 1 << 4, 3235 SRP_OPT_MAX_SECT = 1 << 5, 3236 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, 3237 SRP_OPT_IO_CLASS = 1 << 7, 3238 SRP_OPT_INITIATOR_EXT = 1 << 8, 3239 SRP_OPT_CMD_SG_ENTRIES = 1 << 9, 3240 SRP_OPT_ALLOW_EXT_SG = 1 << 10, 3241 SRP_OPT_SG_TABLESIZE = 1 << 11, 3242 SRP_OPT_COMP_VECTOR = 1 << 12, 3243 SRP_OPT_TL_RETRY_COUNT = 1 << 13, 3244 SRP_OPT_QUEUE_SIZE = 1 << 14, 3245 SRP_OPT_IP_SRC = 1 << 15, 3246 SRP_OPT_IP_DEST = 1 << 16, 3247 SRP_OPT_TARGET_CAN_QUEUE= 1 << 17, 3248 SRP_OPT_MAX_IT_IU_SIZE = 1 << 18, 3249 SRP_OPT_CH_COUNT = 1 << 19, 3250 }; 3251 3252 static unsigned int srp_opt_mandatory[] = { 3253 SRP_OPT_ID_EXT | 3254 SRP_OPT_IOC_GUID | 3255 SRP_OPT_DGID | 3256 SRP_OPT_PKEY | 3257 SRP_OPT_SERVICE_ID, 3258 SRP_OPT_ID_EXT | 3259 SRP_OPT_IOC_GUID | 3260 SRP_OPT_IP_DEST, 3261 }; 3262 3263 static const match_table_t srp_opt_tokens = { 3264 { SRP_OPT_ID_EXT, "id_ext=%s" }, 3265 { SRP_OPT_IOC_GUID, "ioc_guid=%s" }, 3266 { SRP_OPT_DGID, "dgid=%s" }, 3267 { SRP_OPT_PKEY, "pkey=%x" }, 3268 { SRP_OPT_SERVICE_ID, "service_id=%s" }, 3269 { SRP_OPT_MAX_SECT, "max_sect=%d" }, 3270 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, 3271 { SRP_OPT_TARGET_CAN_QUEUE, "target_can_queue=%d" }, 3272 { SRP_OPT_IO_CLASS, "io_class=%x" }, 3273 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, 3274 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, 3275 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, 3276 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, 3277 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" }, 3278 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" }, 3279 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" }, 3280 { SRP_OPT_IP_SRC, "src=%s" }, 3281 { SRP_OPT_IP_DEST, "dest=%s" }, 3282 { SRP_OPT_MAX_IT_IU_SIZE, "max_it_iu_size=%d" }, 3283 { SRP_OPT_CH_COUNT, "ch_count=%u", }, 3284 { SRP_OPT_ERR, NULL } 3285 }; 3286 3287 /** 3288 * srp_parse_in - parse an IP address and port number combination 3289 * @net: [in] Network namespace. 3290 * @sa: [out] Address family, IP address and port number. 3291 * @addr_port_str: [in] IP address and port number. 3292 * @has_port: [out] Whether or not @addr_port_str includes a port number. 3293 * 3294 * Parse the following address formats: 3295 * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5. 3296 * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5. 3297 */ 3298 static int srp_parse_in(struct net *net, struct sockaddr_storage *sa, 3299 const char *addr_port_str, bool *has_port) 3300 { 3301 char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL); 3302 char *port_str; 3303 int ret; 3304 3305 if (!addr) 3306 return -ENOMEM; 3307 port_str = strrchr(addr, ':'); 3308 if (port_str && strchr(port_str, ']')) 3309 port_str = NULL; 3310 if (port_str) 3311 *port_str++ = '\0'; 3312 if (has_port) 3313 *has_port = port_str != NULL; 3314 ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa); 3315 if (ret && addr[0]) { 3316 addr_end = addr + strlen(addr) - 1; 3317 if (addr[0] == '[' && *addr_end == ']') { 3318 *addr_end = '\0'; 3319 ret = inet_pton_with_scope(net, AF_INET6, addr + 1, 3320 port_str, sa); 3321 } 3322 } 3323 kfree(addr); 3324 pr_debug("%s -> %pISpfsc\n", addr_port_str, sa); 3325 return ret; 3326 } 3327 3328 static int srp_parse_options(struct net *net, const char *buf, 3329 struct srp_target_port *target) 3330 { 3331 char *options, *sep_opt; 3332 char *p; 3333 substring_t args[MAX_OPT_ARGS]; 3334 unsigned long long ull; 3335 bool has_port; 3336 int opt_mask = 0; 3337 int token; 3338 int ret = -EINVAL; 3339 int i; 3340 3341 options = kstrdup(buf, GFP_KERNEL); 3342 if (!options) 3343 return -ENOMEM; 3344 3345 sep_opt = options; 3346 while ((p = strsep(&sep_opt, ",\n")) != NULL) { 3347 if (!*p) 3348 continue; 3349 3350 token = match_token(p, srp_opt_tokens, args); 3351 opt_mask |= token; 3352 3353 switch (token) { 3354 case SRP_OPT_ID_EXT: 3355 p = match_strdup(args); 3356 if (!p) { 3357 ret = -ENOMEM; 3358 goto out; 3359 } 3360 ret = kstrtoull(p, 16, &ull); 3361 if (ret) { 3362 pr_warn("invalid id_ext parameter '%s'\n", p); 3363 kfree(p); 3364 goto out; 3365 } 3366 target->id_ext = cpu_to_be64(ull); 3367 kfree(p); 3368 break; 3369 3370 case SRP_OPT_IOC_GUID: 3371 p = match_strdup(args); 3372 if (!p) { 3373 ret = -ENOMEM; 3374 goto out; 3375 } 3376 ret = kstrtoull(p, 16, &ull); 3377 if (ret) { 3378 pr_warn("invalid ioc_guid parameter '%s'\n", p); 3379 kfree(p); 3380 goto out; 3381 } 3382 target->ioc_guid = cpu_to_be64(ull); 3383 kfree(p); 3384 break; 3385 3386 case SRP_OPT_DGID: 3387 p = match_strdup(args); 3388 if (!p) { 3389 ret = -ENOMEM; 3390 goto out; 3391 } 3392 if (strlen(p) != 32) { 3393 pr_warn("bad dest GID parameter '%s'\n", p); 3394 kfree(p); 3395 goto out; 3396 } 3397 3398 ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16); 3399 kfree(p); 3400 if (ret < 0) 3401 goto out; 3402 break; 3403 3404 case SRP_OPT_PKEY: 3405 if (match_hex(args, &token)) { 3406 pr_warn("bad P_Key parameter '%s'\n", p); 3407 goto out; 3408 } 3409 target->ib_cm.pkey = cpu_to_be16(token); 3410 break; 3411 3412 case SRP_OPT_SERVICE_ID: 3413 p = match_strdup(args); 3414 if (!p) { 3415 ret = -ENOMEM; 3416 goto out; 3417 } 3418 ret = kstrtoull(p, 16, &ull); 3419 if (ret) { 3420 pr_warn("bad service_id parameter '%s'\n", p); 3421 kfree(p); 3422 goto out; 3423 } 3424 target->ib_cm.service_id = cpu_to_be64(ull); 3425 kfree(p); 3426 break; 3427 3428 case SRP_OPT_IP_SRC: 3429 p = match_strdup(args); 3430 if (!p) { 3431 ret = -ENOMEM; 3432 goto out; 3433 } 3434 ret = srp_parse_in(net, &target->rdma_cm.src.ss, p, 3435 NULL); 3436 if (ret < 0) { 3437 pr_warn("bad source parameter '%s'\n", p); 3438 kfree(p); 3439 goto out; 3440 } 3441 target->rdma_cm.src_specified = true; 3442 kfree(p); 3443 break; 3444 3445 case SRP_OPT_IP_DEST: 3446 p = match_strdup(args); 3447 if (!p) { 3448 ret = -ENOMEM; 3449 goto out; 3450 } 3451 ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p, 3452 &has_port); 3453 if (!has_port) 3454 ret = -EINVAL; 3455 if (ret < 0) { 3456 pr_warn("bad dest parameter '%s'\n", p); 3457 kfree(p); 3458 goto out; 3459 } 3460 target->using_rdma_cm = true; 3461 kfree(p); 3462 break; 3463 3464 case SRP_OPT_MAX_SECT: 3465 if (match_int(args, &token)) { 3466 pr_warn("bad max sect parameter '%s'\n", p); 3467 goto out; 3468 } 3469 target->scsi_host->max_sectors = token; 3470 break; 3471 3472 case SRP_OPT_QUEUE_SIZE: 3473 if (match_int(args, &token) || token < 1) { 3474 pr_warn("bad queue_size parameter '%s'\n", p); 3475 goto out; 3476 } 3477 target->scsi_host->can_queue = token; 3478 target->queue_size = token + SRP_RSP_SQ_SIZE + 3479 SRP_TSK_MGMT_SQ_SIZE; 3480 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3481 target->scsi_host->cmd_per_lun = token; 3482 break; 3483 3484 case SRP_OPT_MAX_CMD_PER_LUN: 3485 if (match_int(args, &token) || token < 1) { 3486 pr_warn("bad max cmd_per_lun parameter '%s'\n", 3487 p); 3488 goto out; 3489 } 3490 target->scsi_host->cmd_per_lun = token; 3491 break; 3492 3493 case SRP_OPT_TARGET_CAN_QUEUE: 3494 if (match_int(args, &token) || token < 1) { 3495 pr_warn("bad max target_can_queue parameter '%s'\n", 3496 p); 3497 goto out; 3498 } 3499 target->target_can_queue = token; 3500 break; 3501 3502 case SRP_OPT_IO_CLASS: 3503 if (match_hex(args, &token)) { 3504 pr_warn("bad IO class parameter '%s'\n", p); 3505 goto out; 3506 } 3507 if (token != SRP_REV10_IB_IO_CLASS && 3508 token != SRP_REV16A_IB_IO_CLASS) { 3509 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n", 3510 token, SRP_REV10_IB_IO_CLASS, 3511 SRP_REV16A_IB_IO_CLASS); 3512 goto out; 3513 } 3514 target->io_class = token; 3515 break; 3516 3517 case SRP_OPT_INITIATOR_EXT: 3518 p = match_strdup(args); 3519 if (!p) { 3520 ret = -ENOMEM; 3521 goto out; 3522 } 3523 ret = kstrtoull(p, 16, &ull); 3524 if (ret) { 3525 pr_warn("bad initiator_ext value '%s'\n", p); 3526 kfree(p); 3527 goto out; 3528 } 3529 target->initiator_ext = cpu_to_be64(ull); 3530 kfree(p); 3531 break; 3532 3533 case SRP_OPT_CMD_SG_ENTRIES: 3534 if (match_int(args, &token) || token < 1 || token > 255) { 3535 pr_warn("bad max cmd_sg_entries parameter '%s'\n", 3536 p); 3537 goto out; 3538 } 3539 target->cmd_sg_cnt = token; 3540 break; 3541 3542 case SRP_OPT_ALLOW_EXT_SG: 3543 if (match_int(args, &token)) { 3544 pr_warn("bad allow_ext_sg parameter '%s'\n", p); 3545 goto out; 3546 } 3547 target->allow_ext_sg = !!token; 3548 break; 3549 3550 case SRP_OPT_SG_TABLESIZE: 3551 if (match_int(args, &token) || token < 1 || 3552 token > SG_MAX_SEGMENTS) { 3553 pr_warn("bad max sg_tablesize parameter '%s'\n", 3554 p); 3555 goto out; 3556 } 3557 target->sg_tablesize = token; 3558 break; 3559 3560 case SRP_OPT_COMP_VECTOR: 3561 if (match_int(args, &token) || token < 0) { 3562 pr_warn("bad comp_vector parameter '%s'\n", p); 3563 goto out; 3564 } 3565 target->comp_vector = token; 3566 break; 3567 3568 case SRP_OPT_TL_RETRY_COUNT: 3569 if (match_int(args, &token) || token < 2 || token > 7) { 3570 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n", 3571 p); 3572 goto out; 3573 } 3574 target->tl_retry_count = token; 3575 break; 3576 3577 case SRP_OPT_MAX_IT_IU_SIZE: 3578 if (match_int(args, &token) || token < 0) { 3579 pr_warn("bad maximum initiator to target IU size '%s'\n", p); 3580 goto out; 3581 } 3582 target->max_it_iu_size = token; 3583 break; 3584 3585 case SRP_OPT_CH_COUNT: 3586 if (match_int(args, &token) || token < 1) { 3587 pr_warn("bad channel count %s\n", p); 3588 goto out; 3589 } 3590 target->ch_count = token; 3591 break; 3592 3593 default: 3594 pr_warn("unknown parameter or missing value '%s' in target creation request\n", 3595 p); 3596 goto out; 3597 } 3598 } 3599 3600 for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) { 3601 if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) { 3602 ret = 0; 3603 break; 3604 } 3605 } 3606 if (ret) 3607 pr_warn("target creation request is missing one or more parameters\n"); 3608 3609 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue 3610 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3611 pr_warn("cmd_per_lun = %d > queue_size = %d\n", 3612 target->scsi_host->cmd_per_lun, 3613 target->scsi_host->can_queue); 3614 3615 out: 3616 kfree(options); 3617 return ret; 3618 } 3619 3620 static ssize_t srp_create_target(struct device *dev, 3621 struct device_attribute *attr, 3622 const char *buf, size_t count) 3623 { 3624 struct srp_host *host = 3625 container_of(dev, struct srp_host, dev); 3626 struct Scsi_Host *target_host; 3627 struct srp_target_port *target; 3628 struct srp_rdma_ch *ch; 3629 struct srp_device *srp_dev = host->srp_dev; 3630 struct ib_device *ibdev = srp_dev->dev; 3631 int ret, i, ch_idx; 3632 unsigned int max_sectors_per_mr, mr_per_cmd = 0; 3633 bool multich = false; 3634 uint32_t max_iu_len; 3635 3636 target_host = scsi_host_alloc(&srp_template, 3637 sizeof (struct srp_target_port)); 3638 if (!target_host) 3639 return -ENOMEM; 3640 3641 target_host->transportt = ib_srp_transport_template; 3642 target_host->max_channel = 0; 3643 target_host->max_id = 1; 3644 target_host->max_lun = -1LL; 3645 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; 3646 target_host->max_segment_size = ib_dma_max_seg_size(ibdev); 3647 3648 if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)) 3649 target_host->virt_boundary_mask = ~srp_dev->mr_page_mask; 3650 3651 target = host_to_target(target_host); 3652 3653 target->net = kobj_ns_grab_current(KOBJ_NS_TYPE_NET); 3654 target->io_class = SRP_REV16A_IB_IO_CLASS; 3655 target->scsi_host = target_host; 3656 target->srp_host = host; 3657 target->lkey = host->srp_dev->pd->local_dma_lkey; 3658 target->global_rkey = host->srp_dev->global_rkey; 3659 target->cmd_sg_cnt = cmd_sg_entries; 3660 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries; 3661 target->allow_ext_sg = allow_ext_sg; 3662 target->tl_retry_count = 7; 3663 target->queue_size = SRP_DEFAULT_QUEUE_SIZE; 3664 3665 /* 3666 * Avoid that the SCSI host can be removed by srp_remove_target() 3667 * before this function returns. 3668 */ 3669 scsi_host_get(target->scsi_host); 3670 3671 ret = mutex_lock_interruptible(&host->add_target_mutex); 3672 if (ret < 0) 3673 goto put; 3674 3675 ret = srp_parse_options(target->net, buf, target); 3676 if (ret) 3677 goto out; 3678 3679 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE; 3680 3681 if (!srp_conn_unique(target->srp_host, target)) { 3682 if (target->using_rdma_cm) { 3683 shost_printk(KERN_INFO, target->scsi_host, 3684 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n", 3685 be64_to_cpu(target->id_ext), 3686 be64_to_cpu(target->ioc_guid), 3687 &target->rdma_cm.dst); 3688 } else { 3689 shost_printk(KERN_INFO, target->scsi_host, 3690 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n", 3691 be64_to_cpu(target->id_ext), 3692 be64_to_cpu(target->ioc_guid), 3693 be64_to_cpu(target->initiator_ext)); 3694 } 3695 ret = -EEXIST; 3696 goto out; 3697 } 3698 3699 if (!srp_dev->has_fr && !target->allow_ext_sg && 3700 target->cmd_sg_cnt < target->sg_tablesize) { 3701 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n"); 3702 target->sg_tablesize = target->cmd_sg_cnt; 3703 } 3704 3705 if (srp_dev->use_fast_reg) { 3706 bool gaps_reg = (ibdev->attrs.device_cap_flags & 3707 IB_DEVICE_SG_GAPS_REG); 3708 3709 max_sectors_per_mr = srp_dev->max_pages_per_mr << 3710 (ilog2(srp_dev->mr_page_size) - 9); 3711 if (!gaps_reg) { 3712 /* 3713 * FR can only map one HCA page per entry. If the start 3714 * address is not aligned on a HCA page boundary two 3715 * entries will be used for the head and the tail 3716 * although these two entries combined contain at most 3717 * one HCA page of data. Hence the "+ 1" in the 3718 * calculation below. 3719 * 3720 * The indirect data buffer descriptor is contiguous 3721 * so the memory for that buffer will only be 3722 * registered if register_always is true. Hence add 3723 * one to mr_per_cmd if register_always has been set. 3724 */ 3725 mr_per_cmd = register_always + 3726 (target->scsi_host->max_sectors + 1 + 3727 max_sectors_per_mr - 1) / max_sectors_per_mr; 3728 } else { 3729 mr_per_cmd = register_always + 3730 (target->sg_tablesize + 3731 srp_dev->max_pages_per_mr - 1) / 3732 srp_dev->max_pages_per_mr; 3733 } 3734 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n", 3735 target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size, 3736 max_sectors_per_mr, mr_per_cmd); 3737 } 3738 3739 target_host->sg_tablesize = target->sg_tablesize; 3740 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd; 3741 target->mr_per_cmd = mr_per_cmd; 3742 target->indirect_size = target->sg_tablesize * 3743 sizeof (struct srp_direct_buf); 3744 max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 3745 srp_use_imm_data, 3746 target->max_it_iu_size); 3747 3748 INIT_WORK(&target->tl_err_work, srp_tl_err_work); 3749 INIT_WORK(&target->remove_work, srp_remove_work); 3750 spin_lock_init(&target->lock); 3751 ret = rdma_query_gid(ibdev, host->port, 0, &target->sgid); 3752 if (ret) 3753 goto out; 3754 3755 ret = -ENOMEM; 3756 if (target->ch_count == 0) { 3757 target->ch_count = 3758 min(ch_count ?: 3759 max(4 * num_online_nodes(), 3760 ibdev->num_comp_vectors), 3761 num_online_cpus()); 3762 } 3763 3764 target->ch = kcalloc(target->ch_count, sizeof(*target->ch), 3765 GFP_KERNEL); 3766 if (!target->ch) 3767 goto out; 3768 3769 for (ch_idx = 0; ch_idx < target->ch_count; ++ch_idx) { 3770 ch = &target->ch[ch_idx]; 3771 ch->target = target; 3772 ch->comp_vector = ch_idx % ibdev->num_comp_vectors; 3773 spin_lock_init(&ch->lock); 3774 INIT_LIST_HEAD(&ch->free_tx); 3775 ret = srp_new_cm_id(ch); 3776 if (ret) 3777 goto err_disconnect; 3778 3779 ret = srp_create_ch_ib(ch); 3780 if (ret) 3781 goto err_disconnect; 3782 3783 ret = srp_alloc_req_data(ch); 3784 if (ret) 3785 goto err_disconnect; 3786 3787 ret = srp_connect_ch(ch, max_iu_len, multich); 3788 if (ret) { 3789 char dst[64]; 3790 3791 if (target->using_rdma_cm) 3792 snprintf(dst, sizeof(dst), "%pIS", 3793 &target->rdma_cm.dst); 3794 else 3795 snprintf(dst, sizeof(dst), "%pI6", 3796 target->ib_cm.orig_dgid.raw); 3797 shost_printk(KERN_ERR, target->scsi_host, 3798 PFX "Connection %d/%d to %s failed\n", 3799 ch_idx, 3800 target->ch_count, dst); 3801 if (ch_idx == 0) { 3802 goto free_ch; 3803 } else { 3804 srp_free_ch_ib(target, ch); 3805 srp_free_req_data(target, ch); 3806 target->ch_count = ch - target->ch; 3807 goto connected; 3808 } 3809 } 3810 multich = true; 3811 } 3812 3813 connected: 3814 target->scsi_host->nr_hw_queues = target->ch_count; 3815 3816 ret = srp_add_target(host, target); 3817 if (ret) 3818 goto err_disconnect; 3819 3820 if (target->state != SRP_TARGET_REMOVED) { 3821 if (target->using_rdma_cm) { 3822 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3823 "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n", 3824 be64_to_cpu(target->id_ext), 3825 be64_to_cpu(target->ioc_guid), 3826 target->sgid.raw, &target->rdma_cm.dst); 3827 } else { 3828 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3829 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n", 3830 be64_to_cpu(target->id_ext), 3831 be64_to_cpu(target->ioc_guid), 3832 be16_to_cpu(target->ib_cm.pkey), 3833 be64_to_cpu(target->ib_cm.service_id), 3834 target->sgid.raw, 3835 target->ib_cm.orig_dgid.raw); 3836 } 3837 } 3838 3839 ret = count; 3840 3841 out: 3842 mutex_unlock(&host->add_target_mutex); 3843 3844 put: 3845 scsi_host_put(target->scsi_host); 3846 if (ret < 0) { 3847 /* 3848 * If a call to srp_remove_target() has not been scheduled, 3849 * drop the network namespace reference now that was obtained 3850 * earlier in this function. 3851 */ 3852 if (target->state != SRP_TARGET_REMOVED) 3853 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 3854 scsi_host_put(target->scsi_host); 3855 } 3856 3857 return ret; 3858 3859 err_disconnect: 3860 srp_disconnect_target(target); 3861 3862 free_ch: 3863 for (i = 0; i < target->ch_count; i++) { 3864 ch = &target->ch[i]; 3865 srp_free_ch_ib(target, ch); 3866 srp_free_req_data(target, ch); 3867 } 3868 3869 kfree(target->ch); 3870 goto out; 3871 } 3872 3873 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target); 3874 3875 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, 3876 char *buf) 3877 { 3878 struct srp_host *host = container_of(dev, struct srp_host, dev); 3879 3880 return sysfs_emit(buf, "%s\n", dev_name(&host->srp_dev->dev->dev)); 3881 } 3882 3883 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); 3884 3885 static ssize_t show_port(struct device *dev, struct device_attribute *attr, 3886 char *buf) 3887 { 3888 struct srp_host *host = container_of(dev, struct srp_host, dev); 3889 3890 return sysfs_emit(buf, "%d\n", host->port); 3891 } 3892 3893 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL); 3894 3895 static struct srp_host *srp_add_port(struct srp_device *device, u8 port) 3896 { 3897 struct srp_host *host; 3898 3899 host = kzalloc(sizeof *host, GFP_KERNEL); 3900 if (!host) 3901 return NULL; 3902 3903 INIT_LIST_HEAD(&host->target_list); 3904 spin_lock_init(&host->target_lock); 3905 init_completion(&host->released); 3906 mutex_init(&host->add_target_mutex); 3907 host->srp_dev = device; 3908 host->port = port; 3909 3910 host->dev.class = &srp_class; 3911 host->dev.parent = device->dev->dev.parent; 3912 dev_set_name(&host->dev, "srp-%s-%d", dev_name(&device->dev->dev), 3913 port); 3914 3915 if (device_register(&host->dev)) 3916 goto free_host; 3917 if (device_create_file(&host->dev, &dev_attr_add_target)) 3918 goto err_class; 3919 if (device_create_file(&host->dev, &dev_attr_ibdev)) 3920 goto err_class; 3921 if (device_create_file(&host->dev, &dev_attr_port)) 3922 goto err_class; 3923 3924 return host; 3925 3926 err_class: 3927 device_unregister(&host->dev); 3928 3929 free_host: 3930 kfree(host); 3931 3932 return NULL; 3933 } 3934 3935 static void srp_rename_dev(struct ib_device *device, void *client_data) 3936 { 3937 struct srp_device *srp_dev = client_data; 3938 struct srp_host *host, *tmp_host; 3939 3940 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 3941 char name[IB_DEVICE_NAME_MAX + 8]; 3942 3943 snprintf(name, sizeof(name), "srp-%s-%d", 3944 dev_name(&device->dev), host->port); 3945 device_rename(&host->dev, name); 3946 } 3947 } 3948 3949 static int srp_add_one(struct ib_device *device) 3950 { 3951 struct srp_device *srp_dev; 3952 struct ib_device_attr *attr = &device->attrs; 3953 struct srp_host *host; 3954 int mr_page_shift; 3955 unsigned int p; 3956 u64 max_pages_per_mr; 3957 unsigned int flags = 0; 3958 3959 srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL); 3960 if (!srp_dev) 3961 return -ENOMEM; 3962 3963 /* 3964 * Use the smallest page size supported by the HCA, down to a 3965 * minimum of 4096 bytes. We're unlikely to build large sglists 3966 * out of smaller entries. 3967 */ 3968 mr_page_shift = max(12, ffs(attr->page_size_cap) - 1); 3969 srp_dev->mr_page_size = 1 << mr_page_shift; 3970 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1); 3971 max_pages_per_mr = attr->max_mr_size; 3972 do_div(max_pages_per_mr, srp_dev->mr_page_size); 3973 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__, 3974 attr->max_mr_size, srp_dev->mr_page_size, 3975 max_pages_per_mr, SRP_MAX_PAGES_PER_MR); 3976 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR, 3977 max_pages_per_mr); 3978 3979 srp_dev->has_fr = (attr->device_cap_flags & 3980 IB_DEVICE_MEM_MGT_EXTENSIONS); 3981 if (!never_register && !srp_dev->has_fr) 3982 dev_warn(&device->dev, "FR is not supported\n"); 3983 else if (!never_register && 3984 attr->max_mr_size >= 2 * srp_dev->mr_page_size) 3985 srp_dev->use_fast_reg = srp_dev->has_fr; 3986 3987 if (never_register || !register_always || !srp_dev->has_fr) 3988 flags |= IB_PD_UNSAFE_GLOBAL_RKEY; 3989 3990 if (srp_dev->use_fast_reg) { 3991 srp_dev->max_pages_per_mr = 3992 min_t(u32, srp_dev->max_pages_per_mr, 3993 attr->max_fast_reg_page_list_len); 3994 } 3995 srp_dev->mr_max_size = srp_dev->mr_page_size * 3996 srp_dev->max_pages_per_mr; 3997 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n", 3998 dev_name(&device->dev), mr_page_shift, attr->max_mr_size, 3999 attr->max_fast_reg_page_list_len, 4000 srp_dev->max_pages_per_mr, srp_dev->mr_max_size); 4001 4002 INIT_LIST_HEAD(&srp_dev->dev_list); 4003 4004 srp_dev->dev = device; 4005 srp_dev->pd = ib_alloc_pd(device, flags); 4006 if (IS_ERR(srp_dev->pd)) { 4007 int ret = PTR_ERR(srp_dev->pd); 4008 4009 kfree(srp_dev); 4010 return ret; 4011 } 4012 4013 if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) { 4014 srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey; 4015 WARN_ON_ONCE(srp_dev->global_rkey == 0); 4016 } 4017 4018 rdma_for_each_port (device, p) { 4019 host = srp_add_port(srp_dev, p); 4020 if (host) 4021 list_add_tail(&host->list, &srp_dev->dev_list); 4022 } 4023 4024 ib_set_client_data(device, &srp_client, srp_dev); 4025 return 0; 4026 } 4027 4028 static void srp_remove_one(struct ib_device *device, void *client_data) 4029 { 4030 struct srp_device *srp_dev; 4031 struct srp_host *host, *tmp_host; 4032 struct srp_target_port *target; 4033 4034 srp_dev = client_data; 4035 4036 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 4037 device_unregister(&host->dev); 4038 /* 4039 * Wait for the sysfs entry to go away, so that no new 4040 * target ports can be created. 4041 */ 4042 wait_for_completion(&host->released); 4043 4044 /* 4045 * Remove all target ports. 4046 */ 4047 spin_lock(&host->target_lock); 4048 list_for_each_entry(target, &host->target_list, list) 4049 srp_queue_remove_work(target); 4050 spin_unlock(&host->target_lock); 4051 4052 /* 4053 * Wait for tl_err and target port removal tasks. 4054 */ 4055 flush_workqueue(system_long_wq); 4056 flush_workqueue(srp_remove_wq); 4057 4058 kfree(host); 4059 } 4060 4061 ib_dealloc_pd(srp_dev->pd); 4062 4063 kfree(srp_dev); 4064 } 4065 4066 static struct srp_function_template ib_srp_transport_functions = { 4067 .has_rport_state = true, 4068 .reset_timer_if_blocked = true, 4069 .reconnect_delay = &srp_reconnect_delay, 4070 .fast_io_fail_tmo = &srp_fast_io_fail_tmo, 4071 .dev_loss_tmo = &srp_dev_loss_tmo, 4072 .reconnect = srp_rport_reconnect, 4073 .rport_delete = srp_rport_delete, 4074 .terminate_rport_io = srp_terminate_io, 4075 }; 4076 4077 static int __init srp_init_module(void) 4078 { 4079 int ret; 4080 4081 BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 4); 4082 BUILD_BUG_ON(sizeof(struct srp_login_req) != 64); 4083 BUILD_BUG_ON(sizeof(struct srp_login_req_rdma) != 56); 4084 BUILD_BUG_ON(sizeof(struct srp_cmd) != 48); 4085 4086 if (srp_sg_tablesize) { 4087 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); 4088 if (!cmd_sg_entries) 4089 cmd_sg_entries = srp_sg_tablesize; 4090 } 4091 4092 if (!cmd_sg_entries) 4093 cmd_sg_entries = SRP_DEF_SG_TABLESIZE; 4094 4095 if (cmd_sg_entries > 255) { 4096 pr_warn("Clamping cmd_sg_entries to 255\n"); 4097 cmd_sg_entries = 255; 4098 } 4099 4100 if (!indirect_sg_entries) 4101 indirect_sg_entries = cmd_sg_entries; 4102 else if (indirect_sg_entries < cmd_sg_entries) { 4103 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n", 4104 cmd_sg_entries); 4105 indirect_sg_entries = cmd_sg_entries; 4106 } 4107 4108 if (indirect_sg_entries > SG_MAX_SEGMENTS) { 4109 pr_warn("Clamping indirect_sg_entries to %u\n", 4110 SG_MAX_SEGMENTS); 4111 indirect_sg_entries = SG_MAX_SEGMENTS; 4112 } 4113 4114 srp_remove_wq = create_workqueue("srp_remove"); 4115 if (!srp_remove_wq) { 4116 ret = -ENOMEM; 4117 goto out; 4118 } 4119 4120 ret = -ENOMEM; 4121 ib_srp_transport_template = 4122 srp_attach_transport(&ib_srp_transport_functions); 4123 if (!ib_srp_transport_template) 4124 goto destroy_wq; 4125 4126 ret = class_register(&srp_class); 4127 if (ret) { 4128 pr_err("couldn't register class infiniband_srp\n"); 4129 goto release_tr; 4130 } 4131 4132 ib_sa_register_client(&srp_sa_client); 4133 4134 ret = ib_register_client(&srp_client); 4135 if (ret) { 4136 pr_err("couldn't register IB client\n"); 4137 goto unreg_sa; 4138 } 4139 4140 out: 4141 return ret; 4142 4143 unreg_sa: 4144 ib_sa_unregister_client(&srp_sa_client); 4145 class_unregister(&srp_class); 4146 4147 release_tr: 4148 srp_release_transport(ib_srp_transport_template); 4149 4150 destroy_wq: 4151 destroy_workqueue(srp_remove_wq); 4152 goto out; 4153 } 4154 4155 static void __exit srp_cleanup_module(void) 4156 { 4157 ib_unregister_client(&srp_client); 4158 ib_sa_unregister_client(&srp_sa_client); 4159 class_unregister(&srp_class); 4160 srp_release_transport(ib_srp_transport_template); 4161 destroy_workqueue(srp_remove_wq); 4162 } 4163 4164 module_init(srp_init_module); 4165 module_exit(srp_cleanup_module); 4166