1 /* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/module.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/err.h> 39 #include <linux/string.h> 40 #include <linux/parser.h> 41 #include <linux/random.h> 42 #include <linux/jiffies.h> 43 #include <linux/lockdep.h> 44 #include <linux/inet.h> 45 #include <rdma/ib_cache.h> 46 47 #include <linux/atomic.h> 48 49 #include <scsi/scsi.h> 50 #include <scsi/scsi_device.h> 51 #include <scsi/scsi_dbg.h> 52 #include <scsi/scsi_tcq.h> 53 #include <scsi/srp.h> 54 #include <scsi/scsi_transport_srp.h> 55 56 #include "ib_srp.h" 57 58 #define DRV_NAME "ib_srp" 59 #define PFX DRV_NAME ": " 60 61 MODULE_AUTHOR("Roland Dreier"); 62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); 63 MODULE_LICENSE("Dual BSD/GPL"); 64 65 #if !defined(CONFIG_DYNAMIC_DEBUG) 66 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) 67 #define DYNAMIC_DEBUG_BRANCH(descriptor) false 68 #endif 69 70 static unsigned int srp_sg_tablesize; 71 static unsigned int cmd_sg_entries; 72 static unsigned int indirect_sg_entries; 73 static bool allow_ext_sg; 74 static bool register_always = true; 75 static bool never_register; 76 static int topspin_workarounds = 1; 77 78 module_param(srp_sg_tablesize, uint, 0444); 79 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); 80 81 module_param(cmd_sg_entries, uint, 0444); 82 MODULE_PARM_DESC(cmd_sg_entries, 83 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); 84 85 module_param(indirect_sg_entries, uint, 0444); 86 MODULE_PARM_DESC(indirect_sg_entries, 87 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")"); 88 89 module_param(allow_ext_sg, bool, 0444); 90 MODULE_PARM_DESC(allow_ext_sg, 91 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)"); 92 93 module_param(topspin_workarounds, int, 0444); 94 MODULE_PARM_DESC(topspin_workarounds, 95 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0"); 96 97 module_param(register_always, bool, 0444); 98 MODULE_PARM_DESC(register_always, 99 "Use memory registration even for contiguous memory regions"); 100 101 module_param(never_register, bool, 0444); 102 MODULE_PARM_DESC(never_register, "Never register memory"); 103 104 static const struct kernel_param_ops srp_tmo_ops; 105 106 static int srp_reconnect_delay = 10; 107 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 108 S_IRUGO | S_IWUSR); 109 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts"); 110 111 static int srp_fast_io_fail_tmo = 15; 112 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo, 113 S_IRUGO | S_IWUSR); 114 MODULE_PARM_DESC(fast_io_fail_tmo, 115 "Number of seconds between the observation of a transport" 116 " layer error and failing all I/O. \"off\" means that this" 117 " functionality is disabled."); 118 119 static int srp_dev_loss_tmo = 600; 120 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo, 121 S_IRUGO | S_IWUSR); 122 MODULE_PARM_DESC(dev_loss_tmo, 123 "Maximum number of seconds that the SRP transport should" 124 " insulate transport layer errors. After this time has been" 125 " exceeded the SCSI host is removed. Should be" 126 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT) 127 " if fast_io_fail_tmo has not been set. \"off\" means that" 128 " this functionality is disabled."); 129 130 static bool srp_use_imm_data = true; 131 module_param_named(use_imm_data, srp_use_imm_data, bool, 0644); 132 MODULE_PARM_DESC(use_imm_data, 133 "Whether or not to request permission to use immediate data during SRP login."); 134 135 static unsigned int srp_max_imm_data = 8 * 1024; 136 module_param_named(max_imm_data, srp_max_imm_data, uint, 0644); 137 MODULE_PARM_DESC(max_imm_data, "Maximum immediate data size."); 138 139 static unsigned ch_count; 140 module_param(ch_count, uint, 0444); 141 MODULE_PARM_DESC(ch_count, 142 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA."); 143 144 static int srp_add_one(struct ib_device *device); 145 static void srp_remove_one(struct ib_device *device, void *client_data); 146 static void srp_rename_dev(struct ib_device *device, void *client_data); 147 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc); 148 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 149 const char *opname); 150 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 151 const struct ib_cm_event *event); 152 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 153 struct rdma_cm_event *event); 154 155 static struct scsi_transport_template *ib_srp_transport_template; 156 static struct workqueue_struct *srp_remove_wq; 157 158 static struct ib_client srp_client = { 159 .name = "srp", 160 .add = srp_add_one, 161 .remove = srp_remove_one, 162 .rename = srp_rename_dev 163 }; 164 165 static struct ib_sa_client srp_sa_client; 166 167 static int srp_tmo_get(char *buffer, const struct kernel_param *kp) 168 { 169 int tmo = *(int *)kp->arg; 170 171 if (tmo >= 0) 172 return sysfs_emit(buffer, "%d\n", tmo); 173 else 174 return sysfs_emit(buffer, "off\n"); 175 } 176 177 static int srp_tmo_set(const char *val, const struct kernel_param *kp) 178 { 179 int tmo, res; 180 181 res = srp_parse_tmo(&tmo, val); 182 if (res) 183 goto out; 184 185 if (kp->arg == &srp_reconnect_delay) 186 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo, 187 srp_dev_loss_tmo); 188 else if (kp->arg == &srp_fast_io_fail_tmo) 189 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo); 190 else 191 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo, 192 tmo); 193 if (res) 194 goto out; 195 *(int *)kp->arg = tmo; 196 197 out: 198 return res; 199 } 200 201 static const struct kernel_param_ops srp_tmo_ops = { 202 .get = srp_tmo_get, 203 .set = srp_tmo_set, 204 }; 205 206 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) 207 { 208 return (struct srp_target_port *) host->hostdata; 209 } 210 211 static const char *srp_target_info(struct Scsi_Host *host) 212 { 213 return host_to_target(host)->target_name; 214 } 215 216 static int srp_target_is_topspin(struct srp_target_port *target) 217 { 218 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad }; 219 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d }; 220 221 return topspin_workarounds && 222 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) || 223 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui)); 224 } 225 226 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size, 227 gfp_t gfp_mask, 228 enum dma_data_direction direction) 229 { 230 struct srp_iu *iu; 231 232 iu = kmalloc(sizeof *iu, gfp_mask); 233 if (!iu) 234 goto out; 235 236 iu->buf = kzalloc(size, gfp_mask); 237 if (!iu->buf) 238 goto out_free_iu; 239 240 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size, 241 direction); 242 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma)) 243 goto out_free_buf; 244 245 iu->size = size; 246 iu->direction = direction; 247 248 return iu; 249 250 out_free_buf: 251 kfree(iu->buf); 252 out_free_iu: 253 kfree(iu); 254 out: 255 return NULL; 256 } 257 258 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu) 259 { 260 if (!iu) 261 return; 262 263 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size, 264 iu->direction); 265 kfree(iu->buf); 266 kfree(iu); 267 } 268 269 static void srp_qp_event(struct ib_event *event, void *context) 270 { 271 pr_debug("QP event %s (%d)\n", 272 ib_event_msg(event->event), event->event); 273 } 274 275 static int srp_init_ib_qp(struct srp_target_port *target, 276 struct ib_qp *qp) 277 { 278 struct ib_qp_attr *attr; 279 int ret; 280 281 attr = kmalloc(sizeof *attr, GFP_KERNEL); 282 if (!attr) 283 return -ENOMEM; 284 285 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 286 target->srp_host->port, 287 be16_to_cpu(target->ib_cm.pkey), 288 &attr->pkey_index); 289 if (ret) 290 goto out; 291 292 attr->qp_state = IB_QPS_INIT; 293 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ | 294 IB_ACCESS_REMOTE_WRITE); 295 attr->port_num = target->srp_host->port; 296 297 ret = ib_modify_qp(qp, attr, 298 IB_QP_STATE | 299 IB_QP_PKEY_INDEX | 300 IB_QP_ACCESS_FLAGS | 301 IB_QP_PORT); 302 303 out: 304 kfree(attr); 305 return ret; 306 } 307 308 static int srp_new_ib_cm_id(struct srp_rdma_ch *ch) 309 { 310 struct srp_target_port *target = ch->target; 311 struct ib_cm_id *new_cm_id; 312 313 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, 314 srp_ib_cm_handler, ch); 315 if (IS_ERR(new_cm_id)) 316 return PTR_ERR(new_cm_id); 317 318 if (ch->ib_cm.cm_id) 319 ib_destroy_cm_id(ch->ib_cm.cm_id); 320 ch->ib_cm.cm_id = new_cm_id; 321 if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev, 322 target->srp_host->port)) 323 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA; 324 else 325 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB; 326 ch->ib_cm.path.sgid = target->sgid; 327 ch->ib_cm.path.dgid = target->ib_cm.orig_dgid; 328 ch->ib_cm.path.pkey = target->ib_cm.pkey; 329 ch->ib_cm.path.service_id = target->ib_cm.service_id; 330 331 return 0; 332 } 333 334 static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch) 335 { 336 struct srp_target_port *target = ch->target; 337 struct rdma_cm_id *new_cm_id; 338 int ret; 339 340 new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch, 341 RDMA_PS_TCP, IB_QPT_RC); 342 if (IS_ERR(new_cm_id)) { 343 ret = PTR_ERR(new_cm_id); 344 new_cm_id = NULL; 345 goto out; 346 } 347 348 init_completion(&ch->done); 349 ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ? 350 &target->rdma_cm.src.sa : NULL, 351 &target->rdma_cm.dst.sa, 352 SRP_PATH_REC_TIMEOUT_MS); 353 if (ret) { 354 pr_err("No route available from %pISpsc to %pISpsc (%d)\n", 355 &target->rdma_cm.src, &target->rdma_cm.dst, ret); 356 goto out; 357 } 358 ret = wait_for_completion_interruptible(&ch->done); 359 if (ret < 0) 360 goto out; 361 362 ret = ch->status; 363 if (ret) { 364 pr_err("Resolving address %pISpsc failed (%d)\n", 365 &target->rdma_cm.dst, ret); 366 goto out; 367 } 368 369 swap(ch->rdma_cm.cm_id, new_cm_id); 370 371 out: 372 if (new_cm_id) 373 rdma_destroy_id(new_cm_id); 374 375 return ret; 376 } 377 378 static int srp_new_cm_id(struct srp_rdma_ch *ch) 379 { 380 struct srp_target_port *target = ch->target; 381 382 return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) : 383 srp_new_ib_cm_id(ch); 384 } 385 386 /** 387 * srp_destroy_fr_pool() - free the resources owned by a pool 388 * @pool: Fast registration pool to be destroyed. 389 */ 390 static void srp_destroy_fr_pool(struct srp_fr_pool *pool) 391 { 392 int i; 393 struct srp_fr_desc *d; 394 395 if (!pool) 396 return; 397 398 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 399 if (d->mr) 400 ib_dereg_mr(d->mr); 401 } 402 kfree(pool); 403 } 404 405 /** 406 * srp_create_fr_pool() - allocate and initialize a pool for fast registration 407 * @device: IB device to allocate fast registration descriptors for. 408 * @pd: Protection domain associated with the FR descriptors. 409 * @pool_size: Number of descriptors to allocate. 410 * @max_page_list_len: Maximum fast registration work request page list length. 411 */ 412 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, 413 struct ib_pd *pd, int pool_size, 414 int max_page_list_len) 415 { 416 struct srp_fr_pool *pool; 417 struct srp_fr_desc *d; 418 struct ib_mr *mr; 419 int i, ret = -EINVAL; 420 enum ib_mr_type mr_type; 421 422 if (pool_size <= 0) 423 goto err; 424 ret = -ENOMEM; 425 pool = kzalloc(struct_size(pool, desc, pool_size), GFP_KERNEL); 426 if (!pool) 427 goto err; 428 pool->size = pool_size; 429 pool->max_page_list_len = max_page_list_len; 430 spin_lock_init(&pool->lock); 431 INIT_LIST_HEAD(&pool->free_list); 432 433 if (device->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) 434 mr_type = IB_MR_TYPE_SG_GAPS; 435 else 436 mr_type = IB_MR_TYPE_MEM_REG; 437 438 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 439 mr = ib_alloc_mr(pd, mr_type, max_page_list_len); 440 if (IS_ERR(mr)) { 441 ret = PTR_ERR(mr); 442 if (ret == -ENOMEM) 443 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n", 444 dev_name(&device->dev)); 445 goto destroy_pool; 446 } 447 d->mr = mr; 448 list_add_tail(&d->entry, &pool->free_list); 449 } 450 451 out: 452 return pool; 453 454 destroy_pool: 455 srp_destroy_fr_pool(pool); 456 457 err: 458 pool = ERR_PTR(ret); 459 goto out; 460 } 461 462 /** 463 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration 464 * @pool: Pool to obtain descriptor from. 465 */ 466 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool) 467 { 468 struct srp_fr_desc *d = NULL; 469 unsigned long flags; 470 471 spin_lock_irqsave(&pool->lock, flags); 472 if (!list_empty(&pool->free_list)) { 473 d = list_first_entry(&pool->free_list, typeof(*d), entry); 474 list_del(&d->entry); 475 } 476 spin_unlock_irqrestore(&pool->lock, flags); 477 478 return d; 479 } 480 481 /** 482 * srp_fr_pool_put() - put an FR descriptor back in the free list 483 * @pool: Pool the descriptor was allocated from. 484 * @desc: Pointer to an array of fast registration descriptor pointers. 485 * @n: Number of descriptors to put back. 486 * 487 * Note: The caller must already have queued an invalidation request for 488 * desc->mr->rkey before calling this function. 489 */ 490 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc, 491 int n) 492 { 493 unsigned long flags; 494 int i; 495 496 spin_lock_irqsave(&pool->lock, flags); 497 for (i = 0; i < n; i++) 498 list_add(&desc[i]->entry, &pool->free_list); 499 spin_unlock_irqrestore(&pool->lock, flags); 500 } 501 502 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) 503 { 504 struct srp_device *dev = target->srp_host->srp_dev; 505 506 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size, 507 dev->max_pages_per_mr); 508 } 509 510 /** 511 * srp_destroy_qp() - destroy an RDMA queue pair 512 * @ch: SRP RDMA channel. 513 * 514 * Drain the qp before destroying it. This avoids that the receive 515 * completion handler can access the queue pair while it is 516 * being destroyed. 517 */ 518 static void srp_destroy_qp(struct srp_rdma_ch *ch) 519 { 520 spin_lock_irq(&ch->lock); 521 ib_process_cq_direct(ch->send_cq, -1); 522 spin_unlock_irq(&ch->lock); 523 524 ib_drain_qp(ch->qp); 525 ib_destroy_qp(ch->qp); 526 } 527 528 static int srp_create_ch_ib(struct srp_rdma_ch *ch) 529 { 530 struct srp_target_port *target = ch->target; 531 struct srp_device *dev = target->srp_host->srp_dev; 532 const struct ib_device_attr *attr = &dev->dev->attrs; 533 struct ib_qp_init_attr *init_attr; 534 struct ib_cq *recv_cq, *send_cq; 535 struct ib_qp *qp; 536 struct srp_fr_pool *fr_pool = NULL; 537 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2; 538 int ret; 539 540 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL); 541 if (!init_attr) 542 return -ENOMEM; 543 544 /* queue_size + 1 for ib_drain_rq() */ 545 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1, 546 ch->comp_vector, IB_POLL_SOFTIRQ); 547 if (IS_ERR(recv_cq)) { 548 ret = PTR_ERR(recv_cq); 549 goto err; 550 } 551 552 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size, 553 ch->comp_vector, IB_POLL_DIRECT); 554 if (IS_ERR(send_cq)) { 555 ret = PTR_ERR(send_cq); 556 goto err_recv_cq; 557 } 558 559 init_attr->event_handler = srp_qp_event; 560 init_attr->cap.max_send_wr = m * target->queue_size; 561 init_attr->cap.max_recv_wr = target->queue_size + 1; 562 init_attr->cap.max_recv_sge = 1; 563 init_attr->cap.max_send_sge = min(SRP_MAX_SGE, attr->max_send_sge); 564 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 565 init_attr->qp_type = IB_QPT_RC; 566 init_attr->send_cq = send_cq; 567 init_attr->recv_cq = recv_cq; 568 569 ch->max_imm_sge = min(init_attr->cap.max_send_sge - 1U, 255U); 570 571 if (target->using_rdma_cm) { 572 ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr); 573 qp = ch->rdma_cm.cm_id->qp; 574 } else { 575 qp = ib_create_qp(dev->pd, init_attr); 576 if (!IS_ERR(qp)) { 577 ret = srp_init_ib_qp(target, qp); 578 if (ret) 579 ib_destroy_qp(qp); 580 } else { 581 ret = PTR_ERR(qp); 582 } 583 } 584 if (ret) { 585 pr_err("QP creation failed for dev %s: %d\n", 586 dev_name(&dev->dev->dev), ret); 587 goto err_send_cq; 588 } 589 590 if (dev->use_fast_reg) { 591 fr_pool = srp_alloc_fr_pool(target); 592 if (IS_ERR(fr_pool)) { 593 ret = PTR_ERR(fr_pool); 594 shost_printk(KERN_WARNING, target->scsi_host, PFX 595 "FR pool allocation failed (%d)\n", ret); 596 goto err_qp; 597 } 598 } 599 600 if (ch->qp) 601 srp_destroy_qp(ch); 602 if (ch->recv_cq) 603 ib_free_cq(ch->recv_cq); 604 if (ch->send_cq) 605 ib_free_cq(ch->send_cq); 606 607 ch->qp = qp; 608 ch->recv_cq = recv_cq; 609 ch->send_cq = send_cq; 610 611 if (dev->use_fast_reg) { 612 if (ch->fr_pool) 613 srp_destroy_fr_pool(ch->fr_pool); 614 ch->fr_pool = fr_pool; 615 } 616 617 kfree(init_attr); 618 return 0; 619 620 err_qp: 621 if (target->using_rdma_cm) 622 rdma_destroy_qp(ch->rdma_cm.cm_id); 623 else 624 ib_destroy_qp(qp); 625 626 err_send_cq: 627 ib_free_cq(send_cq); 628 629 err_recv_cq: 630 ib_free_cq(recv_cq); 631 632 err: 633 kfree(init_attr); 634 return ret; 635 } 636 637 /* 638 * Note: this function may be called without srp_alloc_iu_bufs() having been 639 * invoked. Hence the ch->[rt]x_ring checks. 640 */ 641 static void srp_free_ch_ib(struct srp_target_port *target, 642 struct srp_rdma_ch *ch) 643 { 644 struct srp_device *dev = target->srp_host->srp_dev; 645 int i; 646 647 if (!ch->target) 648 return; 649 650 if (target->using_rdma_cm) { 651 if (ch->rdma_cm.cm_id) { 652 rdma_destroy_id(ch->rdma_cm.cm_id); 653 ch->rdma_cm.cm_id = NULL; 654 } 655 } else { 656 if (ch->ib_cm.cm_id) { 657 ib_destroy_cm_id(ch->ib_cm.cm_id); 658 ch->ib_cm.cm_id = NULL; 659 } 660 } 661 662 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */ 663 if (!ch->qp) 664 return; 665 666 if (dev->use_fast_reg) { 667 if (ch->fr_pool) 668 srp_destroy_fr_pool(ch->fr_pool); 669 } 670 671 srp_destroy_qp(ch); 672 ib_free_cq(ch->send_cq); 673 ib_free_cq(ch->recv_cq); 674 675 /* 676 * Avoid that the SCSI error handler tries to use this channel after 677 * it has been freed. The SCSI error handler can namely continue 678 * trying to perform recovery actions after scsi_remove_host() 679 * returned. 680 */ 681 ch->target = NULL; 682 683 ch->qp = NULL; 684 ch->send_cq = ch->recv_cq = NULL; 685 686 if (ch->rx_ring) { 687 for (i = 0; i < target->queue_size; ++i) 688 srp_free_iu(target->srp_host, ch->rx_ring[i]); 689 kfree(ch->rx_ring); 690 ch->rx_ring = NULL; 691 } 692 if (ch->tx_ring) { 693 for (i = 0; i < target->queue_size; ++i) 694 srp_free_iu(target->srp_host, ch->tx_ring[i]); 695 kfree(ch->tx_ring); 696 ch->tx_ring = NULL; 697 } 698 } 699 700 static void srp_path_rec_completion(int status, 701 struct sa_path_rec *pathrec, 702 int num_paths, void *ch_ptr) 703 { 704 struct srp_rdma_ch *ch = ch_ptr; 705 struct srp_target_port *target = ch->target; 706 707 ch->status = status; 708 if (status) 709 shost_printk(KERN_ERR, target->scsi_host, 710 PFX "Got failed path rec status %d\n", status); 711 else 712 ch->ib_cm.path = *pathrec; 713 complete(&ch->done); 714 } 715 716 static int srp_ib_lookup_path(struct srp_rdma_ch *ch) 717 { 718 struct srp_target_port *target = ch->target; 719 int ret; 720 721 ch->ib_cm.path.numb_path = 1; 722 723 init_completion(&ch->done); 724 725 ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client, 726 target->srp_host->srp_dev->dev, 727 target->srp_host->port, 728 &ch->ib_cm.path, 729 IB_SA_PATH_REC_SERVICE_ID | 730 IB_SA_PATH_REC_DGID | 731 IB_SA_PATH_REC_SGID | 732 IB_SA_PATH_REC_NUMB_PATH | 733 IB_SA_PATH_REC_PKEY, 734 SRP_PATH_REC_TIMEOUT_MS, 735 GFP_KERNEL, 736 srp_path_rec_completion, 737 ch, &ch->ib_cm.path_query); 738 if (ch->ib_cm.path_query_id < 0) 739 return ch->ib_cm.path_query_id; 740 741 ret = wait_for_completion_interruptible(&ch->done); 742 if (ret < 0) 743 return ret; 744 745 if (ch->status < 0) 746 shost_printk(KERN_WARNING, target->scsi_host, 747 PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n", 748 ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw, 749 be16_to_cpu(target->ib_cm.pkey), 750 be64_to_cpu(target->ib_cm.service_id)); 751 752 return ch->status; 753 } 754 755 static int srp_rdma_lookup_path(struct srp_rdma_ch *ch) 756 { 757 struct srp_target_port *target = ch->target; 758 int ret; 759 760 init_completion(&ch->done); 761 762 ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS); 763 if (ret) 764 return ret; 765 766 wait_for_completion_interruptible(&ch->done); 767 768 if (ch->status != 0) 769 shost_printk(KERN_WARNING, target->scsi_host, 770 PFX "Path resolution failed\n"); 771 772 return ch->status; 773 } 774 775 static int srp_lookup_path(struct srp_rdma_ch *ch) 776 { 777 struct srp_target_port *target = ch->target; 778 779 return target->using_rdma_cm ? srp_rdma_lookup_path(ch) : 780 srp_ib_lookup_path(ch); 781 } 782 783 static u8 srp_get_subnet_timeout(struct srp_host *host) 784 { 785 struct ib_port_attr attr; 786 int ret; 787 u8 subnet_timeout = 18; 788 789 ret = ib_query_port(host->srp_dev->dev, host->port, &attr); 790 if (ret == 0) 791 subnet_timeout = attr.subnet_timeout; 792 793 if (unlikely(subnet_timeout < 15)) 794 pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n", 795 dev_name(&host->srp_dev->dev->dev), subnet_timeout); 796 797 return subnet_timeout; 798 } 799 800 static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len, 801 bool multich) 802 { 803 struct srp_target_port *target = ch->target; 804 struct { 805 struct rdma_conn_param rdma_param; 806 struct srp_login_req_rdma rdma_req; 807 struct ib_cm_req_param ib_param; 808 struct srp_login_req ib_req; 809 } *req = NULL; 810 char *ipi, *tpi; 811 int status; 812 813 req = kzalloc(sizeof *req, GFP_KERNEL); 814 if (!req) 815 return -ENOMEM; 816 817 req->ib_param.flow_control = 1; 818 req->ib_param.retry_count = target->tl_retry_count; 819 820 /* 821 * Pick some arbitrary defaults here; we could make these 822 * module parameters if anyone cared about setting them. 823 */ 824 req->ib_param.responder_resources = 4; 825 req->ib_param.rnr_retry_count = 7; 826 req->ib_param.max_cm_retries = 15; 827 828 req->ib_req.opcode = SRP_LOGIN_REQ; 829 req->ib_req.tag = 0; 830 req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len); 831 req->ib_req.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | 832 SRP_BUF_FORMAT_INDIRECT); 833 req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI : 834 SRP_MULTICHAN_SINGLE); 835 if (srp_use_imm_data) { 836 req->ib_req.req_flags |= SRP_IMMED_REQUESTED; 837 req->ib_req.imm_data_offset = cpu_to_be16(SRP_IMM_DATA_OFFSET); 838 } 839 840 if (target->using_rdma_cm) { 841 req->rdma_param.flow_control = req->ib_param.flow_control; 842 req->rdma_param.responder_resources = 843 req->ib_param.responder_resources; 844 req->rdma_param.initiator_depth = req->ib_param.initiator_depth; 845 req->rdma_param.retry_count = req->ib_param.retry_count; 846 req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count; 847 req->rdma_param.private_data = &req->rdma_req; 848 req->rdma_param.private_data_len = sizeof(req->rdma_req); 849 850 req->rdma_req.opcode = req->ib_req.opcode; 851 req->rdma_req.tag = req->ib_req.tag; 852 req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len; 853 req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt; 854 req->rdma_req.req_flags = req->ib_req.req_flags; 855 req->rdma_req.imm_data_offset = req->ib_req.imm_data_offset; 856 857 ipi = req->rdma_req.initiator_port_id; 858 tpi = req->rdma_req.target_port_id; 859 } else { 860 u8 subnet_timeout; 861 862 subnet_timeout = srp_get_subnet_timeout(target->srp_host); 863 864 req->ib_param.primary_path = &ch->ib_cm.path; 865 req->ib_param.alternate_path = NULL; 866 req->ib_param.service_id = target->ib_cm.service_id; 867 get_random_bytes(&req->ib_param.starting_psn, 4); 868 req->ib_param.starting_psn &= 0xffffff; 869 req->ib_param.qp_num = ch->qp->qp_num; 870 req->ib_param.qp_type = ch->qp->qp_type; 871 req->ib_param.local_cm_response_timeout = subnet_timeout + 2; 872 req->ib_param.remote_cm_response_timeout = subnet_timeout + 2; 873 req->ib_param.private_data = &req->ib_req; 874 req->ib_param.private_data_len = sizeof(req->ib_req); 875 876 ipi = req->ib_req.initiator_port_id; 877 tpi = req->ib_req.target_port_id; 878 } 879 880 /* 881 * In the published SRP specification (draft rev. 16a), the 882 * port identifier format is 8 bytes of ID extension followed 883 * by 8 bytes of GUID. Older drafts put the two halves in the 884 * opposite order, so that the GUID comes first. 885 * 886 * Targets conforming to these obsolete drafts can be 887 * recognized by the I/O Class they report. 888 */ 889 if (target->io_class == SRP_REV10_IB_IO_CLASS) { 890 memcpy(ipi, &target->sgid.global.interface_id, 8); 891 memcpy(ipi + 8, &target->initiator_ext, 8); 892 memcpy(tpi, &target->ioc_guid, 8); 893 memcpy(tpi + 8, &target->id_ext, 8); 894 } else { 895 memcpy(ipi, &target->initiator_ext, 8); 896 memcpy(ipi + 8, &target->sgid.global.interface_id, 8); 897 memcpy(tpi, &target->id_ext, 8); 898 memcpy(tpi + 8, &target->ioc_guid, 8); 899 } 900 901 /* 902 * Topspin/Cisco SRP targets will reject our login unless we 903 * zero out the first 8 bytes of our initiator port ID and set 904 * the second 8 bytes to the local node GUID. 905 */ 906 if (srp_target_is_topspin(target)) { 907 shost_printk(KERN_DEBUG, target->scsi_host, 908 PFX "Topspin/Cisco initiator port ID workaround " 909 "activated for target GUID %016llx\n", 910 be64_to_cpu(target->ioc_guid)); 911 memset(ipi, 0, 8); 912 memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8); 913 } 914 915 if (target->using_rdma_cm) 916 status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param); 917 else 918 status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param); 919 920 kfree(req); 921 922 return status; 923 } 924 925 static bool srp_queue_remove_work(struct srp_target_port *target) 926 { 927 bool changed = false; 928 929 spin_lock_irq(&target->lock); 930 if (target->state != SRP_TARGET_REMOVED) { 931 target->state = SRP_TARGET_REMOVED; 932 changed = true; 933 } 934 spin_unlock_irq(&target->lock); 935 936 if (changed) 937 queue_work(srp_remove_wq, &target->remove_work); 938 939 return changed; 940 } 941 942 static void srp_disconnect_target(struct srp_target_port *target) 943 { 944 struct srp_rdma_ch *ch; 945 int i, ret; 946 947 /* XXX should send SRP_I_LOGOUT request */ 948 949 for (i = 0; i < target->ch_count; i++) { 950 ch = &target->ch[i]; 951 ch->connected = false; 952 ret = 0; 953 if (target->using_rdma_cm) { 954 if (ch->rdma_cm.cm_id) 955 rdma_disconnect(ch->rdma_cm.cm_id); 956 } else { 957 if (ch->ib_cm.cm_id) 958 ret = ib_send_cm_dreq(ch->ib_cm.cm_id, 959 NULL, 0); 960 } 961 if (ret < 0) { 962 shost_printk(KERN_DEBUG, target->scsi_host, 963 PFX "Sending CM DREQ failed\n"); 964 } 965 } 966 } 967 968 static int srp_exit_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 969 { 970 struct srp_target_port *target = host_to_target(shost); 971 struct srp_device *dev = target->srp_host->srp_dev; 972 struct ib_device *ibdev = dev->dev; 973 struct srp_request *req = scsi_cmd_priv(cmd); 974 975 kfree(req->fr_list); 976 if (req->indirect_dma_addr) { 977 ib_dma_unmap_single(ibdev, req->indirect_dma_addr, 978 target->indirect_size, 979 DMA_TO_DEVICE); 980 } 981 kfree(req->indirect_desc); 982 983 return 0; 984 } 985 986 static int srp_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 987 { 988 struct srp_target_port *target = host_to_target(shost); 989 struct srp_device *srp_dev = target->srp_host->srp_dev; 990 struct ib_device *ibdev = srp_dev->dev; 991 struct srp_request *req = scsi_cmd_priv(cmd); 992 dma_addr_t dma_addr; 993 int ret = -ENOMEM; 994 995 if (srp_dev->use_fast_reg) { 996 req->fr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *), 997 GFP_KERNEL); 998 if (!req->fr_list) 999 goto out; 1000 } 1001 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL); 1002 if (!req->indirect_desc) 1003 goto out; 1004 1005 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc, 1006 target->indirect_size, 1007 DMA_TO_DEVICE); 1008 if (ib_dma_mapping_error(ibdev, dma_addr)) { 1009 srp_exit_cmd_priv(shost, cmd); 1010 goto out; 1011 } 1012 1013 req->indirect_dma_addr = dma_addr; 1014 ret = 0; 1015 1016 out: 1017 return ret; 1018 } 1019 1020 /** 1021 * srp_del_scsi_host_attr() - Remove attributes defined in the host template. 1022 * @shost: SCSI host whose attributes to remove from sysfs. 1023 * 1024 * Note: Any attributes defined in the host template and that did not exist 1025 * before invocation of this function will be ignored. 1026 */ 1027 static void srp_del_scsi_host_attr(struct Scsi_Host *shost) 1028 { 1029 const struct attribute_group **g; 1030 struct attribute **attr; 1031 1032 for (g = shost->hostt->shost_groups; *g; ++g) { 1033 for (attr = (*g)->attrs; *attr; ++attr) { 1034 struct device_attribute *dev_attr = 1035 container_of(*attr, typeof(*dev_attr), attr); 1036 1037 device_remove_file(&shost->shost_dev, dev_attr); 1038 } 1039 } 1040 } 1041 1042 static void srp_remove_target(struct srp_target_port *target) 1043 { 1044 struct srp_rdma_ch *ch; 1045 int i; 1046 1047 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1048 1049 srp_del_scsi_host_attr(target->scsi_host); 1050 srp_rport_get(target->rport); 1051 srp_remove_host(target->scsi_host); 1052 scsi_remove_host(target->scsi_host); 1053 srp_stop_rport_timers(target->rport); 1054 srp_disconnect_target(target); 1055 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 1056 for (i = 0; i < target->ch_count; i++) { 1057 ch = &target->ch[i]; 1058 srp_free_ch_ib(target, ch); 1059 } 1060 cancel_work_sync(&target->tl_err_work); 1061 srp_rport_put(target->rport); 1062 kfree(target->ch); 1063 target->ch = NULL; 1064 1065 spin_lock(&target->srp_host->target_lock); 1066 list_del(&target->list); 1067 spin_unlock(&target->srp_host->target_lock); 1068 1069 scsi_host_put(target->scsi_host); 1070 } 1071 1072 static void srp_remove_work(struct work_struct *work) 1073 { 1074 struct srp_target_port *target = 1075 container_of(work, struct srp_target_port, remove_work); 1076 1077 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1078 1079 srp_remove_target(target); 1080 } 1081 1082 static void srp_rport_delete(struct srp_rport *rport) 1083 { 1084 struct srp_target_port *target = rport->lld_data; 1085 1086 srp_queue_remove_work(target); 1087 } 1088 1089 /** 1090 * srp_connected_ch() - number of connected channels 1091 * @target: SRP target port. 1092 */ 1093 static int srp_connected_ch(struct srp_target_port *target) 1094 { 1095 int i, c = 0; 1096 1097 for (i = 0; i < target->ch_count; i++) 1098 c += target->ch[i].connected; 1099 1100 return c; 1101 } 1102 1103 static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len, 1104 bool multich) 1105 { 1106 struct srp_target_port *target = ch->target; 1107 int ret; 1108 1109 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0); 1110 1111 ret = srp_lookup_path(ch); 1112 if (ret) 1113 goto out; 1114 1115 while (1) { 1116 init_completion(&ch->done); 1117 ret = srp_send_req(ch, max_iu_len, multich); 1118 if (ret) 1119 goto out; 1120 ret = wait_for_completion_interruptible(&ch->done); 1121 if (ret < 0) 1122 goto out; 1123 1124 /* 1125 * The CM event handling code will set status to 1126 * SRP_PORT_REDIRECT if we get a port redirect REJ 1127 * back, or SRP_DLID_REDIRECT if we get a lid/qp 1128 * redirect REJ back. 1129 */ 1130 ret = ch->status; 1131 switch (ret) { 1132 case 0: 1133 ch->connected = true; 1134 goto out; 1135 1136 case SRP_PORT_REDIRECT: 1137 ret = srp_lookup_path(ch); 1138 if (ret) 1139 goto out; 1140 break; 1141 1142 case SRP_DLID_REDIRECT: 1143 break; 1144 1145 case SRP_STALE_CONN: 1146 shost_printk(KERN_ERR, target->scsi_host, PFX 1147 "giving up on stale connection\n"); 1148 ret = -ECONNRESET; 1149 goto out; 1150 1151 default: 1152 goto out; 1153 } 1154 } 1155 1156 out: 1157 return ret <= 0 ? ret : -ENODEV; 1158 } 1159 1160 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc) 1161 { 1162 srp_handle_qp_err(cq, wc, "INV RKEY"); 1163 } 1164 1165 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch, 1166 u32 rkey) 1167 { 1168 struct ib_send_wr wr = { 1169 .opcode = IB_WR_LOCAL_INV, 1170 .next = NULL, 1171 .num_sge = 0, 1172 .send_flags = 0, 1173 .ex.invalidate_rkey = rkey, 1174 }; 1175 1176 wr.wr_cqe = &req->reg_cqe; 1177 req->reg_cqe.done = srp_inv_rkey_err_done; 1178 return ib_post_send(ch->qp, &wr, NULL); 1179 } 1180 1181 static void srp_unmap_data(struct scsi_cmnd *scmnd, 1182 struct srp_rdma_ch *ch, 1183 struct srp_request *req) 1184 { 1185 struct srp_target_port *target = ch->target; 1186 struct srp_device *dev = target->srp_host->srp_dev; 1187 struct ib_device *ibdev = dev->dev; 1188 int i, res; 1189 1190 if (!scsi_sglist(scmnd) || 1191 (scmnd->sc_data_direction != DMA_TO_DEVICE && 1192 scmnd->sc_data_direction != DMA_FROM_DEVICE)) 1193 return; 1194 1195 if (dev->use_fast_reg) { 1196 struct srp_fr_desc **pfr; 1197 1198 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { 1199 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey); 1200 if (res < 0) { 1201 shost_printk(KERN_ERR, target->scsi_host, PFX 1202 "Queueing INV WR for rkey %#x failed (%d)\n", 1203 (*pfr)->mr->rkey, res); 1204 queue_work(system_long_wq, 1205 &target->tl_err_work); 1206 } 1207 } 1208 if (req->nmdesc) 1209 srp_fr_pool_put(ch->fr_pool, req->fr_list, 1210 req->nmdesc); 1211 } 1212 1213 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd), 1214 scmnd->sc_data_direction); 1215 } 1216 1217 /** 1218 * srp_claim_req - Take ownership of the scmnd associated with a request. 1219 * @ch: SRP RDMA channel. 1220 * @req: SRP request. 1221 * @sdev: If not NULL, only take ownership for this SCSI device. 1222 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take 1223 * ownership of @req->scmnd if it equals @scmnd. 1224 * 1225 * Return value: 1226 * Either NULL or a pointer to the SCSI command the caller became owner of. 1227 */ 1228 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch, 1229 struct srp_request *req, 1230 struct scsi_device *sdev, 1231 struct scsi_cmnd *scmnd) 1232 { 1233 unsigned long flags; 1234 1235 spin_lock_irqsave(&ch->lock, flags); 1236 if (req->scmnd && 1237 (!sdev || req->scmnd->device == sdev) && 1238 (!scmnd || req->scmnd == scmnd)) { 1239 scmnd = req->scmnd; 1240 req->scmnd = NULL; 1241 } else { 1242 scmnd = NULL; 1243 } 1244 spin_unlock_irqrestore(&ch->lock, flags); 1245 1246 return scmnd; 1247 } 1248 1249 /** 1250 * srp_free_req() - Unmap data and adjust ch->req_lim. 1251 * @ch: SRP RDMA channel. 1252 * @req: Request to be freed. 1253 * @scmnd: SCSI command associated with @req. 1254 * @req_lim_delta: Amount to be added to @target->req_lim. 1255 */ 1256 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req, 1257 struct scsi_cmnd *scmnd, s32 req_lim_delta) 1258 { 1259 unsigned long flags; 1260 1261 srp_unmap_data(scmnd, ch, req); 1262 1263 spin_lock_irqsave(&ch->lock, flags); 1264 ch->req_lim += req_lim_delta; 1265 spin_unlock_irqrestore(&ch->lock, flags); 1266 } 1267 1268 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req, 1269 struct scsi_device *sdev, int result) 1270 { 1271 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL); 1272 1273 if (scmnd) { 1274 srp_free_req(ch, req, scmnd, 0); 1275 scmnd->result = result; 1276 scsi_done(scmnd); 1277 } 1278 } 1279 1280 struct srp_terminate_context { 1281 struct srp_target_port *srp_target; 1282 int scsi_result; 1283 }; 1284 1285 static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr) 1286 { 1287 struct srp_terminate_context *context = context_ptr; 1288 struct srp_target_port *target = context->srp_target; 1289 u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); 1290 struct srp_rdma_ch *ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 1291 struct srp_request *req = scsi_cmd_priv(scmnd); 1292 1293 srp_finish_req(ch, req, NULL, context->scsi_result); 1294 1295 return true; 1296 } 1297 1298 static void srp_terminate_io(struct srp_rport *rport) 1299 { 1300 struct srp_target_port *target = rport->lld_data; 1301 struct srp_terminate_context context = { .srp_target = target, 1302 .scsi_result = DID_TRANSPORT_FAILFAST << 16 }; 1303 1304 scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, &context); 1305 } 1306 1307 /* Calculate maximum initiator to target information unit length. */ 1308 static uint32_t srp_max_it_iu_len(int cmd_sg_cnt, bool use_imm_data, 1309 uint32_t max_it_iu_size) 1310 { 1311 uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN + 1312 sizeof(struct srp_indirect_buf) + 1313 cmd_sg_cnt * sizeof(struct srp_direct_buf); 1314 1315 if (use_imm_data) 1316 max_iu_len = max(max_iu_len, SRP_IMM_DATA_OFFSET + 1317 srp_max_imm_data); 1318 1319 if (max_it_iu_size) 1320 max_iu_len = min(max_iu_len, max_it_iu_size); 1321 1322 pr_debug("max_iu_len = %d\n", max_iu_len); 1323 1324 return max_iu_len; 1325 } 1326 1327 /* 1328 * It is up to the caller to ensure that srp_rport_reconnect() calls are 1329 * serialized and that no concurrent srp_queuecommand(), srp_abort(), 1330 * srp_reset_device() or srp_reset_host() calls will occur while this function 1331 * is in progress. One way to realize that is not to call this function 1332 * directly but to call srp_reconnect_rport() instead since that last function 1333 * serializes calls of this function via rport->mutex and also blocks 1334 * srp_queuecommand() calls before invoking this function. 1335 */ 1336 static int srp_rport_reconnect(struct srp_rport *rport) 1337 { 1338 struct srp_target_port *target = rport->lld_data; 1339 struct srp_rdma_ch *ch; 1340 uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 1341 srp_use_imm_data, 1342 target->max_it_iu_size); 1343 int i, j, ret = 0; 1344 bool multich = false; 1345 1346 srp_disconnect_target(target); 1347 1348 if (target->state == SRP_TARGET_SCANNING) 1349 return -ENODEV; 1350 1351 /* 1352 * Now get a new local CM ID so that we avoid confusing the target in 1353 * case things are really fouled up. Doing so also ensures that all CM 1354 * callbacks will have finished before a new QP is allocated. 1355 */ 1356 for (i = 0; i < target->ch_count; i++) { 1357 ch = &target->ch[i]; 1358 ret += srp_new_cm_id(ch); 1359 } 1360 { 1361 struct srp_terminate_context context = { 1362 .srp_target = target, .scsi_result = DID_RESET << 16}; 1363 1364 scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, 1365 &context); 1366 } 1367 for (i = 0; i < target->ch_count; i++) { 1368 ch = &target->ch[i]; 1369 /* 1370 * Whether or not creating a new CM ID succeeded, create a new 1371 * QP. This guarantees that all completion callback function 1372 * invocations have finished before request resetting starts. 1373 */ 1374 ret += srp_create_ch_ib(ch); 1375 1376 INIT_LIST_HEAD(&ch->free_tx); 1377 for (j = 0; j < target->queue_size; ++j) 1378 list_add(&ch->tx_ring[j]->list, &ch->free_tx); 1379 } 1380 1381 target->qp_in_error = false; 1382 1383 for (i = 0; i < target->ch_count; i++) { 1384 ch = &target->ch[i]; 1385 if (ret) 1386 break; 1387 ret = srp_connect_ch(ch, max_iu_len, multich); 1388 multich = true; 1389 } 1390 1391 if (ret == 0) 1392 shost_printk(KERN_INFO, target->scsi_host, 1393 PFX "reconnect succeeded\n"); 1394 1395 return ret; 1396 } 1397 1398 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, 1399 unsigned int dma_len, u32 rkey) 1400 { 1401 struct srp_direct_buf *desc = state->desc; 1402 1403 WARN_ON_ONCE(!dma_len); 1404 1405 desc->va = cpu_to_be64(dma_addr); 1406 desc->key = cpu_to_be32(rkey); 1407 desc->len = cpu_to_be32(dma_len); 1408 1409 state->total_len += dma_len; 1410 state->desc++; 1411 state->ndesc++; 1412 } 1413 1414 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc) 1415 { 1416 srp_handle_qp_err(cq, wc, "FAST REG"); 1417 } 1418 1419 /* 1420 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset 1421 * where to start in the first element. If sg_offset_p != NULL then 1422 * *sg_offset_p is updated to the offset in state->sg[retval] of the first 1423 * byte that has not yet been mapped. 1424 */ 1425 static int srp_map_finish_fr(struct srp_map_state *state, 1426 struct srp_request *req, 1427 struct srp_rdma_ch *ch, int sg_nents, 1428 unsigned int *sg_offset_p) 1429 { 1430 struct srp_target_port *target = ch->target; 1431 struct srp_device *dev = target->srp_host->srp_dev; 1432 struct ib_reg_wr wr; 1433 struct srp_fr_desc *desc; 1434 u32 rkey; 1435 int n, err; 1436 1437 if (state->fr.next >= state->fr.end) { 1438 shost_printk(KERN_ERR, ch->target->scsi_host, 1439 PFX "Out of MRs (mr_per_cmd = %d)\n", 1440 ch->target->mr_per_cmd); 1441 return -ENOMEM; 1442 } 1443 1444 WARN_ON_ONCE(!dev->use_fast_reg); 1445 1446 if (sg_nents == 1 && target->global_rkey) { 1447 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1448 1449 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset, 1450 sg_dma_len(state->sg) - sg_offset, 1451 target->global_rkey); 1452 if (sg_offset_p) 1453 *sg_offset_p = 0; 1454 return 1; 1455 } 1456 1457 desc = srp_fr_pool_get(ch->fr_pool); 1458 if (!desc) 1459 return -ENOMEM; 1460 1461 rkey = ib_inc_rkey(desc->mr->rkey); 1462 ib_update_fast_reg_key(desc->mr, rkey); 1463 1464 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p, 1465 dev->mr_page_size); 1466 if (unlikely(n < 0)) { 1467 srp_fr_pool_put(ch->fr_pool, &desc, 1); 1468 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n", 1469 dev_name(&req->scmnd->device->sdev_gendev), sg_nents, 1470 sg_offset_p ? *sg_offset_p : -1, n); 1471 return n; 1472 } 1473 1474 WARN_ON_ONCE(desc->mr->length == 0); 1475 1476 req->reg_cqe.done = srp_reg_mr_err_done; 1477 1478 wr.wr.next = NULL; 1479 wr.wr.opcode = IB_WR_REG_MR; 1480 wr.wr.wr_cqe = &req->reg_cqe; 1481 wr.wr.num_sge = 0; 1482 wr.wr.send_flags = 0; 1483 wr.mr = desc->mr; 1484 wr.key = desc->mr->rkey; 1485 wr.access = (IB_ACCESS_LOCAL_WRITE | 1486 IB_ACCESS_REMOTE_READ | 1487 IB_ACCESS_REMOTE_WRITE); 1488 1489 *state->fr.next++ = desc; 1490 state->nmdesc++; 1491 1492 srp_map_desc(state, desc->mr->iova, 1493 desc->mr->length, desc->mr->rkey); 1494 1495 err = ib_post_send(ch->qp, &wr.wr, NULL); 1496 if (unlikely(err)) { 1497 WARN_ON_ONCE(err == -ENOMEM); 1498 return err; 1499 } 1500 1501 return n; 1502 } 1503 1504 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1505 struct srp_request *req, struct scatterlist *scat, 1506 int count) 1507 { 1508 unsigned int sg_offset = 0; 1509 1510 state->fr.next = req->fr_list; 1511 state->fr.end = req->fr_list + ch->target->mr_per_cmd; 1512 state->sg = scat; 1513 1514 if (count == 0) 1515 return 0; 1516 1517 while (count) { 1518 int i, n; 1519 1520 n = srp_map_finish_fr(state, req, ch, count, &sg_offset); 1521 if (unlikely(n < 0)) 1522 return n; 1523 1524 count -= n; 1525 for (i = 0; i < n; i++) 1526 state->sg = sg_next(state->sg); 1527 } 1528 1529 return 0; 1530 } 1531 1532 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch, 1533 struct srp_request *req, struct scatterlist *scat, 1534 int count) 1535 { 1536 struct srp_target_port *target = ch->target; 1537 struct scatterlist *sg; 1538 int i; 1539 1540 for_each_sg(scat, sg, count, i) { 1541 srp_map_desc(state, sg_dma_address(sg), sg_dma_len(sg), 1542 target->global_rkey); 1543 } 1544 1545 return 0; 1546 } 1547 1548 /* 1549 * Register the indirect data buffer descriptor with the HCA. 1550 * 1551 * Note: since the indirect data buffer descriptor has been allocated with 1552 * kmalloc() it is guaranteed that this buffer is a physically contiguous 1553 * memory buffer. 1554 */ 1555 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req, 1556 void **next_mr, void **end_mr, u32 idb_len, 1557 __be32 *idb_rkey) 1558 { 1559 struct srp_target_port *target = ch->target; 1560 struct srp_device *dev = target->srp_host->srp_dev; 1561 struct srp_map_state state; 1562 struct srp_direct_buf idb_desc; 1563 struct scatterlist idb_sg[1]; 1564 int ret; 1565 1566 memset(&state, 0, sizeof(state)); 1567 memset(&idb_desc, 0, sizeof(idb_desc)); 1568 state.gen.next = next_mr; 1569 state.gen.end = end_mr; 1570 state.desc = &idb_desc; 1571 state.base_dma_addr = req->indirect_dma_addr; 1572 state.dma_len = idb_len; 1573 1574 if (dev->use_fast_reg) { 1575 state.sg = idb_sg; 1576 sg_init_one(idb_sg, req->indirect_desc, idb_len); 1577 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */ 1578 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1579 idb_sg->dma_length = idb_sg->length; /* hack^2 */ 1580 #endif 1581 ret = srp_map_finish_fr(&state, req, ch, 1, NULL); 1582 if (ret < 0) 1583 return ret; 1584 WARN_ON_ONCE(ret < 1); 1585 } else { 1586 return -EINVAL; 1587 } 1588 1589 *idb_rkey = idb_desc.key; 1590 1591 return 0; 1592 } 1593 1594 static void srp_check_mapping(struct srp_map_state *state, 1595 struct srp_rdma_ch *ch, struct srp_request *req, 1596 struct scatterlist *scat, int count) 1597 { 1598 struct srp_device *dev = ch->target->srp_host->srp_dev; 1599 struct srp_fr_desc **pfr; 1600 u64 desc_len = 0, mr_len = 0; 1601 int i; 1602 1603 for (i = 0; i < state->ndesc; i++) 1604 desc_len += be32_to_cpu(req->indirect_desc[i].len); 1605 if (dev->use_fast_reg) 1606 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++) 1607 mr_len += (*pfr)->mr->length; 1608 if (desc_len != scsi_bufflen(req->scmnd) || 1609 mr_len > scsi_bufflen(req->scmnd)) 1610 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n", 1611 scsi_bufflen(req->scmnd), desc_len, mr_len, 1612 state->ndesc, state->nmdesc); 1613 } 1614 1615 /** 1616 * srp_map_data() - map SCSI data buffer onto an SRP request 1617 * @scmnd: SCSI command to map 1618 * @ch: SRP RDMA channel 1619 * @req: SRP request 1620 * 1621 * Returns the length in bytes of the SRP_CMD IU or a negative value if 1622 * mapping failed. The size of any immediate data is not included in the 1623 * return value. 1624 */ 1625 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch, 1626 struct srp_request *req) 1627 { 1628 struct srp_target_port *target = ch->target; 1629 struct scatterlist *scat, *sg; 1630 struct srp_cmd *cmd = req->cmd->buf; 1631 int i, len, nents, count, ret; 1632 struct srp_device *dev; 1633 struct ib_device *ibdev; 1634 struct srp_map_state state; 1635 struct srp_indirect_buf *indirect_hdr; 1636 u64 data_len; 1637 u32 idb_len, table_len; 1638 __be32 idb_rkey; 1639 u8 fmt; 1640 1641 req->cmd->num_sge = 1; 1642 1643 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE) 1644 return sizeof(struct srp_cmd) + cmd->add_cdb_len; 1645 1646 if (scmnd->sc_data_direction != DMA_FROM_DEVICE && 1647 scmnd->sc_data_direction != DMA_TO_DEVICE) { 1648 shost_printk(KERN_WARNING, target->scsi_host, 1649 PFX "Unhandled data direction %d\n", 1650 scmnd->sc_data_direction); 1651 return -EINVAL; 1652 } 1653 1654 nents = scsi_sg_count(scmnd); 1655 scat = scsi_sglist(scmnd); 1656 data_len = scsi_bufflen(scmnd); 1657 1658 dev = target->srp_host->srp_dev; 1659 ibdev = dev->dev; 1660 1661 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction); 1662 if (unlikely(count == 0)) 1663 return -EIO; 1664 1665 if (ch->use_imm_data && 1666 count <= ch->max_imm_sge && 1667 SRP_IMM_DATA_OFFSET + data_len <= ch->max_it_iu_len && 1668 scmnd->sc_data_direction == DMA_TO_DEVICE) { 1669 struct srp_imm_buf *buf; 1670 struct ib_sge *sge = &req->cmd->sge[1]; 1671 1672 fmt = SRP_DATA_DESC_IMM; 1673 len = SRP_IMM_DATA_OFFSET; 1674 req->nmdesc = 0; 1675 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1676 buf->len = cpu_to_be32(data_len); 1677 WARN_ON_ONCE((void *)(buf + 1) > (void *)cmd + len); 1678 for_each_sg(scat, sg, count, i) { 1679 sge[i].addr = sg_dma_address(sg); 1680 sge[i].length = sg_dma_len(sg); 1681 sge[i].lkey = target->lkey; 1682 } 1683 req->cmd->num_sge += count; 1684 goto map_complete; 1685 } 1686 1687 fmt = SRP_DATA_DESC_DIRECT; 1688 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1689 sizeof(struct srp_direct_buf); 1690 1691 if (count == 1 && target->global_rkey) { 1692 /* 1693 * The midlayer only generated a single gather/scatter 1694 * entry, or DMA mapping coalesced everything to a 1695 * single entry. So a direct descriptor along with 1696 * the DMA MR suffices. 1697 */ 1698 struct srp_direct_buf *buf; 1699 1700 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1701 buf->va = cpu_to_be64(sg_dma_address(scat)); 1702 buf->key = cpu_to_be32(target->global_rkey); 1703 buf->len = cpu_to_be32(sg_dma_len(scat)); 1704 1705 req->nmdesc = 0; 1706 goto map_complete; 1707 } 1708 1709 /* 1710 * We have more than one scatter/gather entry, so build our indirect 1711 * descriptor table, trying to merge as many entries as we can. 1712 */ 1713 indirect_hdr = (void *)cmd->add_data + cmd->add_cdb_len; 1714 1715 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr, 1716 target->indirect_size, DMA_TO_DEVICE); 1717 1718 memset(&state, 0, sizeof(state)); 1719 state.desc = req->indirect_desc; 1720 if (dev->use_fast_reg) 1721 ret = srp_map_sg_fr(&state, ch, req, scat, count); 1722 else 1723 ret = srp_map_sg_dma(&state, ch, req, scat, count); 1724 req->nmdesc = state.nmdesc; 1725 if (ret < 0) 1726 goto unmap; 1727 1728 { 1729 DEFINE_DYNAMIC_DEBUG_METADATA(ddm, 1730 "Memory mapping consistency check"); 1731 if (DYNAMIC_DEBUG_BRANCH(ddm)) 1732 srp_check_mapping(&state, ch, req, scat, count); 1733 } 1734 1735 /* We've mapped the request, now pull as much of the indirect 1736 * descriptor table as we can into the command buffer. If this 1737 * target is not using an external indirect table, we are 1738 * guaranteed to fit into the command, as the SCSI layer won't 1739 * give us more S/G entries than we allow. 1740 */ 1741 if (state.ndesc == 1) { 1742 /* 1743 * Memory registration collapsed the sg-list into one entry, 1744 * so use a direct descriptor. 1745 */ 1746 struct srp_direct_buf *buf; 1747 1748 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1749 *buf = req->indirect_desc[0]; 1750 goto map_complete; 1751 } 1752 1753 if (unlikely(target->cmd_sg_cnt < state.ndesc && 1754 !target->allow_ext_sg)) { 1755 shost_printk(KERN_ERR, target->scsi_host, 1756 "Could not fit S/G list into SRP_CMD\n"); 1757 ret = -EIO; 1758 goto unmap; 1759 } 1760 1761 count = min(state.ndesc, target->cmd_sg_cnt); 1762 table_len = state.ndesc * sizeof (struct srp_direct_buf); 1763 idb_len = sizeof(struct srp_indirect_buf) + table_len; 1764 1765 fmt = SRP_DATA_DESC_INDIRECT; 1766 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1767 sizeof(struct srp_indirect_buf); 1768 len += count * sizeof (struct srp_direct_buf); 1769 1770 memcpy(indirect_hdr->desc_list, req->indirect_desc, 1771 count * sizeof (struct srp_direct_buf)); 1772 1773 if (!target->global_rkey) { 1774 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end, 1775 idb_len, &idb_rkey); 1776 if (ret < 0) 1777 goto unmap; 1778 req->nmdesc++; 1779 } else { 1780 idb_rkey = cpu_to_be32(target->global_rkey); 1781 } 1782 1783 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr); 1784 indirect_hdr->table_desc.key = idb_rkey; 1785 indirect_hdr->table_desc.len = cpu_to_be32(table_len); 1786 indirect_hdr->len = cpu_to_be32(state.total_len); 1787 1788 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1789 cmd->data_out_desc_cnt = count; 1790 else 1791 cmd->data_in_desc_cnt = count; 1792 1793 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len, 1794 DMA_TO_DEVICE); 1795 1796 map_complete: 1797 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1798 cmd->buf_fmt = fmt << 4; 1799 else 1800 cmd->buf_fmt = fmt; 1801 1802 return len; 1803 1804 unmap: 1805 srp_unmap_data(scmnd, ch, req); 1806 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size) 1807 ret = -E2BIG; 1808 return ret; 1809 } 1810 1811 /* 1812 * Return an IU and possible credit to the free pool 1813 */ 1814 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu, 1815 enum srp_iu_type iu_type) 1816 { 1817 unsigned long flags; 1818 1819 spin_lock_irqsave(&ch->lock, flags); 1820 list_add(&iu->list, &ch->free_tx); 1821 if (iu_type != SRP_IU_RSP) 1822 ++ch->req_lim; 1823 spin_unlock_irqrestore(&ch->lock, flags); 1824 } 1825 1826 /* 1827 * Must be called with ch->lock held to protect req_lim and free_tx. 1828 * If IU is not sent, it must be returned using srp_put_tx_iu(). 1829 * 1830 * Note: 1831 * An upper limit for the number of allocated information units for each 1832 * request type is: 1833 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues 1834 * more than Scsi_Host.can_queue requests. 1835 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE. 1836 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than 1837 * one unanswered SRP request to an initiator. 1838 */ 1839 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch, 1840 enum srp_iu_type iu_type) 1841 { 1842 struct srp_target_port *target = ch->target; 1843 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; 1844 struct srp_iu *iu; 1845 1846 lockdep_assert_held(&ch->lock); 1847 1848 ib_process_cq_direct(ch->send_cq, -1); 1849 1850 if (list_empty(&ch->free_tx)) 1851 return NULL; 1852 1853 /* Initiator responses to target requests do not consume credits */ 1854 if (iu_type != SRP_IU_RSP) { 1855 if (ch->req_lim <= rsv) { 1856 ++target->zero_req_lim; 1857 return NULL; 1858 } 1859 1860 --ch->req_lim; 1861 } 1862 1863 iu = list_first_entry(&ch->free_tx, struct srp_iu, list); 1864 list_del(&iu->list); 1865 return iu; 1866 } 1867 1868 /* 1869 * Note: if this function is called from inside ib_drain_sq() then it will 1870 * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE 1871 * with status IB_WC_SUCCESS then that's a bug. 1872 */ 1873 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc) 1874 { 1875 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1876 struct srp_rdma_ch *ch = cq->cq_context; 1877 1878 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1879 srp_handle_qp_err(cq, wc, "SEND"); 1880 return; 1881 } 1882 1883 lockdep_assert_held(&ch->lock); 1884 1885 list_add(&iu->list, &ch->free_tx); 1886 } 1887 1888 /** 1889 * srp_post_send() - send an SRP information unit 1890 * @ch: RDMA channel over which to send the information unit. 1891 * @iu: Information unit to send. 1892 * @len: Length of the information unit excluding immediate data. 1893 */ 1894 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len) 1895 { 1896 struct srp_target_port *target = ch->target; 1897 struct ib_send_wr wr; 1898 1899 if (WARN_ON_ONCE(iu->num_sge > SRP_MAX_SGE)) 1900 return -EINVAL; 1901 1902 iu->sge[0].addr = iu->dma; 1903 iu->sge[0].length = len; 1904 iu->sge[0].lkey = target->lkey; 1905 1906 iu->cqe.done = srp_send_done; 1907 1908 wr.next = NULL; 1909 wr.wr_cqe = &iu->cqe; 1910 wr.sg_list = &iu->sge[0]; 1911 wr.num_sge = iu->num_sge; 1912 wr.opcode = IB_WR_SEND; 1913 wr.send_flags = IB_SEND_SIGNALED; 1914 1915 return ib_post_send(ch->qp, &wr, NULL); 1916 } 1917 1918 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu) 1919 { 1920 struct srp_target_port *target = ch->target; 1921 struct ib_recv_wr wr; 1922 struct ib_sge list; 1923 1924 list.addr = iu->dma; 1925 list.length = iu->size; 1926 list.lkey = target->lkey; 1927 1928 iu->cqe.done = srp_recv_done; 1929 1930 wr.next = NULL; 1931 wr.wr_cqe = &iu->cqe; 1932 wr.sg_list = &list; 1933 wr.num_sge = 1; 1934 1935 return ib_post_recv(ch->qp, &wr, NULL); 1936 } 1937 1938 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) 1939 { 1940 struct srp_target_port *target = ch->target; 1941 struct srp_request *req; 1942 struct scsi_cmnd *scmnd; 1943 unsigned long flags; 1944 1945 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { 1946 spin_lock_irqsave(&ch->lock, flags); 1947 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1948 if (rsp->tag == ch->tsk_mgmt_tag) { 1949 ch->tsk_mgmt_status = -1; 1950 if (be32_to_cpu(rsp->resp_data_len) >= 4) 1951 ch->tsk_mgmt_status = rsp->data[3]; 1952 complete(&ch->tsk_mgmt_done); 1953 } else { 1954 shost_printk(KERN_ERR, target->scsi_host, 1955 "Received tsk mgmt response too late for tag %#llx\n", 1956 rsp->tag); 1957 } 1958 spin_unlock_irqrestore(&ch->lock, flags); 1959 } else { 1960 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag); 1961 if (scmnd) { 1962 req = scsi_cmd_priv(scmnd); 1963 scmnd = srp_claim_req(ch, req, NULL, scmnd); 1964 } 1965 if (!scmnd) { 1966 shost_printk(KERN_ERR, target->scsi_host, 1967 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", 1968 rsp->tag, ch - target->ch, ch->qp->qp_num); 1969 1970 spin_lock_irqsave(&ch->lock, flags); 1971 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1972 spin_unlock_irqrestore(&ch->lock, flags); 1973 1974 return; 1975 } 1976 scmnd->result = rsp->status; 1977 1978 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { 1979 memcpy(scmnd->sense_buffer, rsp->data + 1980 be32_to_cpu(rsp->resp_data_len), 1981 min_t(int, be32_to_cpu(rsp->sense_data_len), 1982 SCSI_SENSE_BUFFERSIZE)); 1983 } 1984 1985 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER)) 1986 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt)); 1987 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER)) 1988 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt)); 1989 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER)) 1990 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt)); 1991 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER)) 1992 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt)); 1993 1994 srp_free_req(ch, req, scmnd, 1995 be32_to_cpu(rsp->req_lim_delta)); 1996 1997 scsi_done(scmnd); 1998 } 1999 } 2000 2001 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, 2002 void *rsp, int len) 2003 { 2004 struct srp_target_port *target = ch->target; 2005 struct ib_device *dev = target->srp_host->srp_dev->dev; 2006 unsigned long flags; 2007 struct srp_iu *iu; 2008 int err; 2009 2010 spin_lock_irqsave(&ch->lock, flags); 2011 ch->req_lim += req_delta; 2012 iu = __srp_get_tx_iu(ch, SRP_IU_RSP); 2013 spin_unlock_irqrestore(&ch->lock, flags); 2014 2015 if (!iu) { 2016 shost_printk(KERN_ERR, target->scsi_host, PFX 2017 "no IU available to send response\n"); 2018 return 1; 2019 } 2020 2021 iu->num_sge = 1; 2022 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); 2023 memcpy(iu->buf, rsp, len); 2024 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); 2025 2026 err = srp_post_send(ch, iu, len); 2027 if (err) { 2028 shost_printk(KERN_ERR, target->scsi_host, PFX 2029 "unable to post response: %d\n", err); 2030 srp_put_tx_iu(ch, iu, SRP_IU_RSP); 2031 } 2032 2033 return err; 2034 } 2035 2036 static void srp_process_cred_req(struct srp_rdma_ch *ch, 2037 struct srp_cred_req *req) 2038 { 2039 struct srp_cred_rsp rsp = { 2040 .opcode = SRP_CRED_RSP, 2041 .tag = req->tag, 2042 }; 2043 s32 delta = be32_to_cpu(req->req_lim_delta); 2044 2045 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2046 shost_printk(KERN_ERR, ch->target->scsi_host, PFX 2047 "problems processing SRP_CRED_REQ\n"); 2048 } 2049 2050 static void srp_process_aer_req(struct srp_rdma_ch *ch, 2051 struct srp_aer_req *req) 2052 { 2053 struct srp_target_port *target = ch->target; 2054 struct srp_aer_rsp rsp = { 2055 .opcode = SRP_AER_RSP, 2056 .tag = req->tag, 2057 }; 2058 s32 delta = be32_to_cpu(req->req_lim_delta); 2059 2060 shost_printk(KERN_ERR, target->scsi_host, PFX 2061 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); 2062 2063 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2064 shost_printk(KERN_ERR, target->scsi_host, PFX 2065 "problems processing SRP_AER_REQ\n"); 2066 } 2067 2068 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) 2069 { 2070 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 2071 struct srp_rdma_ch *ch = cq->cq_context; 2072 struct srp_target_port *target = ch->target; 2073 struct ib_device *dev = target->srp_host->srp_dev->dev; 2074 int res; 2075 u8 opcode; 2076 2077 if (unlikely(wc->status != IB_WC_SUCCESS)) { 2078 srp_handle_qp_err(cq, wc, "RECV"); 2079 return; 2080 } 2081 2082 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len, 2083 DMA_FROM_DEVICE); 2084 2085 opcode = *(u8 *) iu->buf; 2086 2087 if (0) { 2088 shost_printk(KERN_ERR, target->scsi_host, 2089 PFX "recv completion, opcode 0x%02x\n", opcode); 2090 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1, 2091 iu->buf, wc->byte_len, true); 2092 } 2093 2094 switch (opcode) { 2095 case SRP_RSP: 2096 srp_process_rsp(ch, iu->buf); 2097 break; 2098 2099 case SRP_CRED_REQ: 2100 srp_process_cred_req(ch, iu->buf); 2101 break; 2102 2103 case SRP_AER_REQ: 2104 srp_process_aer_req(ch, iu->buf); 2105 break; 2106 2107 case SRP_T_LOGOUT: 2108 /* XXX Handle target logout */ 2109 shost_printk(KERN_WARNING, target->scsi_host, 2110 PFX "Got target logout request\n"); 2111 break; 2112 2113 default: 2114 shost_printk(KERN_WARNING, target->scsi_host, 2115 PFX "Unhandled SRP opcode 0x%02x\n", opcode); 2116 break; 2117 } 2118 2119 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len, 2120 DMA_FROM_DEVICE); 2121 2122 res = srp_post_recv(ch, iu); 2123 if (res != 0) 2124 shost_printk(KERN_ERR, target->scsi_host, 2125 PFX "Recv failed with error code %d\n", res); 2126 } 2127 2128 /** 2129 * srp_tl_err_work() - handle a transport layer error 2130 * @work: Work structure embedded in an SRP target port. 2131 * 2132 * Note: This function may get invoked before the rport has been created, 2133 * hence the target->rport test. 2134 */ 2135 static void srp_tl_err_work(struct work_struct *work) 2136 { 2137 struct srp_target_port *target; 2138 2139 target = container_of(work, struct srp_target_port, tl_err_work); 2140 if (target->rport) 2141 srp_start_tl_fail_timers(target->rport); 2142 } 2143 2144 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 2145 const char *opname) 2146 { 2147 struct srp_rdma_ch *ch = cq->cq_context; 2148 struct srp_target_port *target = ch->target; 2149 2150 if (ch->connected && !target->qp_in_error) { 2151 shost_printk(KERN_ERR, target->scsi_host, 2152 PFX "failed %s status %s (%d) for CQE %p\n", 2153 opname, ib_wc_status_msg(wc->status), wc->status, 2154 wc->wr_cqe); 2155 queue_work(system_long_wq, &target->tl_err_work); 2156 } 2157 target->qp_in_error = true; 2158 } 2159 2160 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) 2161 { 2162 struct request *rq = scsi_cmd_to_rq(scmnd); 2163 struct srp_target_port *target = host_to_target(shost); 2164 struct srp_rdma_ch *ch; 2165 struct srp_request *req = scsi_cmd_priv(scmnd); 2166 struct srp_iu *iu; 2167 struct srp_cmd *cmd; 2168 struct ib_device *dev; 2169 unsigned long flags; 2170 u32 tag; 2171 int len, ret; 2172 2173 scmnd->result = srp_chkready(target->rport); 2174 if (unlikely(scmnd->result)) 2175 goto err; 2176 2177 WARN_ON_ONCE(rq->tag < 0); 2178 tag = blk_mq_unique_tag(rq); 2179 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 2180 2181 spin_lock_irqsave(&ch->lock, flags); 2182 iu = __srp_get_tx_iu(ch, SRP_IU_CMD); 2183 spin_unlock_irqrestore(&ch->lock, flags); 2184 2185 if (!iu) 2186 goto err; 2187 2188 dev = target->srp_host->srp_dev->dev; 2189 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len, 2190 DMA_TO_DEVICE); 2191 2192 cmd = iu->buf; 2193 memset(cmd, 0, sizeof *cmd); 2194 2195 cmd->opcode = SRP_CMD; 2196 int_to_scsilun(scmnd->device->lun, &cmd->lun); 2197 cmd->tag = tag; 2198 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len); 2199 if (unlikely(scmnd->cmd_len > sizeof(cmd->cdb))) { 2200 cmd->add_cdb_len = round_up(scmnd->cmd_len - sizeof(cmd->cdb), 2201 4); 2202 if (WARN_ON_ONCE(cmd->add_cdb_len > SRP_MAX_ADD_CDB_LEN)) 2203 goto err_iu; 2204 } 2205 2206 req->scmnd = scmnd; 2207 req->cmd = iu; 2208 2209 len = srp_map_data(scmnd, ch, req); 2210 if (len < 0) { 2211 shost_printk(KERN_ERR, target->scsi_host, 2212 PFX "Failed to map data (%d)\n", len); 2213 /* 2214 * If we ran out of memory descriptors (-ENOMEM) because an 2215 * application is queuing many requests with more than 2216 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer 2217 * to reduce queue depth temporarily. 2218 */ 2219 scmnd->result = len == -ENOMEM ? 2220 DID_OK << 16 | SAM_STAT_TASK_SET_FULL : DID_ERROR << 16; 2221 goto err_iu; 2222 } 2223 2224 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len, 2225 DMA_TO_DEVICE); 2226 2227 if (srp_post_send(ch, iu, len)) { 2228 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); 2229 scmnd->result = DID_ERROR << 16; 2230 goto err_unmap; 2231 } 2232 2233 return 0; 2234 2235 err_unmap: 2236 srp_unmap_data(scmnd, ch, req); 2237 2238 err_iu: 2239 srp_put_tx_iu(ch, iu, SRP_IU_CMD); 2240 2241 /* 2242 * Avoid that the loops that iterate over the request ring can 2243 * encounter a dangling SCSI command pointer. 2244 */ 2245 req->scmnd = NULL; 2246 2247 err: 2248 if (scmnd->result) { 2249 scsi_done(scmnd); 2250 ret = 0; 2251 } else { 2252 ret = SCSI_MLQUEUE_HOST_BUSY; 2253 } 2254 2255 return ret; 2256 } 2257 2258 /* 2259 * Note: the resources allocated in this function are freed in 2260 * srp_free_ch_ib(). 2261 */ 2262 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch) 2263 { 2264 struct srp_target_port *target = ch->target; 2265 int i; 2266 2267 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring), 2268 GFP_KERNEL); 2269 if (!ch->rx_ring) 2270 goto err_no_ring; 2271 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring), 2272 GFP_KERNEL); 2273 if (!ch->tx_ring) 2274 goto err_no_ring; 2275 2276 for (i = 0; i < target->queue_size; ++i) { 2277 ch->rx_ring[i] = srp_alloc_iu(target->srp_host, 2278 ch->max_ti_iu_len, 2279 GFP_KERNEL, DMA_FROM_DEVICE); 2280 if (!ch->rx_ring[i]) 2281 goto err; 2282 } 2283 2284 for (i = 0; i < target->queue_size; ++i) { 2285 ch->tx_ring[i] = srp_alloc_iu(target->srp_host, 2286 ch->max_it_iu_len, 2287 GFP_KERNEL, DMA_TO_DEVICE); 2288 if (!ch->tx_ring[i]) 2289 goto err; 2290 2291 list_add(&ch->tx_ring[i]->list, &ch->free_tx); 2292 } 2293 2294 return 0; 2295 2296 err: 2297 for (i = 0; i < target->queue_size; ++i) { 2298 srp_free_iu(target->srp_host, ch->rx_ring[i]); 2299 srp_free_iu(target->srp_host, ch->tx_ring[i]); 2300 } 2301 2302 2303 err_no_ring: 2304 kfree(ch->tx_ring); 2305 ch->tx_ring = NULL; 2306 kfree(ch->rx_ring); 2307 ch->rx_ring = NULL; 2308 2309 return -ENOMEM; 2310 } 2311 2312 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) 2313 { 2314 uint64_t T_tr_ns, max_compl_time_ms; 2315 uint32_t rq_tmo_jiffies; 2316 2317 /* 2318 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair, 2319 * table 91), both the QP timeout and the retry count have to be set 2320 * for RC QP's during the RTR to RTS transition. 2321 */ 2322 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) != 2323 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)); 2324 2325 /* 2326 * Set target->rq_tmo_jiffies to one second more than the largest time 2327 * it can take before an error completion is generated. See also 2328 * C9-140..142 in the IBTA spec for more information about how to 2329 * convert the QP Local ACK Timeout value to nanoseconds. 2330 */ 2331 T_tr_ns = 4096 * (1ULL << qp_attr->timeout); 2332 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns; 2333 do_div(max_compl_time_ms, NSEC_PER_MSEC); 2334 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000); 2335 2336 return rq_tmo_jiffies; 2337 } 2338 2339 static void srp_cm_rep_handler(struct ib_cm_id *cm_id, 2340 const struct srp_login_rsp *lrsp, 2341 struct srp_rdma_ch *ch) 2342 { 2343 struct srp_target_port *target = ch->target; 2344 struct ib_qp_attr *qp_attr = NULL; 2345 int attr_mask = 0; 2346 int ret = 0; 2347 int i; 2348 2349 if (lrsp->opcode == SRP_LOGIN_RSP) { 2350 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); 2351 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); 2352 ch->use_imm_data = srp_use_imm_data && 2353 (lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP); 2354 ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 2355 ch->use_imm_data, 2356 target->max_it_iu_size); 2357 WARN_ON_ONCE(ch->max_it_iu_len > 2358 be32_to_cpu(lrsp->max_it_iu_len)); 2359 2360 if (ch->use_imm_data) 2361 shost_printk(KERN_DEBUG, target->scsi_host, 2362 PFX "using immediate data\n"); 2363 2364 /* 2365 * Reserve credits for task management so we don't 2366 * bounce requests back to the SCSI mid-layer. 2367 */ 2368 target->scsi_host->can_queue 2369 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE, 2370 target->scsi_host->can_queue); 2371 target->scsi_host->cmd_per_lun 2372 = min_t(int, target->scsi_host->can_queue, 2373 target->scsi_host->cmd_per_lun); 2374 } else { 2375 shost_printk(KERN_WARNING, target->scsi_host, 2376 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode); 2377 ret = -ECONNRESET; 2378 goto error; 2379 } 2380 2381 if (!ch->rx_ring) { 2382 ret = srp_alloc_iu_bufs(ch); 2383 if (ret) 2384 goto error; 2385 } 2386 2387 for (i = 0; i < target->queue_size; i++) { 2388 struct srp_iu *iu = ch->rx_ring[i]; 2389 2390 ret = srp_post_recv(ch, iu); 2391 if (ret) 2392 goto error; 2393 } 2394 2395 if (!target->using_rdma_cm) { 2396 ret = -ENOMEM; 2397 qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL); 2398 if (!qp_attr) 2399 goto error; 2400 2401 qp_attr->qp_state = IB_QPS_RTR; 2402 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2403 if (ret) 2404 goto error_free; 2405 2406 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2407 if (ret) 2408 goto error_free; 2409 2410 qp_attr->qp_state = IB_QPS_RTS; 2411 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2412 if (ret) 2413 goto error_free; 2414 2415 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); 2416 2417 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2418 if (ret) 2419 goto error_free; 2420 2421 ret = ib_send_cm_rtu(cm_id, NULL, 0); 2422 } 2423 2424 error_free: 2425 kfree(qp_attr); 2426 2427 error: 2428 ch->status = ret; 2429 } 2430 2431 static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id, 2432 const struct ib_cm_event *event, 2433 struct srp_rdma_ch *ch) 2434 { 2435 struct srp_target_port *target = ch->target; 2436 struct Scsi_Host *shost = target->scsi_host; 2437 struct ib_class_port_info *cpi; 2438 int opcode; 2439 u16 dlid; 2440 2441 switch (event->param.rej_rcvd.reason) { 2442 case IB_CM_REJ_PORT_CM_REDIRECT: 2443 cpi = event->param.rej_rcvd.ari; 2444 dlid = be16_to_cpu(cpi->redirect_lid); 2445 sa_path_set_dlid(&ch->ib_cm.path, dlid); 2446 ch->ib_cm.path.pkey = cpi->redirect_pkey; 2447 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; 2448 memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16); 2449 2450 ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; 2451 break; 2452 2453 case IB_CM_REJ_PORT_REDIRECT: 2454 if (srp_target_is_topspin(target)) { 2455 union ib_gid *dgid = &ch->ib_cm.path.dgid; 2456 2457 /* 2458 * Topspin/Cisco SRP gateways incorrectly send 2459 * reject reason code 25 when they mean 24 2460 * (port redirect). 2461 */ 2462 memcpy(dgid->raw, event->param.rej_rcvd.ari, 16); 2463 2464 shost_printk(KERN_DEBUG, shost, 2465 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", 2466 be64_to_cpu(dgid->global.subnet_prefix), 2467 be64_to_cpu(dgid->global.interface_id)); 2468 2469 ch->status = SRP_PORT_REDIRECT; 2470 } else { 2471 shost_printk(KERN_WARNING, shost, 2472 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); 2473 ch->status = -ECONNRESET; 2474 } 2475 break; 2476 2477 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2478 shost_printk(KERN_WARNING, shost, 2479 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2480 ch->status = -ECONNRESET; 2481 break; 2482 2483 case IB_CM_REJ_CONSUMER_DEFINED: 2484 opcode = *(u8 *) event->private_data; 2485 if (opcode == SRP_LOGIN_REJ) { 2486 struct srp_login_rej *rej = event->private_data; 2487 u32 reason = be32_to_cpu(rej->reason); 2488 2489 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2490 shost_printk(KERN_WARNING, shost, 2491 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2492 else 2493 shost_printk(KERN_WARNING, shost, PFX 2494 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n", 2495 target->sgid.raw, 2496 target->ib_cm.orig_dgid.raw, 2497 reason); 2498 } else 2499 shost_printk(KERN_WARNING, shost, 2500 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED," 2501 " opcode 0x%02x\n", opcode); 2502 ch->status = -ECONNRESET; 2503 break; 2504 2505 case IB_CM_REJ_STALE_CONN: 2506 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); 2507 ch->status = SRP_STALE_CONN; 2508 break; 2509 2510 default: 2511 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2512 event->param.rej_rcvd.reason); 2513 ch->status = -ECONNRESET; 2514 } 2515 } 2516 2517 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 2518 const struct ib_cm_event *event) 2519 { 2520 struct srp_rdma_ch *ch = cm_id->context; 2521 struct srp_target_port *target = ch->target; 2522 int comp = 0; 2523 2524 switch (event->event) { 2525 case IB_CM_REQ_ERROR: 2526 shost_printk(KERN_DEBUG, target->scsi_host, 2527 PFX "Sending CM REQ failed\n"); 2528 comp = 1; 2529 ch->status = -ECONNRESET; 2530 break; 2531 2532 case IB_CM_REP_RECEIVED: 2533 comp = 1; 2534 srp_cm_rep_handler(cm_id, event->private_data, ch); 2535 break; 2536 2537 case IB_CM_REJ_RECEIVED: 2538 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2539 comp = 1; 2540 2541 srp_ib_cm_rej_handler(cm_id, event, ch); 2542 break; 2543 2544 case IB_CM_DREQ_RECEIVED: 2545 shost_printk(KERN_WARNING, target->scsi_host, 2546 PFX "DREQ received - connection closed\n"); 2547 ch->connected = false; 2548 if (ib_send_cm_drep(cm_id, NULL, 0)) 2549 shost_printk(KERN_ERR, target->scsi_host, 2550 PFX "Sending CM DREP failed\n"); 2551 queue_work(system_long_wq, &target->tl_err_work); 2552 break; 2553 2554 case IB_CM_TIMEWAIT_EXIT: 2555 shost_printk(KERN_ERR, target->scsi_host, 2556 PFX "connection closed\n"); 2557 comp = 1; 2558 2559 ch->status = 0; 2560 break; 2561 2562 case IB_CM_MRA_RECEIVED: 2563 case IB_CM_DREQ_ERROR: 2564 case IB_CM_DREP_RECEIVED: 2565 break; 2566 2567 default: 2568 shost_printk(KERN_WARNING, target->scsi_host, 2569 PFX "Unhandled CM event %d\n", event->event); 2570 break; 2571 } 2572 2573 if (comp) 2574 complete(&ch->done); 2575 2576 return 0; 2577 } 2578 2579 static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch, 2580 struct rdma_cm_event *event) 2581 { 2582 struct srp_target_port *target = ch->target; 2583 struct Scsi_Host *shost = target->scsi_host; 2584 int opcode; 2585 2586 switch (event->status) { 2587 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2588 shost_printk(KERN_WARNING, shost, 2589 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2590 ch->status = -ECONNRESET; 2591 break; 2592 2593 case IB_CM_REJ_CONSUMER_DEFINED: 2594 opcode = *(u8 *) event->param.conn.private_data; 2595 if (opcode == SRP_LOGIN_REJ) { 2596 struct srp_login_rej *rej = 2597 (struct srp_login_rej *) 2598 event->param.conn.private_data; 2599 u32 reason = be32_to_cpu(rej->reason); 2600 2601 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2602 shost_printk(KERN_WARNING, shost, 2603 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2604 else 2605 shost_printk(KERN_WARNING, shost, 2606 PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason); 2607 } else { 2608 shost_printk(KERN_WARNING, shost, 2609 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n", 2610 opcode); 2611 } 2612 ch->status = -ECONNRESET; 2613 break; 2614 2615 case IB_CM_REJ_STALE_CONN: 2616 shost_printk(KERN_WARNING, shost, 2617 " REJ reason: stale connection\n"); 2618 ch->status = SRP_STALE_CONN; 2619 break; 2620 2621 default: 2622 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2623 event->status); 2624 ch->status = -ECONNRESET; 2625 break; 2626 } 2627 } 2628 2629 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 2630 struct rdma_cm_event *event) 2631 { 2632 struct srp_rdma_ch *ch = cm_id->context; 2633 struct srp_target_port *target = ch->target; 2634 int comp = 0; 2635 2636 switch (event->event) { 2637 case RDMA_CM_EVENT_ADDR_RESOLVED: 2638 ch->status = 0; 2639 comp = 1; 2640 break; 2641 2642 case RDMA_CM_EVENT_ADDR_ERROR: 2643 ch->status = -ENXIO; 2644 comp = 1; 2645 break; 2646 2647 case RDMA_CM_EVENT_ROUTE_RESOLVED: 2648 ch->status = 0; 2649 comp = 1; 2650 break; 2651 2652 case RDMA_CM_EVENT_ROUTE_ERROR: 2653 case RDMA_CM_EVENT_UNREACHABLE: 2654 ch->status = -EHOSTUNREACH; 2655 comp = 1; 2656 break; 2657 2658 case RDMA_CM_EVENT_CONNECT_ERROR: 2659 shost_printk(KERN_DEBUG, target->scsi_host, 2660 PFX "Sending CM REQ failed\n"); 2661 comp = 1; 2662 ch->status = -ECONNRESET; 2663 break; 2664 2665 case RDMA_CM_EVENT_ESTABLISHED: 2666 comp = 1; 2667 srp_cm_rep_handler(NULL, event->param.conn.private_data, ch); 2668 break; 2669 2670 case RDMA_CM_EVENT_REJECTED: 2671 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2672 comp = 1; 2673 2674 srp_rdma_cm_rej_handler(ch, event); 2675 break; 2676 2677 case RDMA_CM_EVENT_DISCONNECTED: 2678 if (ch->connected) { 2679 shost_printk(KERN_WARNING, target->scsi_host, 2680 PFX "received DREQ\n"); 2681 rdma_disconnect(ch->rdma_cm.cm_id); 2682 comp = 1; 2683 ch->status = 0; 2684 queue_work(system_long_wq, &target->tl_err_work); 2685 } 2686 break; 2687 2688 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 2689 shost_printk(KERN_ERR, target->scsi_host, 2690 PFX "connection closed\n"); 2691 2692 comp = 1; 2693 ch->status = 0; 2694 break; 2695 2696 default: 2697 shost_printk(KERN_WARNING, target->scsi_host, 2698 PFX "Unhandled CM event %d\n", event->event); 2699 break; 2700 } 2701 2702 if (comp) 2703 complete(&ch->done); 2704 2705 return 0; 2706 } 2707 2708 /** 2709 * srp_change_queue_depth - setting device queue depth 2710 * @sdev: scsi device struct 2711 * @qdepth: requested queue depth 2712 * 2713 * Returns queue depth. 2714 */ 2715 static int 2716 srp_change_queue_depth(struct scsi_device *sdev, int qdepth) 2717 { 2718 if (!sdev->tagged_supported) 2719 qdepth = 1; 2720 return scsi_change_queue_depth(sdev, qdepth); 2721 } 2722 2723 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, 2724 u8 func, u8 *status) 2725 { 2726 struct srp_target_port *target = ch->target; 2727 struct srp_rport *rport = target->rport; 2728 struct ib_device *dev = target->srp_host->srp_dev->dev; 2729 struct srp_iu *iu; 2730 struct srp_tsk_mgmt *tsk_mgmt; 2731 int res; 2732 2733 if (!ch->connected || target->qp_in_error) 2734 return -1; 2735 2736 /* 2737 * Lock the rport mutex to avoid that srp_create_ch_ib() is 2738 * invoked while a task management function is being sent. 2739 */ 2740 mutex_lock(&rport->mutex); 2741 spin_lock_irq(&ch->lock); 2742 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT); 2743 spin_unlock_irq(&ch->lock); 2744 2745 if (!iu) { 2746 mutex_unlock(&rport->mutex); 2747 2748 return -1; 2749 } 2750 2751 iu->num_sge = 1; 2752 2753 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, 2754 DMA_TO_DEVICE); 2755 tsk_mgmt = iu->buf; 2756 memset(tsk_mgmt, 0, sizeof *tsk_mgmt); 2757 2758 tsk_mgmt->opcode = SRP_TSK_MGMT; 2759 int_to_scsilun(lun, &tsk_mgmt->lun); 2760 tsk_mgmt->tsk_mgmt_func = func; 2761 tsk_mgmt->task_tag = req_tag; 2762 2763 spin_lock_irq(&ch->lock); 2764 ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT; 2765 tsk_mgmt->tag = ch->tsk_mgmt_tag; 2766 spin_unlock_irq(&ch->lock); 2767 2768 init_completion(&ch->tsk_mgmt_done); 2769 2770 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, 2771 DMA_TO_DEVICE); 2772 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) { 2773 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT); 2774 mutex_unlock(&rport->mutex); 2775 2776 return -1; 2777 } 2778 res = wait_for_completion_timeout(&ch->tsk_mgmt_done, 2779 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)); 2780 if (res > 0 && status) 2781 *status = ch->tsk_mgmt_status; 2782 mutex_unlock(&rport->mutex); 2783 2784 WARN_ON_ONCE(res < 0); 2785 2786 return res > 0 ? 0 : -1; 2787 } 2788 2789 static int srp_abort(struct scsi_cmnd *scmnd) 2790 { 2791 struct srp_target_port *target = host_to_target(scmnd->device->host); 2792 struct srp_request *req = scsi_cmd_priv(scmnd); 2793 u32 tag; 2794 u16 ch_idx; 2795 struct srp_rdma_ch *ch; 2796 int ret; 2797 2798 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); 2799 2800 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); 2801 ch_idx = blk_mq_unique_tag_to_hwq(tag); 2802 if (WARN_ON_ONCE(ch_idx >= target->ch_count)) 2803 return SUCCESS; 2804 ch = &target->ch[ch_idx]; 2805 if (!srp_claim_req(ch, req, NULL, scmnd)) 2806 return SUCCESS; 2807 shost_printk(KERN_ERR, target->scsi_host, 2808 "Sending SRP abort for tag %#x\n", tag); 2809 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun, 2810 SRP_TSK_ABORT_TASK, NULL) == 0) 2811 ret = SUCCESS; 2812 else if (target->rport->state == SRP_RPORT_LOST) 2813 ret = FAST_IO_FAIL; 2814 else 2815 ret = FAILED; 2816 if (ret == SUCCESS) { 2817 srp_free_req(ch, req, scmnd, 0); 2818 scmnd->result = DID_ABORT << 16; 2819 scsi_done(scmnd); 2820 } 2821 2822 return ret; 2823 } 2824 2825 static int srp_reset_device(struct scsi_cmnd *scmnd) 2826 { 2827 struct srp_target_port *target = host_to_target(scmnd->device->host); 2828 struct srp_rdma_ch *ch; 2829 u8 status; 2830 2831 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); 2832 2833 ch = &target->ch[0]; 2834 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun, 2835 SRP_TSK_LUN_RESET, &status)) 2836 return FAILED; 2837 if (status) 2838 return FAILED; 2839 2840 return SUCCESS; 2841 } 2842 2843 static int srp_reset_host(struct scsi_cmnd *scmnd) 2844 { 2845 struct srp_target_port *target = host_to_target(scmnd->device->host); 2846 2847 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n"); 2848 2849 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED; 2850 } 2851 2852 static int srp_target_alloc(struct scsi_target *starget) 2853 { 2854 struct Scsi_Host *shost = dev_to_shost(starget->dev.parent); 2855 struct srp_target_port *target = host_to_target(shost); 2856 2857 if (target->target_can_queue) 2858 starget->can_queue = target->target_can_queue; 2859 return 0; 2860 } 2861 2862 static int srp_slave_configure(struct scsi_device *sdev) 2863 { 2864 struct Scsi_Host *shost = sdev->host; 2865 struct srp_target_port *target = host_to_target(shost); 2866 struct request_queue *q = sdev->request_queue; 2867 unsigned long timeout; 2868 2869 if (sdev->type == TYPE_DISK) { 2870 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies); 2871 blk_queue_rq_timeout(q, timeout); 2872 } 2873 2874 return 0; 2875 } 2876 2877 static ssize_t id_ext_show(struct device *dev, struct device_attribute *attr, 2878 char *buf) 2879 { 2880 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2881 2882 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->id_ext)); 2883 } 2884 2885 static DEVICE_ATTR_RO(id_ext); 2886 2887 static ssize_t ioc_guid_show(struct device *dev, struct device_attribute *attr, 2888 char *buf) 2889 { 2890 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2891 2892 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid)); 2893 } 2894 2895 static DEVICE_ATTR_RO(ioc_guid); 2896 2897 static ssize_t service_id_show(struct device *dev, 2898 struct device_attribute *attr, char *buf) 2899 { 2900 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2901 2902 if (target->using_rdma_cm) 2903 return -ENOENT; 2904 return sysfs_emit(buf, "0x%016llx\n", 2905 be64_to_cpu(target->ib_cm.service_id)); 2906 } 2907 2908 static DEVICE_ATTR_RO(service_id); 2909 2910 static ssize_t pkey_show(struct device *dev, struct device_attribute *attr, 2911 char *buf) 2912 { 2913 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2914 2915 if (target->using_rdma_cm) 2916 return -ENOENT; 2917 2918 return sysfs_emit(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey)); 2919 } 2920 2921 static DEVICE_ATTR_RO(pkey); 2922 2923 static ssize_t sgid_show(struct device *dev, struct device_attribute *attr, 2924 char *buf) 2925 { 2926 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2927 2928 return sysfs_emit(buf, "%pI6\n", target->sgid.raw); 2929 } 2930 2931 static DEVICE_ATTR_RO(sgid); 2932 2933 static ssize_t dgid_show(struct device *dev, struct device_attribute *attr, 2934 char *buf) 2935 { 2936 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2937 struct srp_rdma_ch *ch = &target->ch[0]; 2938 2939 if (target->using_rdma_cm) 2940 return -ENOENT; 2941 2942 return sysfs_emit(buf, "%pI6\n", ch->ib_cm.path.dgid.raw); 2943 } 2944 2945 static DEVICE_ATTR_RO(dgid); 2946 2947 static ssize_t orig_dgid_show(struct device *dev, struct device_attribute *attr, 2948 char *buf) 2949 { 2950 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2951 2952 if (target->using_rdma_cm) 2953 return -ENOENT; 2954 2955 return sysfs_emit(buf, "%pI6\n", target->ib_cm.orig_dgid.raw); 2956 } 2957 2958 static DEVICE_ATTR_RO(orig_dgid); 2959 2960 static ssize_t req_lim_show(struct device *dev, struct device_attribute *attr, 2961 char *buf) 2962 { 2963 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2964 struct srp_rdma_ch *ch; 2965 int i, req_lim = INT_MAX; 2966 2967 for (i = 0; i < target->ch_count; i++) { 2968 ch = &target->ch[i]; 2969 req_lim = min(req_lim, ch->req_lim); 2970 } 2971 2972 return sysfs_emit(buf, "%d\n", req_lim); 2973 } 2974 2975 static DEVICE_ATTR_RO(req_lim); 2976 2977 static ssize_t zero_req_lim_show(struct device *dev, 2978 struct device_attribute *attr, char *buf) 2979 { 2980 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2981 2982 return sysfs_emit(buf, "%d\n", target->zero_req_lim); 2983 } 2984 2985 static DEVICE_ATTR_RO(zero_req_lim); 2986 2987 static ssize_t local_ib_port_show(struct device *dev, 2988 struct device_attribute *attr, char *buf) 2989 { 2990 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2991 2992 return sysfs_emit(buf, "%u\n", target->srp_host->port); 2993 } 2994 2995 static DEVICE_ATTR_RO(local_ib_port); 2996 2997 static ssize_t local_ib_device_show(struct device *dev, 2998 struct device_attribute *attr, char *buf) 2999 { 3000 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3001 3002 return sysfs_emit(buf, "%s\n", 3003 dev_name(&target->srp_host->srp_dev->dev->dev)); 3004 } 3005 3006 static DEVICE_ATTR_RO(local_ib_device); 3007 3008 static ssize_t ch_count_show(struct device *dev, struct device_attribute *attr, 3009 char *buf) 3010 { 3011 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3012 3013 return sysfs_emit(buf, "%d\n", target->ch_count); 3014 } 3015 3016 static DEVICE_ATTR_RO(ch_count); 3017 3018 static ssize_t comp_vector_show(struct device *dev, 3019 struct device_attribute *attr, char *buf) 3020 { 3021 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3022 3023 return sysfs_emit(buf, "%d\n", target->comp_vector); 3024 } 3025 3026 static DEVICE_ATTR_RO(comp_vector); 3027 3028 static ssize_t tl_retry_count_show(struct device *dev, 3029 struct device_attribute *attr, char *buf) 3030 { 3031 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3032 3033 return sysfs_emit(buf, "%d\n", target->tl_retry_count); 3034 } 3035 3036 static DEVICE_ATTR_RO(tl_retry_count); 3037 3038 static ssize_t cmd_sg_entries_show(struct device *dev, 3039 struct device_attribute *attr, char *buf) 3040 { 3041 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3042 3043 return sysfs_emit(buf, "%u\n", target->cmd_sg_cnt); 3044 } 3045 3046 static DEVICE_ATTR_RO(cmd_sg_entries); 3047 3048 static ssize_t allow_ext_sg_show(struct device *dev, 3049 struct device_attribute *attr, char *buf) 3050 { 3051 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3052 3053 return sysfs_emit(buf, "%s\n", target->allow_ext_sg ? "true" : "false"); 3054 } 3055 3056 static DEVICE_ATTR_RO(allow_ext_sg); 3057 3058 static struct attribute *srp_host_attrs[] = { 3059 &dev_attr_id_ext.attr, 3060 &dev_attr_ioc_guid.attr, 3061 &dev_attr_service_id.attr, 3062 &dev_attr_pkey.attr, 3063 &dev_attr_sgid.attr, 3064 &dev_attr_dgid.attr, 3065 &dev_attr_orig_dgid.attr, 3066 &dev_attr_req_lim.attr, 3067 &dev_attr_zero_req_lim.attr, 3068 &dev_attr_local_ib_port.attr, 3069 &dev_attr_local_ib_device.attr, 3070 &dev_attr_ch_count.attr, 3071 &dev_attr_comp_vector.attr, 3072 &dev_attr_tl_retry_count.attr, 3073 &dev_attr_cmd_sg_entries.attr, 3074 &dev_attr_allow_ext_sg.attr, 3075 NULL 3076 }; 3077 3078 ATTRIBUTE_GROUPS(srp_host); 3079 3080 static struct scsi_host_template srp_template = { 3081 .module = THIS_MODULE, 3082 .name = "InfiniBand SRP initiator", 3083 .proc_name = DRV_NAME, 3084 .target_alloc = srp_target_alloc, 3085 .slave_configure = srp_slave_configure, 3086 .info = srp_target_info, 3087 .init_cmd_priv = srp_init_cmd_priv, 3088 .exit_cmd_priv = srp_exit_cmd_priv, 3089 .queuecommand = srp_queuecommand, 3090 .change_queue_depth = srp_change_queue_depth, 3091 .eh_timed_out = srp_timed_out, 3092 .eh_abort_handler = srp_abort, 3093 .eh_device_reset_handler = srp_reset_device, 3094 .eh_host_reset_handler = srp_reset_host, 3095 .skip_settle_delay = true, 3096 .sg_tablesize = SRP_DEF_SG_TABLESIZE, 3097 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE, 3098 .this_id = -1, 3099 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE, 3100 .shost_groups = srp_host_groups, 3101 .track_queue_depth = 1, 3102 .cmd_size = sizeof(struct srp_request), 3103 }; 3104 3105 static int srp_sdev_count(struct Scsi_Host *host) 3106 { 3107 struct scsi_device *sdev; 3108 int c = 0; 3109 3110 shost_for_each_device(sdev, host) 3111 c++; 3112 3113 return c; 3114 } 3115 3116 /* 3117 * Return values: 3118 * < 0 upon failure. Caller is responsible for SRP target port cleanup. 3119 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port 3120 * removal has been scheduled. 3121 * 0 and target->state != SRP_TARGET_REMOVED upon success. 3122 */ 3123 static int srp_add_target(struct srp_host *host, struct srp_target_port *target) 3124 { 3125 struct srp_rport_identifiers ids; 3126 struct srp_rport *rport; 3127 3128 target->state = SRP_TARGET_SCANNING; 3129 sprintf(target->target_name, "SRP.T10:%016llX", 3130 be64_to_cpu(target->id_ext)); 3131 3132 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent)) 3133 return -ENODEV; 3134 3135 memcpy(ids.port_id, &target->id_ext, 8); 3136 memcpy(ids.port_id + 8, &target->ioc_guid, 8); 3137 ids.roles = SRP_RPORT_ROLE_TARGET; 3138 rport = srp_rport_add(target->scsi_host, &ids); 3139 if (IS_ERR(rport)) { 3140 scsi_remove_host(target->scsi_host); 3141 return PTR_ERR(rport); 3142 } 3143 3144 rport->lld_data = target; 3145 target->rport = rport; 3146 3147 spin_lock(&host->target_lock); 3148 list_add_tail(&target->list, &host->target_list); 3149 spin_unlock(&host->target_lock); 3150 3151 scsi_scan_target(&target->scsi_host->shost_gendev, 3152 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); 3153 3154 if (srp_connected_ch(target) < target->ch_count || 3155 target->qp_in_error) { 3156 shost_printk(KERN_INFO, target->scsi_host, 3157 PFX "SCSI scan failed - removing SCSI host\n"); 3158 srp_queue_remove_work(target); 3159 goto out; 3160 } 3161 3162 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n", 3163 dev_name(&target->scsi_host->shost_gendev), 3164 srp_sdev_count(target->scsi_host)); 3165 3166 spin_lock_irq(&target->lock); 3167 if (target->state == SRP_TARGET_SCANNING) 3168 target->state = SRP_TARGET_LIVE; 3169 spin_unlock_irq(&target->lock); 3170 3171 out: 3172 return 0; 3173 } 3174 3175 static void srp_release_dev(struct device *dev) 3176 { 3177 struct srp_host *host = 3178 container_of(dev, struct srp_host, dev); 3179 3180 kfree(host); 3181 } 3182 3183 static struct attribute *srp_class_attrs[]; 3184 3185 ATTRIBUTE_GROUPS(srp_class); 3186 3187 static struct class srp_class = { 3188 .name = "infiniband_srp", 3189 .dev_groups = srp_class_groups, 3190 .dev_release = srp_release_dev 3191 }; 3192 3193 /** 3194 * srp_conn_unique() - check whether the connection to a target is unique 3195 * @host: SRP host. 3196 * @target: SRP target port. 3197 */ 3198 static bool srp_conn_unique(struct srp_host *host, 3199 struct srp_target_port *target) 3200 { 3201 struct srp_target_port *t; 3202 bool ret = false; 3203 3204 if (target->state == SRP_TARGET_REMOVED) 3205 goto out; 3206 3207 ret = true; 3208 3209 spin_lock(&host->target_lock); 3210 list_for_each_entry(t, &host->target_list, list) { 3211 if (t != target && 3212 target->id_ext == t->id_ext && 3213 target->ioc_guid == t->ioc_guid && 3214 target->initiator_ext == t->initiator_ext) { 3215 ret = false; 3216 break; 3217 } 3218 } 3219 spin_unlock(&host->target_lock); 3220 3221 out: 3222 return ret; 3223 } 3224 3225 /* 3226 * Target ports are added by writing 3227 * 3228 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>, 3229 * pkey=<P_Key>,service_id=<service ID> 3230 * or 3231 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>, 3232 * [src=<IPv4 address>,]dest=<IPv4 address>:<port number> 3233 * 3234 * to the add_target sysfs attribute. 3235 */ 3236 enum { 3237 SRP_OPT_ERR = 0, 3238 SRP_OPT_ID_EXT = 1 << 0, 3239 SRP_OPT_IOC_GUID = 1 << 1, 3240 SRP_OPT_DGID = 1 << 2, 3241 SRP_OPT_PKEY = 1 << 3, 3242 SRP_OPT_SERVICE_ID = 1 << 4, 3243 SRP_OPT_MAX_SECT = 1 << 5, 3244 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, 3245 SRP_OPT_IO_CLASS = 1 << 7, 3246 SRP_OPT_INITIATOR_EXT = 1 << 8, 3247 SRP_OPT_CMD_SG_ENTRIES = 1 << 9, 3248 SRP_OPT_ALLOW_EXT_SG = 1 << 10, 3249 SRP_OPT_SG_TABLESIZE = 1 << 11, 3250 SRP_OPT_COMP_VECTOR = 1 << 12, 3251 SRP_OPT_TL_RETRY_COUNT = 1 << 13, 3252 SRP_OPT_QUEUE_SIZE = 1 << 14, 3253 SRP_OPT_IP_SRC = 1 << 15, 3254 SRP_OPT_IP_DEST = 1 << 16, 3255 SRP_OPT_TARGET_CAN_QUEUE= 1 << 17, 3256 SRP_OPT_MAX_IT_IU_SIZE = 1 << 18, 3257 SRP_OPT_CH_COUNT = 1 << 19, 3258 }; 3259 3260 static unsigned int srp_opt_mandatory[] = { 3261 SRP_OPT_ID_EXT | 3262 SRP_OPT_IOC_GUID | 3263 SRP_OPT_DGID | 3264 SRP_OPT_PKEY | 3265 SRP_OPT_SERVICE_ID, 3266 SRP_OPT_ID_EXT | 3267 SRP_OPT_IOC_GUID | 3268 SRP_OPT_IP_DEST, 3269 }; 3270 3271 static const match_table_t srp_opt_tokens = { 3272 { SRP_OPT_ID_EXT, "id_ext=%s" }, 3273 { SRP_OPT_IOC_GUID, "ioc_guid=%s" }, 3274 { SRP_OPT_DGID, "dgid=%s" }, 3275 { SRP_OPT_PKEY, "pkey=%x" }, 3276 { SRP_OPT_SERVICE_ID, "service_id=%s" }, 3277 { SRP_OPT_MAX_SECT, "max_sect=%d" }, 3278 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, 3279 { SRP_OPT_TARGET_CAN_QUEUE, "target_can_queue=%d" }, 3280 { SRP_OPT_IO_CLASS, "io_class=%x" }, 3281 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, 3282 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, 3283 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, 3284 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, 3285 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" }, 3286 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" }, 3287 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" }, 3288 { SRP_OPT_IP_SRC, "src=%s" }, 3289 { SRP_OPT_IP_DEST, "dest=%s" }, 3290 { SRP_OPT_MAX_IT_IU_SIZE, "max_it_iu_size=%d" }, 3291 { SRP_OPT_CH_COUNT, "ch_count=%u", }, 3292 { SRP_OPT_ERR, NULL } 3293 }; 3294 3295 /** 3296 * srp_parse_in - parse an IP address and port number combination 3297 * @net: [in] Network namespace. 3298 * @sa: [out] Address family, IP address and port number. 3299 * @addr_port_str: [in] IP address and port number. 3300 * @has_port: [out] Whether or not @addr_port_str includes a port number. 3301 * 3302 * Parse the following address formats: 3303 * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5. 3304 * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5. 3305 */ 3306 static int srp_parse_in(struct net *net, struct sockaddr_storage *sa, 3307 const char *addr_port_str, bool *has_port) 3308 { 3309 char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL); 3310 char *port_str; 3311 int ret; 3312 3313 if (!addr) 3314 return -ENOMEM; 3315 port_str = strrchr(addr, ':'); 3316 if (port_str && strchr(port_str, ']')) 3317 port_str = NULL; 3318 if (port_str) 3319 *port_str++ = '\0'; 3320 if (has_port) 3321 *has_port = port_str != NULL; 3322 ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa); 3323 if (ret && addr[0]) { 3324 addr_end = addr + strlen(addr) - 1; 3325 if (addr[0] == '[' && *addr_end == ']') { 3326 *addr_end = '\0'; 3327 ret = inet_pton_with_scope(net, AF_INET6, addr + 1, 3328 port_str, sa); 3329 } 3330 } 3331 kfree(addr); 3332 pr_debug("%s -> %pISpfsc\n", addr_port_str, sa); 3333 return ret; 3334 } 3335 3336 static int srp_parse_options(struct net *net, const char *buf, 3337 struct srp_target_port *target) 3338 { 3339 char *options, *sep_opt; 3340 char *p; 3341 substring_t args[MAX_OPT_ARGS]; 3342 unsigned long long ull; 3343 bool has_port; 3344 int opt_mask = 0; 3345 int token; 3346 int ret = -EINVAL; 3347 int i; 3348 3349 options = kstrdup(buf, GFP_KERNEL); 3350 if (!options) 3351 return -ENOMEM; 3352 3353 sep_opt = options; 3354 while ((p = strsep(&sep_opt, ",\n")) != NULL) { 3355 if (!*p) 3356 continue; 3357 3358 token = match_token(p, srp_opt_tokens, args); 3359 opt_mask |= token; 3360 3361 switch (token) { 3362 case SRP_OPT_ID_EXT: 3363 p = match_strdup(args); 3364 if (!p) { 3365 ret = -ENOMEM; 3366 goto out; 3367 } 3368 ret = kstrtoull(p, 16, &ull); 3369 if (ret) { 3370 pr_warn("invalid id_ext parameter '%s'\n", p); 3371 kfree(p); 3372 goto out; 3373 } 3374 target->id_ext = cpu_to_be64(ull); 3375 kfree(p); 3376 break; 3377 3378 case SRP_OPT_IOC_GUID: 3379 p = match_strdup(args); 3380 if (!p) { 3381 ret = -ENOMEM; 3382 goto out; 3383 } 3384 ret = kstrtoull(p, 16, &ull); 3385 if (ret) { 3386 pr_warn("invalid ioc_guid parameter '%s'\n", p); 3387 kfree(p); 3388 goto out; 3389 } 3390 target->ioc_guid = cpu_to_be64(ull); 3391 kfree(p); 3392 break; 3393 3394 case SRP_OPT_DGID: 3395 p = match_strdup(args); 3396 if (!p) { 3397 ret = -ENOMEM; 3398 goto out; 3399 } 3400 if (strlen(p) != 32) { 3401 pr_warn("bad dest GID parameter '%s'\n", p); 3402 kfree(p); 3403 goto out; 3404 } 3405 3406 ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16); 3407 kfree(p); 3408 if (ret < 0) 3409 goto out; 3410 break; 3411 3412 case SRP_OPT_PKEY: 3413 if (match_hex(args, &token)) { 3414 pr_warn("bad P_Key parameter '%s'\n", p); 3415 goto out; 3416 } 3417 target->ib_cm.pkey = cpu_to_be16(token); 3418 break; 3419 3420 case SRP_OPT_SERVICE_ID: 3421 p = match_strdup(args); 3422 if (!p) { 3423 ret = -ENOMEM; 3424 goto out; 3425 } 3426 ret = kstrtoull(p, 16, &ull); 3427 if (ret) { 3428 pr_warn("bad service_id parameter '%s'\n", p); 3429 kfree(p); 3430 goto out; 3431 } 3432 target->ib_cm.service_id = cpu_to_be64(ull); 3433 kfree(p); 3434 break; 3435 3436 case SRP_OPT_IP_SRC: 3437 p = match_strdup(args); 3438 if (!p) { 3439 ret = -ENOMEM; 3440 goto out; 3441 } 3442 ret = srp_parse_in(net, &target->rdma_cm.src.ss, p, 3443 NULL); 3444 if (ret < 0) { 3445 pr_warn("bad source parameter '%s'\n", p); 3446 kfree(p); 3447 goto out; 3448 } 3449 target->rdma_cm.src_specified = true; 3450 kfree(p); 3451 break; 3452 3453 case SRP_OPT_IP_DEST: 3454 p = match_strdup(args); 3455 if (!p) { 3456 ret = -ENOMEM; 3457 goto out; 3458 } 3459 ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p, 3460 &has_port); 3461 if (!has_port) 3462 ret = -EINVAL; 3463 if (ret < 0) { 3464 pr_warn("bad dest parameter '%s'\n", p); 3465 kfree(p); 3466 goto out; 3467 } 3468 target->using_rdma_cm = true; 3469 kfree(p); 3470 break; 3471 3472 case SRP_OPT_MAX_SECT: 3473 if (match_int(args, &token)) { 3474 pr_warn("bad max sect parameter '%s'\n", p); 3475 goto out; 3476 } 3477 target->scsi_host->max_sectors = token; 3478 break; 3479 3480 case SRP_OPT_QUEUE_SIZE: 3481 if (match_int(args, &token) || token < 1) { 3482 pr_warn("bad queue_size parameter '%s'\n", p); 3483 goto out; 3484 } 3485 target->scsi_host->can_queue = token; 3486 target->queue_size = token + SRP_RSP_SQ_SIZE + 3487 SRP_TSK_MGMT_SQ_SIZE; 3488 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3489 target->scsi_host->cmd_per_lun = token; 3490 break; 3491 3492 case SRP_OPT_MAX_CMD_PER_LUN: 3493 if (match_int(args, &token) || token < 1) { 3494 pr_warn("bad max cmd_per_lun parameter '%s'\n", 3495 p); 3496 goto out; 3497 } 3498 target->scsi_host->cmd_per_lun = token; 3499 break; 3500 3501 case SRP_OPT_TARGET_CAN_QUEUE: 3502 if (match_int(args, &token) || token < 1) { 3503 pr_warn("bad max target_can_queue parameter '%s'\n", 3504 p); 3505 goto out; 3506 } 3507 target->target_can_queue = token; 3508 break; 3509 3510 case SRP_OPT_IO_CLASS: 3511 if (match_hex(args, &token)) { 3512 pr_warn("bad IO class parameter '%s'\n", p); 3513 goto out; 3514 } 3515 if (token != SRP_REV10_IB_IO_CLASS && 3516 token != SRP_REV16A_IB_IO_CLASS) { 3517 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n", 3518 token, SRP_REV10_IB_IO_CLASS, 3519 SRP_REV16A_IB_IO_CLASS); 3520 goto out; 3521 } 3522 target->io_class = token; 3523 break; 3524 3525 case SRP_OPT_INITIATOR_EXT: 3526 p = match_strdup(args); 3527 if (!p) { 3528 ret = -ENOMEM; 3529 goto out; 3530 } 3531 ret = kstrtoull(p, 16, &ull); 3532 if (ret) { 3533 pr_warn("bad initiator_ext value '%s'\n", p); 3534 kfree(p); 3535 goto out; 3536 } 3537 target->initiator_ext = cpu_to_be64(ull); 3538 kfree(p); 3539 break; 3540 3541 case SRP_OPT_CMD_SG_ENTRIES: 3542 if (match_int(args, &token) || token < 1 || token > 255) { 3543 pr_warn("bad max cmd_sg_entries parameter '%s'\n", 3544 p); 3545 goto out; 3546 } 3547 target->cmd_sg_cnt = token; 3548 break; 3549 3550 case SRP_OPT_ALLOW_EXT_SG: 3551 if (match_int(args, &token)) { 3552 pr_warn("bad allow_ext_sg parameter '%s'\n", p); 3553 goto out; 3554 } 3555 target->allow_ext_sg = !!token; 3556 break; 3557 3558 case SRP_OPT_SG_TABLESIZE: 3559 if (match_int(args, &token) || token < 1 || 3560 token > SG_MAX_SEGMENTS) { 3561 pr_warn("bad max sg_tablesize parameter '%s'\n", 3562 p); 3563 goto out; 3564 } 3565 target->sg_tablesize = token; 3566 break; 3567 3568 case SRP_OPT_COMP_VECTOR: 3569 if (match_int(args, &token) || token < 0) { 3570 pr_warn("bad comp_vector parameter '%s'\n", p); 3571 goto out; 3572 } 3573 target->comp_vector = token; 3574 break; 3575 3576 case SRP_OPT_TL_RETRY_COUNT: 3577 if (match_int(args, &token) || token < 2 || token > 7) { 3578 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n", 3579 p); 3580 goto out; 3581 } 3582 target->tl_retry_count = token; 3583 break; 3584 3585 case SRP_OPT_MAX_IT_IU_SIZE: 3586 if (match_int(args, &token) || token < 0) { 3587 pr_warn("bad maximum initiator to target IU size '%s'\n", p); 3588 goto out; 3589 } 3590 target->max_it_iu_size = token; 3591 break; 3592 3593 case SRP_OPT_CH_COUNT: 3594 if (match_int(args, &token) || token < 1) { 3595 pr_warn("bad channel count %s\n", p); 3596 goto out; 3597 } 3598 target->ch_count = token; 3599 break; 3600 3601 default: 3602 pr_warn("unknown parameter or missing value '%s' in target creation request\n", 3603 p); 3604 goto out; 3605 } 3606 } 3607 3608 for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) { 3609 if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) { 3610 ret = 0; 3611 break; 3612 } 3613 } 3614 if (ret) 3615 pr_warn("target creation request is missing one or more parameters\n"); 3616 3617 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue 3618 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3619 pr_warn("cmd_per_lun = %d > queue_size = %d\n", 3620 target->scsi_host->cmd_per_lun, 3621 target->scsi_host->can_queue); 3622 3623 out: 3624 kfree(options); 3625 return ret; 3626 } 3627 3628 static ssize_t add_target_store(struct device *dev, 3629 struct device_attribute *attr, const char *buf, 3630 size_t count) 3631 { 3632 struct srp_host *host = 3633 container_of(dev, struct srp_host, dev); 3634 struct Scsi_Host *target_host; 3635 struct srp_target_port *target; 3636 struct srp_rdma_ch *ch; 3637 struct srp_device *srp_dev = host->srp_dev; 3638 struct ib_device *ibdev = srp_dev->dev; 3639 int ret, i, ch_idx; 3640 unsigned int max_sectors_per_mr, mr_per_cmd = 0; 3641 bool multich = false; 3642 uint32_t max_iu_len; 3643 3644 target_host = scsi_host_alloc(&srp_template, 3645 sizeof (struct srp_target_port)); 3646 if (!target_host) 3647 return -ENOMEM; 3648 3649 target_host->transportt = ib_srp_transport_template; 3650 target_host->max_channel = 0; 3651 target_host->max_id = 1; 3652 target_host->max_lun = -1LL; 3653 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; 3654 target_host->max_segment_size = ib_dma_max_seg_size(ibdev); 3655 3656 if (!(ibdev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)) 3657 target_host->virt_boundary_mask = ~srp_dev->mr_page_mask; 3658 3659 target = host_to_target(target_host); 3660 3661 target->net = kobj_ns_grab_current(KOBJ_NS_TYPE_NET); 3662 target->io_class = SRP_REV16A_IB_IO_CLASS; 3663 target->scsi_host = target_host; 3664 target->srp_host = host; 3665 target->lkey = host->srp_dev->pd->local_dma_lkey; 3666 target->global_rkey = host->srp_dev->global_rkey; 3667 target->cmd_sg_cnt = cmd_sg_entries; 3668 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries; 3669 target->allow_ext_sg = allow_ext_sg; 3670 target->tl_retry_count = 7; 3671 target->queue_size = SRP_DEFAULT_QUEUE_SIZE; 3672 3673 /* 3674 * Avoid that the SCSI host can be removed by srp_remove_target() 3675 * before this function returns. 3676 */ 3677 scsi_host_get(target->scsi_host); 3678 3679 ret = mutex_lock_interruptible(&host->add_target_mutex); 3680 if (ret < 0) 3681 goto put; 3682 3683 ret = srp_parse_options(target->net, buf, target); 3684 if (ret) 3685 goto out; 3686 3687 if (!srp_conn_unique(target->srp_host, target)) { 3688 if (target->using_rdma_cm) { 3689 shost_printk(KERN_INFO, target->scsi_host, 3690 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n", 3691 be64_to_cpu(target->id_ext), 3692 be64_to_cpu(target->ioc_guid), 3693 &target->rdma_cm.dst); 3694 } else { 3695 shost_printk(KERN_INFO, target->scsi_host, 3696 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n", 3697 be64_to_cpu(target->id_ext), 3698 be64_to_cpu(target->ioc_guid), 3699 be64_to_cpu(target->initiator_ext)); 3700 } 3701 ret = -EEXIST; 3702 goto out; 3703 } 3704 3705 if (!srp_dev->has_fr && !target->allow_ext_sg && 3706 target->cmd_sg_cnt < target->sg_tablesize) { 3707 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n"); 3708 target->sg_tablesize = target->cmd_sg_cnt; 3709 } 3710 3711 if (srp_dev->use_fast_reg) { 3712 bool gaps_reg = ibdev->attrs.kernel_cap_flags & 3713 IBK_SG_GAPS_REG; 3714 3715 max_sectors_per_mr = srp_dev->max_pages_per_mr << 3716 (ilog2(srp_dev->mr_page_size) - 9); 3717 if (!gaps_reg) { 3718 /* 3719 * FR can only map one HCA page per entry. If the start 3720 * address is not aligned on a HCA page boundary two 3721 * entries will be used for the head and the tail 3722 * although these two entries combined contain at most 3723 * one HCA page of data. Hence the "+ 1" in the 3724 * calculation below. 3725 * 3726 * The indirect data buffer descriptor is contiguous 3727 * so the memory for that buffer will only be 3728 * registered if register_always is true. Hence add 3729 * one to mr_per_cmd if register_always has been set. 3730 */ 3731 mr_per_cmd = register_always + 3732 (target->scsi_host->max_sectors + 1 + 3733 max_sectors_per_mr - 1) / max_sectors_per_mr; 3734 } else { 3735 mr_per_cmd = register_always + 3736 (target->sg_tablesize + 3737 srp_dev->max_pages_per_mr - 1) / 3738 srp_dev->max_pages_per_mr; 3739 } 3740 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n", 3741 target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size, 3742 max_sectors_per_mr, mr_per_cmd); 3743 } 3744 3745 target_host->sg_tablesize = target->sg_tablesize; 3746 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd; 3747 target->mr_per_cmd = mr_per_cmd; 3748 target->indirect_size = target->sg_tablesize * 3749 sizeof (struct srp_direct_buf); 3750 max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 3751 srp_use_imm_data, 3752 target->max_it_iu_size); 3753 3754 INIT_WORK(&target->tl_err_work, srp_tl_err_work); 3755 INIT_WORK(&target->remove_work, srp_remove_work); 3756 spin_lock_init(&target->lock); 3757 ret = rdma_query_gid(ibdev, host->port, 0, &target->sgid); 3758 if (ret) 3759 goto out; 3760 3761 ret = -ENOMEM; 3762 if (target->ch_count == 0) { 3763 target->ch_count = 3764 min(ch_count ?: 3765 max(4 * num_online_nodes(), 3766 ibdev->num_comp_vectors), 3767 num_online_cpus()); 3768 } 3769 3770 target->ch = kcalloc(target->ch_count, sizeof(*target->ch), 3771 GFP_KERNEL); 3772 if (!target->ch) 3773 goto out; 3774 3775 for (ch_idx = 0; ch_idx < target->ch_count; ++ch_idx) { 3776 ch = &target->ch[ch_idx]; 3777 ch->target = target; 3778 ch->comp_vector = ch_idx % ibdev->num_comp_vectors; 3779 spin_lock_init(&ch->lock); 3780 INIT_LIST_HEAD(&ch->free_tx); 3781 ret = srp_new_cm_id(ch); 3782 if (ret) 3783 goto err_disconnect; 3784 3785 ret = srp_create_ch_ib(ch); 3786 if (ret) 3787 goto err_disconnect; 3788 3789 ret = srp_connect_ch(ch, max_iu_len, multich); 3790 if (ret) { 3791 char dst[64]; 3792 3793 if (target->using_rdma_cm) 3794 snprintf(dst, sizeof(dst), "%pIS", 3795 &target->rdma_cm.dst); 3796 else 3797 snprintf(dst, sizeof(dst), "%pI6", 3798 target->ib_cm.orig_dgid.raw); 3799 shost_printk(KERN_ERR, target->scsi_host, 3800 PFX "Connection %d/%d to %s failed\n", 3801 ch_idx, 3802 target->ch_count, dst); 3803 if (ch_idx == 0) { 3804 goto free_ch; 3805 } else { 3806 srp_free_ch_ib(target, ch); 3807 target->ch_count = ch - target->ch; 3808 goto connected; 3809 } 3810 } 3811 multich = true; 3812 } 3813 3814 connected: 3815 target->scsi_host->nr_hw_queues = target->ch_count; 3816 3817 ret = srp_add_target(host, target); 3818 if (ret) 3819 goto err_disconnect; 3820 3821 if (target->state != SRP_TARGET_REMOVED) { 3822 if (target->using_rdma_cm) { 3823 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3824 "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n", 3825 be64_to_cpu(target->id_ext), 3826 be64_to_cpu(target->ioc_guid), 3827 target->sgid.raw, &target->rdma_cm.dst); 3828 } else { 3829 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3830 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n", 3831 be64_to_cpu(target->id_ext), 3832 be64_to_cpu(target->ioc_guid), 3833 be16_to_cpu(target->ib_cm.pkey), 3834 be64_to_cpu(target->ib_cm.service_id), 3835 target->sgid.raw, 3836 target->ib_cm.orig_dgid.raw); 3837 } 3838 } 3839 3840 ret = count; 3841 3842 out: 3843 mutex_unlock(&host->add_target_mutex); 3844 3845 put: 3846 scsi_host_put(target->scsi_host); 3847 if (ret < 0) { 3848 /* 3849 * If a call to srp_remove_target() has not been scheduled, 3850 * drop the network namespace reference now that was obtained 3851 * earlier in this function. 3852 */ 3853 if (target->state != SRP_TARGET_REMOVED) 3854 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 3855 scsi_host_put(target->scsi_host); 3856 } 3857 3858 return ret; 3859 3860 err_disconnect: 3861 srp_disconnect_target(target); 3862 3863 free_ch: 3864 for (i = 0; i < target->ch_count; i++) { 3865 ch = &target->ch[i]; 3866 srp_free_ch_ib(target, ch); 3867 } 3868 3869 kfree(target->ch); 3870 goto out; 3871 } 3872 3873 static DEVICE_ATTR_WO(add_target); 3874 3875 static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr, 3876 char *buf) 3877 { 3878 struct srp_host *host = container_of(dev, struct srp_host, dev); 3879 3880 return sysfs_emit(buf, "%s\n", dev_name(&host->srp_dev->dev->dev)); 3881 } 3882 3883 static DEVICE_ATTR_RO(ibdev); 3884 3885 static ssize_t port_show(struct device *dev, struct device_attribute *attr, 3886 char *buf) 3887 { 3888 struct srp_host *host = container_of(dev, struct srp_host, dev); 3889 3890 return sysfs_emit(buf, "%u\n", host->port); 3891 } 3892 3893 static DEVICE_ATTR_RO(port); 3894 3895 static struct attribute *srp_class_attrs[] = { 3896 &dev_attr_add_target.attr, 3897 &dev_attr_ibdev.attr, 3898 &dev_attr_port.attr, 3899 NULL 3900 }; 3901 3902 static struct srp_host *srp_add_port(struct srp_device *device, u32 port) 3903 { 3904 struct srp_host *host; 3905 3906 host = kzalloc(sizeof *host, GFP_KERNEL); 3907 if (!host) 3908 return NULL; 3909 3910 INIT_LIST_HEAD(&host->target_list); 3911 spin_lock_init(&host->target_lock); 3912 mutex_init(&host->add_target_mutex); 3913 host->srp_dev = device; 3914 host->port = port; 3915 3916 device_initialize(&host->dev); 3917 host->dev.class = &srp_class; 3918 host->dev.parent = device->dev->dev.parent; 3919 if (dev_set_name(&host->dev, "srp-%s-%u", dev_name(&device->dev->dev), 3920 port)) 3921 goto put_host; 3922 if (device_add(&host->dev)) 3923 goto put_host; 3924 3925 return host; 3926 3927 put_host: 3928 device_del(&host->dev); 3929 put_device(&host->dev); 3930 return NULL; 3931 } 3932 3933 static void srp_rename_dev(struct ib_device *device, void *client_data) 3934 { 3935 struct srp_device *srp_dev = client_data; 3936 struct srp_host *host, *tmp_host; 3937 3938 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 3939 char name[IB_DEVICE_NAME_MAX + 8]; 3940 3941 snprintf(name, sizeof(name), "srp-%s-%u", 3942 dev_name(&device->dev), host->port); 3943 device_rename(&host->dev, name); 3944 } 3945 } 3946 3947 static int srp_add_one(struct ib_device *device) 3948 { 3949 struct srp_device *srp_dev; 3950 struct ib_device_attr *attr = &device->attrs; 3951 struct srp_host *host; 3952 int mr_page_shift; 3953 u32 p; 3954 u64 max_pages_per_mr; 3955 unsigned int flags = 0; 3956 3957 srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL); 3958 if (!srp_dev) 3959 return -ENOMEM; 3960 3961 /* 3962 * Use the smallest page size supported by the HCA, down to a 3963 * minimum of 4096 bytes. We're unlikely to build large sglists 3964 * out of smaller entries. 3965 */ 3966 mr_page_shift = max(12, ffs(attr->page_size_cap) - 1); 3967 srp_dev->mr_page_size = 1 << mr_page_shift; 3968 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1); 3969 max_pages_per_mr = attr->max_mr_size; 3970 do_div(max_pages_per_mr, srp_dev->mr_page_size); 3971 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__, 3972 attr->max_mr_size, srp_dev->mr_page_size, 3973 max_pages_per_mr, SRP_MAX_PAGES_PER_MR); 3974 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR, 3975 max_pages_per_mr); 3976 3977 srp_dev->has_fr = (attr->device_cap_flags & 3978 IB_DEVICE_MEM_MGT_EXTENSIONS); 3979 if (!never_register && !srp_dev->has_fr) 3980 dev_warn(&device->dev, "FR is not supported\n"); 3981 else if (!never_register && 3982 attr->max_mr_size >= 2 * srp_dev->mr_page_size) 3983 srp_dev->use_fast_reg = srp_dev->has_fr; 3984 3985 if (never_register || !register_always || !srp_dev->has_fr) 3986 flags |= IB_PD_UNSAFE_GLOBAL_RKEY; 3987 3988 if (srp_dev->use_fast_reg) { 3989 srp_dev->max_pages_per_mr = 3990 min_t(u32, srp_dev->max_pages_per_mr, 3991 attr->max_fast_reg_page_list_len); 3992 } 3993 srp_dev->mr_max_size = srp_dev->mr_page_size * 3994 srp_dev->max_pages_per_mr; 3995 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n", 3996 dev_name(&device->dev), mr_page_shift, attr->max_mr_size, 3997 attr->max_fast_reg_page_list_len, 3998 srp_dev->max_pages_per_mr, srp_dev->mr_max_size); 3999 4000 INIT_LIST_HEAD(&srp_dev->dev_list); 4001 4002 srp_dev->dev = device; 4003 srp_dev->pd = ib_alloc_pd(device, flags); 4004 if (IS_ERR(srp_dev->pd)) { 4005 int ret = PTR_ERR(srp_dev->pd); 4006 4007 kfree(srp_dev); 4008 return ret; 4009 } 4010 4011 if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) { 4012 srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey; 4013 WARN_ON_ONCE(srp_dev->global_rkey == 0); 4014 } 4015 4016 rdma_for_each_port (device, p) { 4017 host = srp_add_port(srp_dev, p); 4018 if (host) 4019 list_add_tail(&host->list, &srp_dev->dev_list); 4020 } 4021 4022 ib_set_client_data(device, &srp_client, srp_dev); 4023 return 0; 4024 } 4025 4026 static void srp_remove_one(struct ib_device *device, void *client_data) 4027 { 4028 struct srp_device *srp_dev; 4029 struct srp_host *host, *tmp_host; 4030 struct srp_target_port *target; 4031 4032 srp_dev = client_data; 4033 4034 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 4035 /* 4036 * Remove the add_target sysfs entry so that no new target ports 4037 * can be created. 4038 */ 4039 device_del(&host->dev); 4040 4041 /* 4042 * Remove all target ports. 4043 */ 4044 spin_lock(&host->target_lock); 4045 list_for_each_entry(target, &host->target_list, list) 4046 srp_queue_remove_work(target); 4047 spin_unlock(&host->target_lock); 4048 4049 /* 4050 * srp_queue_remove_work() queues a call to 4051 * srp_remove_target(). The latter function cancels 4052 * target->tl_err_work so waiting for the remove works to 4053 * finish is sufficient. 4054 */ 4055 flush_workqueue(srp_remove_wq); 4056 4057 put_device(&host->dev); 4058 } 4059 4060 ib_dealloc_pd(srp_dev->pd); 4061 4062 kfree(srp_dev); 4063 } 4064 4065 static struct srp_function_template ib_srp_transport_functions = { 4066 .has_rport_state = true, 4067 .reset_timer_if_blocked = true, 4068 .reconnect_delay = &srp_reconnect_delay, 4069 .fast_io_fail_tmo = &srp_fast_io_fail_tmo, 4070 .dev_loss_tmo = &srp_dev_loss_tmo, 4071 .reconnect = srp_rport_reconnect, 4072 .rport_delete = srp_rport_delete, 4073 .terminate_rport_io = srp_terminate_io, 4074 }; 4075 4076 static int __init srp_init_module(void) 4077 { 4078 int ret; 4079 4080 BUILD_BUG_ON(sizeof(struct srp_aer_req) != 36); 4081 BUILD_BUG_ON(sizeof(struct srp_cmd) != 48); 4082 BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 4); 4083 BUILD_BUG_ON(sizeof(struct srp_indirect_buf) != 20); 4084 BUILD_BUG_ON(sizeof(struct srp_login_req) != 64); 4085 BUILD_BUG_ON(sizeof(struct srp_login_req_rdma) != 56); 4086 BUILD_BUG_ON(sizeof(struct srp_rsp) != 36); 4087 4088 if (srp_sg_tablesize) { 4089 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); 4090 if (!cmd_sg_entries) 4091 cmd_sg_entries = srp_sg_tablesize; 4092 } 4093 4094 if (!cmd_sg_entries) 4095 cmd_sg_entries = SRP_DEF_SG_TABLESIZE; 4096 4097 if (cmd_sg_entries > 255) { 4098 pr_warn("Clamping cmd_sg_entries to 255\n"); 4099 cmd_sg_entries = 255; 4100 } 4101 4102 if (!indirect_sg_entries) 4103 indirect_sg_entries = cmd_sg_entries; 4104 else if (indirect_sg_entries < cmd_sg_entries) { 4105 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n", 4106 cmd_sg_entries); 4107 indirect_sg_entries = cmd_sg_entries; 4108 } 4109 4110 if (indirect_sg_entries > SG_MAX_SEGMENTS) { 4111 pr_warn("Clamping indirect_sg_entries to %u\n", 4112 SG_MAX_SEGMENTS); 4113 indirect_sg_entries = SG_MAX_SEGMENTS; 4114 } 4115 4116 srp_remove_wq = create_workqueue("srp_remove"); 4117 if (!srp_remove_wq) { 4118 ret = -ENOMEM; 4119 goto out; 4120 } 4121 4122 ret = -ENOMEM; 4123 ib_srp_transport_template = 4124 srp_attach_transport(&ib_srp_transport_functions); 4125 if (!ib_srp_transport_template) 4126 goto destroy_wq; 4127 4128 ret = class_register(&srp_class); 4129 if (ret) { 4130 pr_err("couldn't register class infiniband_srp\n"); 4131 goto release_tr; 4132 } 4133 4134 ib_sa_register_client(&srp_sa_client); 4135 4136 ret = ib_register_client(&srp_client); 4137 if (ret) { 4138 pr_err("couldn't register IB client\n"); 4139 goto unreg_sa; 4140 } 4141 4142 out: 4143 return ret; 4144 4145 unreg_sa: 4146 ib_sa_unregister_client(&srp_sa_client); 4147 class_unregister(&srp_class); 4148 4149 release_tr: 4150 srp_release_transport(ib_srp_transport_template); 4151 4152 destroy_wq: 4153 destroy_workqueue(srp_remove_wq); 4154 goto out; 4155 } 4156 4157 static void __exit srp_cleanup_module(void) 4158 { 4159 ib_unregister_client(&srp_client); 4160 ib_sa_unregister_client(&srp_sa_client); 4161 class_unregister(&srp_class); 4162 srp_release_transport(ib_srp_transport_template); 4163 destroy_workqueue(srp_remove_wq); 4164 } 4165 4166 module_init(srp_init_module); 4167 module_exit(srp_cleanup_module); 4168