1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 static int isert_debug_level; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static void 51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static void 56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 57 static int 58 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 59 struct isert_rdma_wr *wr); 60 static int 61 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 62 static int 63 isert_rdma_post_recvl(struct isert_conn *isert_conn); 64 static int 65 isert_rdma_accept(struct isert_conn *isert_conn); 66 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 67 68 static void isert_release_work(struct work_struct *work); 69 70 static inline bool 71 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 72 { 73 return (conn->pi_support && 74 cmd->prot_op != TARGET_PROT_NORMAL); 75 } 76 77 78 static void 79 isert_qp_event_callback(struct ib_event *e, void *context) 80 { 81 struct isert_conn *isert_conn = context; 82 83 isert_err("%s (%d): conn %p\n", 84 ib_event_msg(e->event), e->event, isert_conn); 85 86 switch (e->event) { 87 case IB_EVENT_COMM_EST: 88 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 89 break; 90 case IB_EVENT_QP_LAST_WQE_REACHED: 91 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 92 break; 93 default: 94 break; 95 } 96 } 97 98 static int 99 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 100 { 101 int ret; 102 103 ret = ib_query_device(ib_dev, devattr); 104 if (ret) { 105 isert_err("ib_query_device() failed: %d\n", ret); 106 return ret; 107 } 108 isert_dbg("devattr->max_sge: %d\n", devattr->max_sge); 109 isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 110 111 return 0; 112 } 113 114 static struct isert_comp * 115 isert_comp_get(struct isert_conn *isert_conn) 116 { 117 struct isert_device *device = isert_conn->device; 118 struct isert_comp *comp; 119 int i, min = 0; 120 121 mutex_lock(&device_list_mutex); 122 for (i = 0; i < device->comps_used; i++) 123 if (device->comps[i].active_qps < 124 device->comps[min].active_qps) 125 min = i; 126 comp = &device->comps[min]; 127 comp->active_qps++; 128 mutex_unlock(&device_list_mutex); 129 130 isert_info("conn %p, using comp %p min_index: %d\n", 131 isert_conn, comp, min); 132 133 return comp; 134 } 135 136 static void 137 isert_comp_put(struct isert_comp *comp) 138 { 139 mutex_lock(&device_list_mutex); 140 comp->active_qps--; 141 mutex_unlock(&device_list_mutex); 142 } 143 144 static struct ib_qp * 145 isert_create_qp(struct isert_conn *isert_conn, 146 struct isert_comp *comp, 147 struct rdma_cm_id *cma_id) 148 { 149 struct isert_device *device = isert_conn->device; 150 struct ib_qp_init_attr attr; 151 int ret; 152 153 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 154 attr.event_handler = isert_qp_event_callback; 155 attr.qp_context = isert_conn; 156 attr.send_cq = comp->cq; 157 attr.recv_cq = comp->cq; 158 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 159 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 160 /* 161 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 162 * work-around for RDMA_READs with ConnectX-2. 163 * 164 * Also, still make sure to have at least two SGEs for 165 * outgoing control PDU responses. 166 */ 167 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 168 isert_conn->max_sge = attr.cap.max_send_sge; 169 170 attr.cap.max_recv_sge = 1; 171 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 172 attr.qp_type = IB_QPT_RC; 173 if (device->pi_capable) 174 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 175 176 ret = rdma_create_qp(cma_id, device->pd, &attr); 177 if (ret) { 178 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 179 return ERR_PTR(ret); 180 } 181 182 return cma_id->qp; 183 } 184 185 static int 186 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 187 { 188 struct isert_comp *comp; 189 int ret; 190 191 comp = isert_comp_get(isert_conn); 192 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 193 if (IS_ERR(isert_conn->qp)) { 194 ret = PTR_ERR(isert_conn->qp); 195 goto err; 196 } 197 198 return 0; 199 err: 200 isert_comp_put(comp); 201 return ret; 202 } 203 204 static void 205 isert_cq_event_callback(struct ib_event *e, void *context) 206 { 207 isert_dbg("event: %d\n", e->event); 208 } 209 210 static int 211 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 212 { 213 struct isert_device *device = isert_conn->device; 214 struct ib_device *ib_dev = device->ib_device; 215 struct iser_rx_desc *rx_desc; 216 struct ib_sge *rx_sg; 217 u64 dma_addr; 218 int i, j; 219 220 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 221 sizeof(struct iser_rx_desc), GFP_KERNEL); 222 if (!isert_conn->rx_descs) 223 goto fail; 224 225 rx_desc = isert_conn->rx_descs; 226 227 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 228 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 229 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 230 if (ib_dma_mapping_error(ib_dev, dma_addr)) 231 goto dma_map_fail; 232 233 rx_desc->dma_addr = dma_addr; 234 235 rx_sg = &rx_desc->rx_sg; 236 rx_sg->addr = rx_desc->dma_addr; 237 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 238 rx_sg->lkey = device->pd->local_dma_lkey; 239 } 240 241 return 0; 242 243 dma_map_fail: 244 rx_desc = isert_conn->rx_descs; 245 for (j = 0; j < i; j++, rx_desc++) { 246 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 247 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 248 } 249 kfree(isert_conn->rx_descs); 250 isert_conn->rx_descs = NULL; 251 fail: 252 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 253 254 return -ENOMEM; 255 } 256 257 static void 258 isert_free_rx_descriptors(struct isert_conn *isert_conn) 259 { 260 struct ib_device *ib_dev = isert_conn->device->ib_device; 261 struct iser_rx_desc *rx_desc; 262 int i; 263 264 if (!isert_conn->rx_descs) 265 return; 266 267 rx_desc = isert_conn->rx_descs; 268 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 269 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 270 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 271 } 272 273 kfree(isert_conn->rx_descs); 274 isert_conn->rx_descs = NULL; 275 } 276 277 static void isert_cq_work(struct work_struct *); 278 static void isert_cq_callback(struct ib_cq *, void *); 279 280 static void 281 isert_free_comps(struct isert_device *device) 282 { 283 int i; 284 285 for (i = 0; i < device->comps_used; i++) { 286 struct isert_comp *comp = &device->comps[i]; 287 288 if (comp->cq) { 289 cancel_work_sync(&comp->work); 290 ib_destroy_cq(comp->cq); 291 } 292 } 293 kfree(device->comps); 294 } 295 296 static int 297 isert_alloc_comps(struct isert_device *device, 298 struct ib_device_attr *attr) 299 { 300 int i, max_cqe, ret = 0; 301 302 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 303 device->ib_device->num_comp_vectors)); 304 305 isert_info("Using %d CQs, %s supports %d vectors support " 306 "Fast registration %d pi_capable %d\n", 307 device->comps_used, device->ib_device->name, 308 device->ib_device->num_comp_vectors, device->use_fastreg, 309 device->pi_capable); 310 311 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 312 GFP_KERNEL); 313 if (!device->comps) { 314 isert_err("Unable to allocate completion contexts\n"); 315 return -ENOMEM; 316 } 317 318 max_cqe = min(ISER_MAX_CQ_LEN, attr->max_cqe); 319 320 for (i = 0; i < device->comps_used; i++) { 321 struct ib_cq_init_attr cq_attr = {}; 322 struct isert_comp *comp = &device->comps[i]; 323 324 comp->device = device; 325 INIT_WORK(&comp->work, isert_cq_work); 326 cq_attr.cqe = max_cqe; 327 cq_attr.comp_vector = i; 328 comp->cq = ib_create_cq(device->ib_device, 329 isert_cq_callback, 330 isert_cq_event_callback, 331 (void *)comp, 332 &cq_attr); 333 if (IS_ERR(comp->cq)) { 334 isert_err("Unable to allocate cq\n"); 335 ret = PTR_ERR(comp->cq); 336 comp->cq = NULL; 337 goto out_cq; 338 } 339 340 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 341 if (ret) 342 goto out_cq; 343 } 344 345 return 0; 346 out_cq: 347 isert_free_comps(device); 348 return ret; 349 } 350 351 static int 352 isert_create_device_ib_res(struct isert_device *device) 353 { 354 struct ib_device_attr *dev_attr; 355 int ret; 356 357 dev_attr = &device->dev_attr; 358 ret = isert_query_device(device->ib_device, dev_attr); 359 if (ret) 360 return ret; 361 362 /* asign function handlers */ 363 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 364 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 365 device->use_fastreg = 1; 366 device->reg_rdma_mem = isert_reg_rdma; 367 device->unreg_rdma_mem = isert_unreg_rdma; 368 } else { 369 device->use_fastreg = 0; 370 device->reg_rdma_mem = isert_map_rdma; 371 device->unreg_rdma_mem = isert_unmap_cmd; 372 } 373 374 ret = isert_alloc_comps(device, dev_attr); 375 if (ret) 376 return ret; 377 378 device->pd = ib_alloc_pd(device->ib_device); 379 if (IS_ERR(device->pd)) { 380 ret = PTR_ERR(device->pd); 381 isert_err("failed to allocate pd, device %p, ret=%d\n", 382 device, ret); 383 goto out_cq; 384 } 385 386 /* Check signature cap */ 387 device->pi_capable = dev_attr->device_cap_flags & 388 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 389 390 return 0; 391 392 out_cq: 393 isert_free_comps(device); 394 return ret; 395 } 396 397 static void 398 isert_free_device_ib_res(struct isert_device *device) 399 { 400 isert_info("device %p\n", device); 401 402 ib_dealloc_pd(device->pd); 403 isert_free_comps(device); 404 } 405 406 static void 407 isert_device_put(struct isert_device *device) 408 { 409 mutex_lock(&device_list_mutex); 410 device->refcount--; 411 isert_info("device %p refcount %d\n", device, device->refcount); 412 if (!device->refcount) { 413 isert_free_device_ib_res(device); 414 list_del(&device->dev_node); 415 kfree(device); 416 } 417 mutex_unlock(&device_list_mutex); 418 } 419 420 static struct isert_device * 421 isert_device_get(struct rdma_cm_id *cma_id) 422 { 423 struct isert_device *device; 424 int ret; 425 426 mutex_lock(&device_list_mutex); 427 list_for_each_entry(device, &device_list, dev_node) { 428 if (device->ib_device->node_guid == cma_id->device->node_guid) { 429 device->refcount++; 430 isert_info("Found iser device %p refcount %d\n", 431 device, device->refcount); 432 mutex_unlock(&device_list_mutex); 433 return device; 434 } 435 } 436 437 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 438 if (!device) { 439 mutex_unlock(&device_list_mutex); 440 return ERR_PTR(-ENOMEM); 441 } 442 443 INIT_LIST_HEAD(&device->dev_node); 444 445 device->ib_device = cma_id->device; 446 ret = isert_create_device_ib_res(device); 447 if (ret) { 448 kfree(device); 449 mutex_unlock(&device_list_mutex); 450 return ERR_PTR(ret); 451 } 452 453 device->refcount++; 454 list_add_tail(&device->dev_node, &device_list); 455 isert_info("Created a new iser device %p refcount %d\n", 456 device, device->refcount); 457 mutex_unlock(&device_list_mutex); 458 459 return device; 460 } 461 462 static void 463 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 464 { 465 struct fast_reg_descriptor *fr_desc, *tmp; 466 int i = 0; 467 468 if (list_empty(&isert_conn->fr_pool)) 469 return; 470 471 isert_info("Freeing conn %p fastreg pool", isert_conn); 472 473 list_for_each_entry_safe(fr_desc, tmp, 474 &isert_conn->fr_pool, list) { 475 list_del(&fr_desc->list); 476 ib_free_fast_reg_page_list(fr_desc->data_frpl); 477 ib_dereg_mr(fr_desc->data_mr); 478 if (fr_desc->pi_ctx) { 479 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 480 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 481 ib_dereg_mr(fr_desc->pi_ctx->sig_mr); 482 kfree(fr_desc->pi_ctx); 483 } 484 kfree(fr_desc); 485 ++i; 486 } 487 488 if (i < isert_conn->fr_pool_size) 489 isert_warn("Pool still has %d regions registered\n", 490 isert_conn->fr_pool_size - i); 491 } 492 493 static int 494 isert_create_pi_ctx(struct fast_reg_descriptor *desc, 495 struct ib_device *device, 496 struct ib_pd *pd) 497 { 498 struct pi_context *pi_ctx; 499 int ret; 500 501 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 502 if (!pi_ctx) { 503 isert_err("Failed to allocate pi context\n"); 504 return -ENOMEM; 505 } 506 507 pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(device, 508 ISCSI_ISER_SG_TABLESIZE); 509 if (IS_ERR(pi_ctx->prot_frpl)) { 510 isert_err("Failed to allocate prot frpl err=%ld\n", 511 PTR_ERR(pi_ctx->prot_frpl)); 512 ret = PTR_ERR(pi_ctx->prot_frpl); 513 goto err_pi_ctx; 514 } 515 516 pi_ctx->prot_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 517 ISCSI_ISER_SG_TABLESIZE); 518 if (IS_ERR(pi_ctx->prot_mr)) { 519 isert_err("Failed to allocate prot frmr err=%ld\n", 520 PTR_ERR(pi_ctx->prot_mr)); 521 ret = PTR_ERR(pi_ctx->prot_mr); 522 goto err_prot_frpl; 523 } 524 desc->ind |= ISERT_PROT_KEY_VALID; 525 526 pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2); 527 if (IS_ERR(pi_ctx->sig_mr)) { 528 isert_err("Failed to allocate signature enabled mr err=%ld\n", 529 PTR_ERR(pi_ctx->sig_mr)); 530 ret = PTR_ERR(pi_ctx->sig_mr); 531 goto err_prot_mr; 532 } 533 534 desc->pi_ctx = pi_ctx; 535 desc->ind |= ISERT_SIG_KEY_VALID; 536 desc->ind &= ~ISERT_PROTECTED; 537 538 return 0; 539 540 err_prot_mr: 541 ib_dereg_mr(pi_ctx->prot_mr); 542 err_prot_frpl: 543 ib_free_fast_reg_page_list(pi_ctx->prot_frpl); 544 err_pi_ctx: 545 kfree(pi_ctx); 546 547 return ret; 548 } 549 550 static int 551 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 552 struct fast_reg_descriptor *fr_desc) 553 { 554 int ret; 555 556 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 557 ISCSI_ISER_SG_TABLESIZE); 558 if (IS_ERR(fr_desc->data_frpl)) { 559 isert_err("Failed to allocate data frpl err=%ld\n", 560 PTR_ERR(fr_desc->data_frpl)); 561 return PTR_ERR(fr_desc->data_frpl); 562 } 563 564 fr_desc->data_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 565 ISCSI_ISER_SG_TABLESIZE); 566 if (IS_ERR(fr_desc->data_mr)) { 567 isert_err("Failed to allocate data frmr err=%ld\n", 568 PTR_ERR(fr_desc->data_mr)); 569 ret = PTR_ERR(fr_desc->data_mr); 570 goto err_data_frpl; 571 } 572 fr_desc->ind |= ISERT_DATA_KEY_VALID; 573 574 isert_dbg("Created fr_desc %p\n", fr_desc); 575 576 return 0; 577 578 err_data_frpl: 579 ib_free_fast_reg_page_list(fr_desc->data_frpl); 580 581 return ret; 582 } 583 584 static int 585 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 586 { 587 struct fast_reg_descriptor *fr_desc; 588 struct isert_device *device = isert_conn->device; 589 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 590 struct se_node_acl *se_nacl = se_sess->se_node_acl; 591 int i, ret, tag_num; 592 /* 593 * Setup the number of FRMRs based upon the number of tags 594 * available to session in iscsi_target_locate_portal(). 595 */ 596 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 597 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 598 599 isert_conn->fr_pool_size = 0; 600 for (i = 0; i < tag_num; i++) { 601 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 602 if (!fr_desc) { 603 isert_err("Failed to allocate fast_reg descriptor\n"); 604 ret = -ENOMEM; 605 goto err; 606 } 607 608 ret = isert_create_fr_desc(device->ib_device, 609 device->pd, fr_desc); 610 if (ret) { 611 isert_err("Failed to create fastreg descriptor err=%d\n", 612 ret); 613 kfree(fr_desc); 614 goto err; 615 } 616 617 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 618 isert_conn->fr_pool_size++; 619 } 620 621 isert_dbg("Creating conn %p fastreg pool size=%d", 622 isert_conn, isert_conn->fr_pool_size); 623 624 return 0; 625 626 err: 627 isert_conn_free_fastreg_pool(isert_conn); 628 return ret; 629 } 630 631 static void 632 isert_init_conn(struct isert_conn *isert_conn) 633 { 634 isert_conn->state = ISER_CONN_INIT; 635 INIT_LIST_HEAD(&isert_conn->node); 636 init_completion(&isert_conn->login_comp); 637 init_completion(&isert_conn->login_req_comp); 638 init_completion(&isert_conn->wait); 639 kref_init(&isert_conn->kref); 640 mutex_init(&isert_conn->mutex); 641 spin_lock_init(&isert_conn->pool_lock); 642 INIT_LIST_HEAD(&isert_conn->fr_pool); 643 INIT_WORK(&isert_conn->release_work, isert_release_work); 644 } 645 646 static void 647 isert_free_login_buf(struct isert_conn *isert_conn) 648 { 649 struct ib_device *ib_dev = isert_conn->device->ib_device; 650 651 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 652 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 653 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 654 ISCSI_DEF_MAX_RECV_SEG_LEN, 655 DMA_FROM_DEVICE); 656 kfree(isert_conn->login_buf); 657 } 658 659 static int 660 isert_alloc_login_buf(struct isert_conn *isert_conn, 661 struct ib_device *ib_dev) 662 { 663 int ret; 664 665 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 666 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 667 if (!isert_conn->login_buf) { 668 isert_err("Unable to allocate isert_conn->login_buf\n"); 669 return -ENOMEM; 670 } 671 672 isert_conn->login_req_buf = isert_conn->login_buf; 673 isert_conn->login_rsp_buf = isert_conn->login_buf + 674 ISCSI_DEF_MAX_RECV_SEG_LEN; 675 676 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 677 isert_conn->login_buf, isert_conn->login_req_buf, 678 isert_conn->login_rsp_buf); 679 680 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 681 (void *)isert_conn->login_req_buf, 682 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 683 684 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 685 if (ret) { 686 isert_err("login_req_dma mapping error: %d\n", ret); 687 isert_conn->login_req_dma = 0; 688 goto out_login_buf; 689 } 690 691 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 692 (void *)isert_conn->login_rsp_buf, 693 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 694 695 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 696 if (ret) { 697 isert_err("login_rsp_dma mapping error: %d\n", ret); 698 isert_conn->login_rsp_dma = 0; 699 goto out_req_dma_map; 700 } 701 702 return 0; 703 704 out_req_dma_map: 705 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 706 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 707 out_login_buf: 708 kfree(isert_conn->login_buf); 709 return ret; 710 } 711 712 static int 713 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 714 { 715 struct isert_np *isert_np = cma_id->context; 716 struct iscsi_np *np = isert_np->np; 717 struct isert_conn *isert_conn; 718 struct isert_device *device; 719 int ret = 0; 720 721 spin_lock_bh(&np->np_thread_lock); 722 if (!np->enabled) { 723 spin_unlock_bh(&np->np_thread_lock); 724 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 725 return rdma_reject(cma_id, NULL, 0); 726 } 727 spin_unlock_bh(&np->np_thread_lock); 728 729 isert_dbg("cma_id: %p, portal: %p\n", 730 cma_id, cma_id->context); 731 732 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 733 if (!isert_conn) 734 return -ENOMEM; 735 736 isert_init_conn(isert_conn); 737 isert_conn->cm_id = cma_id; 738 739 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 740 if (ret) 741 goto out; 742 743 device = isert_device_get(cma_id); 744 if (IS_ERR(device)) { 745 ret = PTR_ERR(device); 746 goto out_rsp_dma_map; 747 } 748 isert_conn->device = device; 749 750 /* Set max inflight RDMA READ requests */ 751 isert_conn->initiator_depth = min_t(u8, 752 event->param.conn.initiator_depth, 753 device->dev_attr.max_qp_init_rd_atom); 754 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 755 756 ret = isert_conn_setup_qp(isert_conn, cma_id); 757 if (ret) 758 goto out_conn_dev; 759 760 ret = isert_rdma_post_recvl(isert_conn); 761 if (ret) 762 goto out_conn_dev; 763 764 ret = isert_rdma_accept(isert_conn); 765 if (ret) 766 goto out_conn_dev; 767 768 mutex_lock(&isert_np->mutex); 769 list_add_tail(&isert_conn->node, &isert_np->accepted); 770 mutex_unlock(&isert_np->mutex); 771 772 return 0; 773 774 out_conn_dev: 775 isert_device_put(device); 776 out_rsp_dma_map: 777 isert_free_login_buf(isert_conn); 778 out: 779 kfree(isert_conn); 780 rdma_reject(cma_id, NULL, 0); 781 return ret; 782 } 783 784 static void 785 isert_connect_release(struct isert_conn *isert_conn) 786 { 787 struct isert_device *device = isert_conn->device; 788 789 isert_dbg("conn %p\n", isert_conn); 790 791 BUG_ON(!device); 792 793 if (device->use_fastreg) 794 isert_conn_free_fastreg_pool(isert_conn); 795 796 isert_free_rx_descriptors(isert_conn); 797 if (isert_conn->cm_id) 798 rdma_destroy_id(isert_conn->cm_id); 799 800 if (isert_conn->qp) { 801 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 802 803 isert_comp_put(comp); 804 ib_destroy_qp(isert_conn->qp); 805 } 806 807 if (isert_conn->login_buf) 808 isert_free_login_buf(isert_conn); 809 810 isert_device_put(device); 811 812 kfree(isert_conn); 813 } 814 815 static void 816 isert_connected_handler(struct rdma_cm_id *cma_id) 817 { 818 struct isert_conn *isert_conn = cma_id->qp->qp_context; 819 struct isert_np *isert_np = cma_id->context; 820 821 isert_info("conn %p\n", isert_conn); 822 823 mutex_lock(&isert_conn->mutex); 824 isert_conn->state = ISER_CONN_UP; 825 kref_get(&isert_conn->kref); 826 mutex_unlock(&isert_conn->mutex); 827 828 mutex_lock(&isert_np->mutex); 829 list_move_tail(&isert_conn->node, &isert_np->pending); 830 mutex_unlock(&isert_np->mutex); 831 832 isert_info("np %p: Allow accept_np to continue\n", isert_np); 833 up(&isert_np->sem); 834 } 835 836 static void 837 isert_release_kref(struct kref *kref) 838 { 839 struct isert_conn *isert_conn = container_of(kref, 840 struct isert_conn, kref); 841 842 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 843 current->pid); 844 845 isert_connect_release(isert_conn); 846 } 847 848 static void 849 isert_put_conn(struct isert_conn *isert_conn) 850 { 851 kref_put(&isert_conn->kref, isert_release_kref); 852 } 853 854 /** 855 * isert_conn_terminate() - Initiate connection termination 856 * @isert_conn: isert connection struct 857 * 858 * Notes: 859 * In case the connection state is FULL_FEATURE, move state 860 * to TEMINATING and start teardown sequence (rdma_disconnect). 861 * In case the connection state is UP, complete flush as well. 862 * 863 * This routine must be called with mutex held. Thus it is 864 * safe to call multiple times. 865 */ 866 static void 867 isert_conn_terminate(struct isert_conn *isert_conn) 868 { 869 int err; 870 871 switch (isert_conn->state) { 872 case ISER_CONN_TERMINATING: 873 break; 874 case ISER_CONN_UP: 875 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 876 isert_info("Terminating conn %p state %d\n", 877 isert_conn, isert_conn->state); 878 isert_conn->state = ISER_CONN_TERMINATING; 879 err = rdma_disconnect(isert_conn->cm_id); 880 if (err) 881 isert_warn("Failed rdma_disconnect isert_conn %p\n", 882 isert_conn); 883 break; 884 default: 885 isert_warn("conn %p teminating in state %d\n", 886 isert_conn, isert_conn->state); 887 } 888 } 889 890 static int 891 isert_np_cma_handler(struct isert_np *isert_np, 892 enum rdma_cm_event_type event) 893 { 894 isert_dbg("%s (%d): isert np %p\n", 895 rdma_event_msg(event), event, isert_np); 896 897 switch (event) { 898 case RDMA_CM_EVENT_DEVICE_REMOVAL: 899 isert_np->cm_id = NULL; 900 break; 901 case RDMA_CM_EVENT_ADDR_CHANGE: 902 isert_np->cm_id = isert_setup_id(isert_np); 903 if (IS_ERR(isert_np->cm_id)) { 904 isert_err("isert np %p setup id failed: %ld\n", 905 isert_np, PTR_ERR(isert_np->cm_id)); 906 isert_np->cm_id = NULL; 907 } 908 break; 909 default: 910 isert_err("isert np %p Unexpected event %d\n", 911 isert_np, event); 912 } 913 914 return -1; 915 } 916 917 static int 918 isert_disconnected_handler(struct rdma_cm_id *cma_id, 919 enum rdma_cm_event_type event) 920 { 921 struct isert_np *isert_np = cma_id->context; 922 struct isert_conn *isert_conn; 923 bool terminating = false; 924 925 if (isert_np->cm_id == cma_id) 926 return isert_np_cma_handler(cma_id->context, event); 927 928 isert_conn = cma_id->qp->qp_context; 929 930 mutex_lock(&isert_conn->mutex); 931 terminating = (isert_conn->state == ISER_CONN_TERMINATING); 932 isert_conn_terminate(isert_conn); 933 mutex_unlock(&isert_conn->mutex); 934 935 isert_info("conn %p completing wait\n", isert_conn); 936 complete(&isert_conn->wait); 937 938 if (terminating) 939 goto out; 940 941 mutex_lock(&isert_np->mutex); 942 if (!list_empty(&isert_conn->node)) { 943 list_del_init(&isert_conn->node); 944 isert_put_conn(isert_conn); 945 queue_work(isert_release_wq, &isert_conn->release_work); 946 } 947 mutex_unlock(&isert_np->mutex); 948 949 out: 950 return 0; 951 } 952 953 static int 954 isert_connect_error(struct rdma_cm_id *cma_id) 955 { 956 struct isert_conn *isert_conn = cma_id->qp->qp_context; 957 958 list_del_init(&isert_conn->node); 959 isert_conn->cm_id = NULL; 960 isert_put_conn(isert_conn); 961 962 return -1; 963 } 964 965 static int 966 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 967 { 968 int ret = 0; 969 970 isert_info("%s (%d): status %d id %p np %p\n", 971 rdma_event_msg(event->event), event->event, 972 event->status, cma_id, cma_id->context); 973 974 switch (event->event) { 975 case RDMA_CM_EVENT_CONNECT_REQUEST: 976 ret = isert_connect_request(cma_id, event); 977 if (ret) 978 isert_err("failed handle connect request %d\n", ret); 979 break; 980 case RDMA_CM_EVENT_ESTABLISHED: 981 isert_connected_handler(cma_id); 982 break; 983 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 984 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 985 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 986 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 987 ret = isert_disconnected_handler(cma_id, event->event); 988 break; 989 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 990 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 991 case RDMA_CM_EVENT_CONNECT_ERROR: 992 ret = isert_connect_error(cma_id); 993 break; 994 default: 995 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 996 break; 997 } 998 999 return ret; 1000 } 1001 1002 static int 1003 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 1004 { 1005 struct ib_recv_wr *rx_wr, *rx_wr_failed; 1006 int i, ret; 1007 struct iser_rx_desc *rx_desc; 1008 1009 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 1010 rx_desc = &isert_conn->rx_descs[i]; 1011 rx_wr->wr_id = (uintptr_t)rx_desc; 1012 rx_wr->sg_list = &rx_desc->rx_sg; 1013 rx_wr->num_sge = 1; 1014 rx_wr->next = rx_wr + 1; 1015 } 1016 rx_wr--; 1017 rx_wr->next = NULL; /* mark end of work requests list */ 1018 1019 isert_conn->post_recv_buf_count += count; 1020 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 1021 &rx_wr_failed); 1022 if (ret) { 1023 isert_err("ib_post_recv() failed with ret: %d\n", ret); 1024 isert_conn->post_recv_buf_count -= count; 1025 } 1026 1027 return ret; 1028 } 1029 1030 static int 1031 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 1032 { 1033 struct ib_recv_wr *rx_wr_failed, rx_wr; 1034 int ret; 1035 1036 rx_wr.wr_id = (uintptr_t)rx_desc; 1037 rx_wr.sg_list = &rx_desc->rx_sg; 1038 rx_wr.num_sge = 1; 1039 rx_wr.next = NULL; 1040 1041 isert_conn->post_recv_buf_count++; 1042 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 1043 if (ret) { 1044 isert_err("ib_post_recv() failed with ret: %d\n", ret); 1045 isert_conn->post_recv_buf_count--; 1046 } 1047 1048 return ret; 1049 } 1050 1051 static int 1052 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1053 { 1054 struct ib_device *ib_dev = isert_conn->cm_id->device; 1055 struct ib_send_wr send_wr, *send_wr_failed; 1056 int ret; 1057 1058 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1059 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1060 1061 send_wr.next = NULL; 1062 send_wr.wr_id = (uintptr_t)tx_desc; 1063 send_wr.sg_list = tx_desc->tx_sg; 1064 send_wr.num_sge = tx_desc->num_sge; 1065 send_wr.opcode = IB_WR_SEND; 1066 send_wr.send_flags = IB_SEND_SIGNALED; 1067 1068 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1069 if (ret) 1070 isert_err("ib_post_send() failed, ret: %d\n", ret); 1071 1072 return ret; 1073 } 1074 1075 static void 1076 isert_create_send_desc(struct isert_conn *isert_conn, 1077 struct isert_cmd *isert_cmd, 1078 struct iser_tx_desc *tx_desc) 1079 { 1080 struct isert_device *device = isert_conn->device; 1081 struct ib_device *ib_dev = device->ib_device; 1082 1083 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1084 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1085 1086 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 1087 tx_desc->iser_header.flags = ISER_VER; 1088 1089 tx_desc->num_sge = 1; 1090 tx_desc->isert_cmd = isert_cmd; 1091 1092 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 1093 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1094 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1095 } 1096 } 1097 1098 static int 1099 isert_init_tx_hdrs(struct isert_conn *isert_conn, 1100 struct iser_tx_desc *tx_desc) 1101 { 1102 struct isert_device *device = isert_conn->device; 1103 struct ib_device *ib_dev = device->ib_device; 1104 u64 dma_addr; 1105 1106 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1107 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1108 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1109 isert_err("ib_dma_mapping_error() failed\n"); 1110 return -ENOMEM; 1111 } 1112 1113 tx_desc->dma_addr = dma_addr; 1114 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1115 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1116 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1117 1118 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 1119 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 1120 tx_desc->tx_sg[0].lkey); 1121 1122 return 0; 1123 } 1124 1125 static void 1126 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1127 struct ib_send_wr *send_wr) 1128 { 1129 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 1130 1131 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1132 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1133 send_wr->opcode = IB_WR_SEND; 1134 send_wr->sg_list = &tx_desc->tx_sg[0]; 1135 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 1136 send_wr->send_flags = IB_SEND_SIGNALED; 1137 } 1138 1139 static int 1140 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1141 { 1142 struct ib_recv_wr rx_wr, *rx_wr_fail; 1143 struct ib_sge sge; 1144 int ret; 1145 1146 memset(&sge, 0, sizeof(struct ib_sge)); 1147 sge.addr = isert_conn->login_req_dma; 1148 sge.length = ISER_RX_LOGIN_SIZE; 1149 sge.lkey = isert_conn->device->pd->local_dma_lkey; 1150 1151 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1152 sge.addr, sge.length, sge.lkey); 1153 1154 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1155 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1156 rx_wr.sg_list = &sge; 1157 rx_wr.num_sge = 1; 1158 1159 isert_conn->post_recv_buf_count++; 1160 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 1161 if (ret) { 1162 isert_err("ib_post_recv() failed: %d\n", ret); 1163 isert_conn->post_recv_buf_count--; 1164 } 1165 1166 return ret; 1167 } 1168 1169 static int 1170 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1171 u32 length) 1172 { 1173 struct isert_conn *isert_conn = conn->context; 1174 struct isert_device *device = isert_conn->device; 1175 struct ib_device *ib_dev = device->ib_device; 1176 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1177 int ret; 1178 1179 isert_create_send_desc(isert_conn, NULL, tx_desc); 1180 1181 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1182 sizeof(struct iscsi_hdr)); 1183 1184 isert_init_tx_hdrs(isert_conn, tx_desc); 1185 1186 if (length > 0) { 1187 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1188 1189 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1190 length, DMA_TO_DEVICE); 1191 1192 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1193 1194 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1195 length, DMA_TO_DEVICE); 1196 1197 tx_dsg->addr = isert_conn->login_rsp_dma; 1198 tx_dsg->length = length; 1199 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1200 tx_desc->num_sge = 2; 1201 } 1202 if (!login->login_failed) { 1203 if (login->login_complete) { 1204 if (!conn->sess->sess_ops->SessionType && 1205 isert_conn->device->use_fastreg) { 1206 ret = isert_conn_create_fastreg_pool(isert_conn); 1207 if (ret) { 1208 isert_err("Conn: %p failed to create" 1209 " fastreg pool\n", isert_conn); 1210 return ret; 1211 } 1212 } 1213 1214 ret = isert_alloc_rx_descriptors(isert_conn); 1215 if (ret) 1216 return ret; 1217 1218 ret = isert_post_recvm(isert_conn, 1219 ISERT_QP_MAX_RECV_DTOS); 1220 if (ret) 1221 return ret; 1222 1223 /* Now we are in FULL_FEATURE phase */ 1224 mutex_lock(&isert_conn->mutex); 1225 isert_conn->state = ISER_CONN_FULL_FEATURE; 1226 mutex_unlock(&isert_conn->mutex); 1227 goto post_send; 1228 } 1229 1230 ret = isert_rdma_post_recvl(isert_conn); 1231 if (ret) 1232 return ret; 1233 } 1234 post_send: 1235 ret = isert_post_send(isert_conn, tx_desc); 1236 if (ret) 1237 return ret; 1238 1239 return 0; 1240 } 1241 1242 static void 1243 isert_rx_login_req(struct isert_conn *isert_conn) 1244 { 1245 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 1246 int rx_buflen = isert_conn->login_req_len; 1247 struct iscsi_conn *conn = isert_conn->conn; 1248 struct iscsi_login *login = conn->conn_login; 1249 int size; 1250 1251 isert_info("conn %p\n", isert_conn); 1252 1253 WARN_ON_ONCE(!login); 1254 1255 if (login->first_request) { 1256 struct iscsi_login_req *login_req = 1257 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1258 /* 1259 * Setup the initial iscsi_login values from the leading 1260 * login request PDU. 1261 */ 1262 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1263 login->current_stage = 1264 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1265 >> 2; 1266 login->version_min = login_req->min_version; 1267 login->version_max = login_req->max_version; 1268 memcpy(login->isid, login_req->isid, 6); 1269 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1270 login->init_task_tag = login_req->itt; 1271 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1272 login->cid = be16_to_cpu(login_req->cid); 1273 login->tsih = be16_to_cpu(login_req->tsih); 1274 } 1275 1276 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1277 1278 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1279 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1280 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1281 MAX_KEY_VALUE_PAIRS); 1282 memcpy(login->req_buf, &rx_desc->data[0], size); 1283 1284 if (login->first_request) { 1285 complete(&isert_conn->login_comp); 1286 return; 1287 } 1288 schedule_delayed_work(&conn->login_work, 0); 1289 } 1290 1291 static struct iscsi_cmd 1292 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1293 { 1294 struct isert_conn *isert_conn = conn->context; 1295 struct isert_cmd *isert_cmd; 1296 struct iscsi_cmd *cmd; 1297 1298 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1299 if (!cmd) { 1300 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1301 return NULL; 1302 } 1303 isert_cmd = iscsit_priv_cmd(cmd); 1304 isert_cmd->conn = isert_conn; 1305 isert_cmd->iscsi_cmd = cmd; 1306 isert_cmd->rx_desc = rx_desc; 1307 1308 return cmd; 1309 } 1310 1311 static int 1312 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1313 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1314 struct iser_rx_desc *rx_desc, unsigned char *buf) 1315 { 1316 struct iscsi_conn *conn = isert_conn->conn; 1317 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1318 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1319 bool dump_payload = false; 1320 unsigned int data_len; 1321 1322 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1323 if (rc < 0) 1324 return rc; 1325 1326 imm_data = cmd->immediate_data; 1327 imm_data_len = cmd->first_burst_len; 1328 unsol_data = cmd->unsolicited_data; 1329 data_len = cmd->se_cmd.data_length; 1330 1331 if (imm_data && imm_data_len == data_len) 1332 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1333 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1334 if (rc < 0) { 1335 return 0; 1336 } else if (rc > 0) { 1337 dump_payload = true; 1338 goto sequence_cmd; 1339 } 1340 1341 if (!imm_data) 1342 return 0; 1343 1344 if (imm_data_len != data_len) { 1345 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1346 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1347 &rx_desc->data[0], imm_data_len); 1348 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1349 sg_nents, imm_data_len); 1350 } else { 1351 sg_init_table(&isert_cmd->sg, 1); 1352 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1353 cmd->se_cmd.t_data_nents = 1; 1354 sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 1355 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1356 imm_data_len); 1357 } 1358 1359 cmd->write_data_done += imm_data_len; 1360 1361 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1362 spin_lock_bh(&cmd->istate_lock); 1363 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1364 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1365 spin_unlock_bh(&cmd->istate_lock); 1366 } 1367 1368 sequence_cmd: 1369 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1370 1371 if (!rc && dump_payload == false && unsol_data) 1372 iscsit_set_unsoliticed_dataout(cmd); 1373 else if (dump_payload && imm_data) 1374 target_put_sess_cmd(&cmd->se_cmd); 1375 1376 return 0; 1377 } 1378 1379 static int 1380 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1381 struct iser_rx_desc *rx_desc, unsigned char *buf) 1382 { 1383 struct scatterlist *sg_start; 1384 struct iscsi_conn *conn = isert_conn->conn; 1385 struct iscsi_cmd *cmd = NULL; 1386 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1387 u32 unsol_data_len = ntoh24(hdr->dlength); 1388 int rc, sg_nents, sg_off, page_off; 1389 1390 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1391 if (rc < 0) 1392 return rc; 1393 else if (!cmd) 1394 return 0; 1395 /* 1396 * FIXME: Unexpected unsolicited_data out 1397 */ 1398 if (!cmd->unsolicited_data) { 1399 isert_err("Received unexpected solicited data payload\n"); 1400 dump_stack(); 1401 return -1; 1402 } 1403 1404 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1405 "write_data_done: %u, data_length: %u\n", 1406 unsol_data_len, cmd->write_data_done, 1407 cmd->se_cmd.data_length); 1408 1409 sg_off = cmd->write_data_done / PAGE_SIZE; 1410 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1411 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1412 page_off = cmd->write_data_done % PAGE_SIZE; 1413 /* 1414 * FIXME: Non page-aligned unsolicited_data out 1415 */ 1416 if (page_off) { 1417 isert_err("unexpected non-page aligned data payload\n"); 1418 dump_stack(); 1419 return -1; 1420 } 1421 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1422 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1423 sg_nents, &rx_desc->data[0], unsol_data_len); 1424 1425 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1426 unsol_data_len); 1427 1428 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1429 if (rc < 0) 1430 return rc; 1431 1432 /* 1433 * multiple data-outs on the same command can arrive - 1434 * so post the buffer before hand 1435 */ 1436 rc = isert_post_recv(isert_conn, rx_desc); 1437 if (rc) { 1438 isert_err("ib_post_recv failed with %d\n", rc); 1439 return rc; 1440 } 1441 return 0; 1442 } 1443 1444 static int 1445 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1446 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1447 unsigned char *buf) 1448 { 1449 struct iscsi_conn *conn = isert_conn->conn; 1450 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1451 int rc; 1452 1453 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1454 if (rc < 0) 1455 return rc; 1456 /* 1457 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1458 */ 1459 1460 return iscsit_process_nop_out(conn, cmd, hdr); 1461 } 1462 1463 static int 1464 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1465 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1466 struct iscsi_text *hdr) 1467 { 1468 struct iscsi_conn *conn = isert_conn->conn; 1469 u32 payload_length = ntoh24(hdr->dlength); 1470 int rc; 1471 unsigned char *text_in = NULL; 1472 1473 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1474 if (rc < 0) 1475 return rc; 1476 1477 if (payload_length) { 1478 text_in = kzalloc(payload_length, GFP_KERNEL); 1479 if (!text_in) { 1480 isert_err("Unable to allocate text_in of payload_length: %u\n", 1481 payload_length); 1482 return -ENOMEM; 1483 } 1484 } 1485 cmd->text_in_ptr = text_in; 1486 1487 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1488 1489 return iscsit_process_text_cmd(conn, cmd, hdr); 1490 } 1491 1492 static int 1493 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1494 uint32_t read_stag, uint64_t read_va, 1495 uint32_t write_stag, uint64_t write_va) 1496 { 1497 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1498 struct iscsi_conn *conn = isert_conn->conn; 1499 struct iscsi_cmd *cmd; 1500 struct isert_cmd *isert_cmd; 1501 int ret = -EINVAL; 1502 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1503 1504 if (conn->sess->sess_ops->SessionType && 1505 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1506 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1507 " ignoring\n", opcode); 1508 return 0; 1509 } 1510 1511 switch (opcode) { 1512 case ISCSI_OP_SCSI_CMD: 1513 cmd = isert_allocate_cmd(conn, rx_desc); 1514 if (!cmd) 1515 break; 1516 1517 isert_cmd = iscsit_priv_cmd(cmd); 1518 isert_cmd->read_stag = read_stag; 1519 isert_cmd->read_va = read_va; 1520 isert_cmd->write_stag = write_stag; 1521 isert_cmd->write_va = write_va; 1522 1523 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1524 rx_desc, (unsigned char *)hdr); 1525 break; 1526 case ISCSI_OP_NOOP_OUT: 1527 cmd = isert_allocate_cmd(conn, rx_desc); 1528 if (!cmd) 1529 break; 1530 1531 isert_cmd = iscsit_priv_cmd(cmd); 1532 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1533 rx_desc, (unsigned char *)hdr); 1534 break; 1535 case ISCSI_OP_SCSI_DATA_OUT: 1536 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1537 (unsigned char *)hdr); 1538 break; 1539 case ISCSI_OP_SCSI_TMFUNC: 1540 cmd = isert_allocate_cmd(conn, rx_desc); 1541 if (!cmd) 1542 break; 1543 1544 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1545 (unsigned char *)hdr); 1546 break; 1547 case ISCSI_OP_LOGOUT: 1548 cmd = isert_allocate_cmd(conn, rx_desc); 1549 if (!cmd) 1550 break; 1551 1552 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1553 break; 1554 case ISCSI_OP_TEXT: 1555 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1556 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1557 else 1558 cmd = isert_allocate_cmd(conn, rx_desc); 1559 1560 if (!cmd) 1561 break; 1562 1563 isert_cmd = iscsit_priv_cmd(cmd); 1564 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1565 rx_desc, (struct iscsi_text *)hdr); 1566 break; 1567 default: 1568 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1569 dump_stack(); 1570 break; 1571 } 1572 1573 return ret; 1574 } 1575 1576 static void 1577 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1578 { 1579 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1580 uint64_t read_va = 0, write_va = 0; 1581 uint32_t read_stag = 0, write_stag = 0; 1582 int rc; 1583 1584 switch (iser_hdr->flags & 0xF0) { 1585 case ISCSI_CTRL: 1586 if (iser_hdr->flags & ISER_RSV) { 1587 read_stag = be32_to_cpu(iser_hdr->read_stag); 1588 read_va = be64_to_cpu(iser_hdr->read_va); 1589 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1590 read_stag, (unsigned long long)read_va); 1591 } 1592 if (iser_hdr->flags & ISER_WSV) { 1593 write_stag = be32_to_cpu(iser_hdr->write_stag); 1594 write_va = be64_to_cpu(iser_hdr->write_va); 1595 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1596 write_stag, (unsigned long long)write_va); 1597 } 1598 1599 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1600 break; 1601 case ISER_HELLO: 1602 isert_err("iSER Hello message\n"); 1603 break; 1604 default: 1605 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1606 break; 1607 } 1608 1609 rc = isert_rx_opcode(isert_conn, rx_desc, 1610 read_stag, read_va, write_stag, write_va); 1611 } 1612 1613 static void 1614 isert_rcv_completion(struct iser_rx_desc *desc, 1615 struct isert_conn *isert_conn, 1616 u32 xfer_len) 1617 { 1618 struct ib_device *ib_dev = isert_conn->cm_id->device; 1619 struct iscsi_hdr *hdr; 1620 u64 rx_dma; 1621 int rx_buflen; 1622 1623 if ((char *)desc == isert_conn->login_req_buf) { 1624 rx_dma = isert_conn->login_req_dma; 1625 rx_buflen = ISER_RX_LOGIN_SIZE; 1626 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1627 rx_dma, rx_buflen); 1628 } else { 1629 rx_dma = desc->dma_addr; 1630 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1631 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1632 rx_dma, rx_buflen); 1633 } 1634 1635 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1636 1637 hdr = &desc->iscsi_header; 1638 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1639 hdr->opcode, hdr->itt, hdr->flags, 1640 (int)(xfer_len - ISER_HEADERS_LEN)); 1641 1642 if ((char *)desc == isert_conn->login_req_buf) { 1643 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 1644 if (isert_conn->conn) { 1645 struct iscsi_login *login = isert_conn->conn->conn_login; 1646 1647 if (login && !login->first_request) 1648 isert_rx_login_req(isert_conn); 1649 } 1650 mutex_lock(&isert_conn->mutex); 1651 complete(&isert_conn->login_req_comp); 1652 mutex_unlock(&isert_conn->mutex); 1653 } else { 1654 isert_rx_do_work(desc, isert_conn); 1655 } 1656 1657 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1658 DMA_FROM_DEVICE); 1659 1660 isert_conn->post_recv_buf_count--; 1661 } 1662 1663 static int 1664 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1665 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1666 enum iser_ib_op_code op, struct isert_data_buf *data) 1667 { 1668 struct ib_device *ib_dev = isert_conn->cm_id->device; 1669 1670 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1671 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1672 1673 data->len = length - offset; 1674 data->offset = offset; 1675 data->sg_off = data->offset / PAGE_SIZE; 1676 1677 data->sg = &sg[data->sg_off]; 1678 data->nents = min_t(unsigned int, nents - data->sg_off, 1679 ISCSI_ISER_SG_TABLESIZE); 1680 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1681 PAGE_SIZE); 1682 1683 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1684 data->dma_dir); 1685 if (unlikely(!data->dma_nents)) { 1686 isert_err("Cmd: unable to dma map SGs %p\n", sg); 1687 return -EINVAL; 1688 } 1689 1690 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1691 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1692 1693 return 0; 1694 } 1695 1696 static void 1697 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1698 { 1699 struct ib_device *ib_dev = isert_conn->cm_id->device; 1700 1701 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1702 memset(data, 0, sizeof(*data)); 1703 } 1704 1705 1706 1707 static void 1708 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1709 { 1710 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1711 1712 isert_dbg("Cmd %p\n", isert_cmd); 1713 1714 if (wr->data.sg) { 1715 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1716 isert_unmap_data_buf(isert_conn, &wr->data); 1717 } 1718 1719 if (wr->send_wr) { 1720 isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1721 kfree(wr->send_wr); 1722 wr->send_wr = NULL; 1723 } 1724 1725 if (wr->ib_sge) { 1726 isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1727 kfree(wr->ib_sge); 1728 wr->ib_sge = NULL; 1729 } 1730 } 1731 1732 static void 1733 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1734 { 1735 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1736 1737 isert_dbg("Cmd %p\n", isert_cmd); 1738 1739 if (wr->fr_desc) { 1740 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 1741 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1742 isert_unmap_data_buf(isert_conn, &wr->prot); 1743 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1744 } 1745 spin_lock_bh(&isert_conn->pool_lock); 1746 list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); 1747 spin_unlock_bh(&isert_conn->pool_lock); 1748 wr->fr_desc = NULL; 1749 } 1750 1751 if (wr->data.sg) { 1752 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1753 isert_unmap_data_buf(isert_conn, &wr->data); 1754 } 1755 1756 wr->ib_sge = NULL; 1757 wr->send_wr = NULL; 1758 } 1759 1760 static void 1761 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1762 { 1763 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1764 struct isert_conn *isert_conn = isert_cmd->conn; 1765 struct iscsi_conn *conn = isert_conn->conn; 1766 struct isert_device *device = isert_conn->device; 1767 struct iscsi_text_rsp *hdr; 1768 1769 isert_dbg("Cmd %p\n", isert_cmd); 1770 1771 switch (cmd->iscsi_opcode) { 1772 case ISCSI_OP_SCSI_CMD: 1773 spin_lock_bh(&conn->cmd_lock); 1774 if (!list_empty(&cmd->i_conn_node)) 1775 list_del_init(&cmd->i_conn_node); 1776 spin_unlock_bh(&conn->cmd_lock); 1777 1778 if (cmd->data_direction == DMA_TO_DEVICE) { 1779 iscsit_stop_dataout_timer(cmd); 1780 /* 1781 * Check for special case during comp_err where 1782 * WRITE_PENDING has been handed off from core, 1783 * but requires an extra target_put_sess_cmd() 1784 * before transport_generic_free_cmd() below. 1785 */ 1786 if (comp_err && 1787 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1788 struct se_cmd *se_cmd = &cmd->se_cmd; 1789 1790 target_put_sess_cmd(se_cmd); 1791 } 1792 } 1793 1794 device->unreg_rdma_mem(isert_cmd, isert_conn); 1795 transport_generic_free_cmd(&cmd->se_cmd, 0); 1796 break; 1797 case ISCSI_OP_SCSI_TMFUNC: 1798 spin_lock_bh(&conn->cmd_lock); 1799 if (!list_empty(&cmd->i_conn_node)) 1800 list_del_init(&cmd->i_conn_node); 1801 spin_unlock_bh(&conn->cmd_lock); 1802 1803 transport_generic_free_cmd(&cmd->se_cmd, 0); 1804 break; 1805 case ISCSI_OP_REJECT: 1806 case ISCSI_OP_NOOP_OUT: 1807 case ISCSI_OP_TEXT: 1808 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1809 /* If the continue bit is on, keep the command alive */ 1810 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1811 break; 1812 1813 spin_lock_bh(&conn->cmd_lock); 1814 if (!list_empty(&cmd->i_conn_node)) 1815 list_del_init(&cmd->i_conn_node); 1816 spin_unlock_bh(&conn->cmd_lock); 1817 1818 /* 1819 * Handle special case for REJECT when iscsi_add_reject*() has 1820 * overwritten the original iscsi_opcode assignment, and the 1821 * associated cmd->se_cmd needs to be released. 1822 */ 1823 if (cmd->se_cmd.se_tfo != NULL) { 1824 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1825 cmd->iscsi_opcode); 1826 transport_generic_free_cmd(&cmd->se_cmd, 0); 1827 break; 1828 } 1829 /* 1830 * Fall-through 1831 */ 1832 default: 1833 iscsit_release_cmd(cmd); 1834 break; 1835 } 1836 } 1837 1838 static void 1839 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1840 { 1841 if (tx_desc->dma_addr != 0) { 1842 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1843 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1844 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1845 tx_desc->dma_addr = 0; 1846 } 1847 } 1848 1849 static void 1850 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1851 struct ib_device *ib_dev, bool comp_err) 1852 { 1853 if (isert_cmd->pdu_buf_dma != 0) { 1854 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1855 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1856 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1857 isert_cmd->pdu_buf_dma = 0; 1858 } 1859 1860 isert_unmap_tx_desc(tx_desc, ib_dev); 1861 isert_put_cmd(isert_cmd, comp_err); 1862 } 1863 1864 static int 1865 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1866 { 1867 struct ib_mr_status mr_status; 1868 int ret; 1869 1870 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1871 if (ret) { 1872 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1873 goto fail_mr_status; 1874 } 1875 1876 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1877 u64 sec_offset_err; 1878 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1879 1880 switch (mr_status.sig_err.err_type) { 1881 case IB_SIG_BAD_GUARD: 1882 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1883 break; 1884 case IB_SIG_BAD_REFTAG: 1885 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1886 break; 1887 case IB_SIG_BAD_APPTAG: 1888 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1889 break; 1890 } 1891 sec_offset_err = mr_status.sig_err.sig_err_offset; 1892 do_div(sec_offset_err, block_size); 1893 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1894 1895 isert_err("PI error found type %d at sector 0x%llx " 1896 "expected 0x%x vs actual 0x%x\n", 1897 mr_status.sig_err.err_type, 1898 (unsigned long long)se_cmd->bad_sector, 1899 mr_status.sig_err.expected, 1900 mr_status.sig_err.actual); 1901 ret = 1; 1902 } 1903 1904 fail_mr_status: 1905 return ret; 1906 } 1907 1908 static void 1909 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1910 struct isert_cmd *isert_cmd) 1911 { 1912 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1913 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1914 struct se_cmd *se_cmd = &cmd->se_cmd; 1915 struct isert_conn *isert_conn = isert_cmd->conn; 1916 struct isert_device *device = isert_conn->device; 1917 int ret = 0; 1918 1919 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1920 ret = isert_check_pi_status(se_cmd, 1921 wr->fr_desc->pi_ctx->sig_mr); 1922 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1923 } 1924 1925 device->unreg_rdma_mem(isert_cmd, isert_conn); 1926 wr->send_wr_num = 0; 1927 if (ret) 1928 transport_send_check_condition_and_sense(se_cmd, 1929 se_cmd->pi_err, 0); 1930 else 1931 isert_put_response(isert_conn->conn, cmd); 1932 } 1933 1934 static void 1935 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1936 struct isert_cmd *isert_cmd) 1937 { 1938 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1939 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1940 struct se_cmd *se_cmd = &cmd->se_cmd; 1941 struct isert_conn *isert_conn = isert_cmd->conn; 1942 struct isert_device *device = isert_conn->device; 1943 int ret = 0; 1944 1945 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1946 ret = isert_check_pi_status(se_cmd, 1947 wr->fr_desc->pi_ctx->sig_mr); 1948 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1949 } 1950 1951 iscsit_stop_dataout_timer(cmd); 1952 device->unreg_rdma_mem(isert_cmd, isert_conn); 1953 cmd->write_data_done = wr->data.len; 1954 wr->send_wr_num = 0; 1955 1956 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1957 spin_lock_bh(&cmd->istate_lock); 1958 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1959 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1960 spin_unlock_bh(&cmd->istate_lock); 1961 1962 if (ret) { 1963 target_put_sess_cmd(se_cmd); 1964 transport_send_check_condition_and_sense(se_cmd, 1965 se_cmd->pi_err, 0); 1966 } else { 1967 target_execute_cmd(se_cmd); 1968 } 1969 } 1970 1971 static void 1972 isert_do_control_comp(struct work_struct *work) 1973 { 1974 struct isert_cmd *isert_cmd = container_of(work, 1975 struct isert_cmd, comp_work); 1976 struct isert_conn *isert_conn = isert_cmd->conn; 1977 struct ib_device *ib_dev = isert_conn->cm_id->device; 1978 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1979 1980 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1981 1982 switch (cmd->i_state) { 1983 case ISTATE_SEND_TASKMGTRSP: 1984 iscsit_tmr_post_handler(cmd, cmd->conn); 1985 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1986 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1987 cmd->i_state = ISTATE_SENT_STATUS; 1988 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1989 ib_dev, false); 1990 break; 1991 case ISTATE_SEND_LOGOUTRSP: 1992 iscsit_logout_post_handler(cmd, cmd->conn); 1993 break; 1994 default: 1995 isert_err("Unknown i_state %d\n", cmd->i_state); 1996 dump_stack(); 1997 break; 1998 } 1999 } 2000 2001 static void 2002 isert_response_completion(struct iser_tx_desc *tx_desc, 2003 struct isert_cmd *isert_cmd, 2004 struct isert_conn *isert_conn, 2005 struct ib_device *ib_dev) 2006 { 2007 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2008 2009 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 2010 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 2011 cmd->i_state == ISTATE_SEND_REJECT || 2012 cmd->i_state == ISTATE_SEND_TEXTRSP) { 2013 isert_unmap_tx_desc(tx_desc, ib_dev); 2014 2015 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 2016 queue_work(isert_comp_wq, &isert_cmd->comp_work); 2017 return; 2018 } 2019 2020 cmd->i_state = ISTATE_SENT_STATUS; 2021 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 2022 } 2023 2024 static void 2025 isert_snd_completion(struct iser_tx_desc *tx_desc, 2026 struct isert_conn *isert_conn) 2027 { 2028 struct ib_device *ib_dev = isert_conn->cm_id->device; 2029 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2030 struct isert_rdma_wr *wr; 2031 2032 if (!isert_cmd) { 2033 isert_unmap_tx_desc(tx_desc, ib_dev); 2034 return; 2035 } 2036 wr = &isert_cmd->rdma_wr; 2037 2038 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 2039 2040 switch (wr->iser_ib_op) { 2041 case ISER_IB_SEND: 2042 isert_response_completion(tx_desc, isert_cmd, 2043 isert_conn, ib_dev); 2044 break; 2045 case ISER_IB_RDMA_WRITE: 2046 isert_completion_rdma_write(tx_desc, isert_cmd); 2047 break; 2048 case ISER_IB_RDMA_READ: 2049 isert_completion_rdma_read(tx_desc, isert_cmd); 2050 break; 2051 default: 2052 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 2053 dump_stack(); 2054 break; 2055 } 2056 } 2057 2058 /** 2059 * is_isert_tx_desc() - Indicate if the completion wr_id 2060 * is a TX descriptor or not. 2061 * @isert_conn: iser connection 2062 * @wr_id: completion WR identifier 2063 * 2064 * Since we cannot rely on wc opcode in FLUSH errors 2065 * we must work around it by checking if the wr_id address 2066 * falls in the iser connection rx_descs buffer. If so 2067 * it is an RX descriptor, otherwize it is a TX. 2068 */ 2069 static inline bool 2070 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 2071 { 2072 void *start = isert_conn->rx_descs; 2073 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); 2074 2075 if (wr_id >= start && wr_id < start + len) 2076 return false; 2077 2078 return true; 2079 } 2080 2081 static void 2082 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 2083 { 2084 if (wc->wr_id == ISER_BEACON_WRID) { 2085 isert_info("conn %p completing wait_comp_err\n", 2086 isert_conn); 2087 complete(&isert_conn->wait_comp_err); 2088 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 2089 struct ib_device *ib_dev = isert_conn->cm_id->device; 2090 struct isert_cmd *isert_cmd; 2091 struct iser_tx_desc *desc; 2092 2093 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2094 isert_cmd = desc->isert_cmd; 2095 if (!isert_cmd) 2096 isert_unmap_tx_desc(desc, ib_dev); 2097 else 2098 isert_completion_put(desc, isert_cmd, ib_dev, true); 2099 } else { 2100 isert_conn->post_recv_buf_count--; 2101 if (!isert_conn->post_recv_buf_count) 2102 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2103 } 2104 } 2105 2106 static void 2107 isert_handle_wc(struct ib_wc *wc) 2108 { 2109 struct isert_conn *isert_conn; 2110 struct iser_tx_desc *tx_desc; 2111 struct iser_rx_desc *rx_desc; 2112 2113 isert_conn = wc->qp->qp_context; 2114 if (likely(wc->status == IB_WC_SUCCESS)) { 2115 if (wc->opcode == IB_WC_RECV) { 2116 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 2117 isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); 2118 } else { 2119 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2120 isert_snd_completion(tx_desc, isert_conn); 2121 } 2122 } else { 2123 if (wc->status != IB_WC_WR_FLUSH_ERR) 2124 isert_err("%s (%d): wr id %llx vend_err %x\n", 2125 ib_wc_status_msg(wc->status), wc->status, 2126 wc->wr_id, wc->vendor_err); 2127 else 2128 isert_dbg("%s (%d): wr id %llx\n", 2129 ib_wc_status_msg(wc->status), wc->status, 2130 wc->wr_id); 2131 2132 if (wc->wr_id != ISER_FASTREG_LI_WRID) 2133 isert_cq_comp_err(isert_conn, wc); 2134 } 2135 } 2136 2137 static void 2138 isert_cq_work(struct work_struct *work) 2139 { 2140 enum { isert_poll_budget = 65536 }; 2141 struct isert_comp *comp = container_of(work, struct isert_comp, 2142 work); 2143 struct ib_wc *const wcs = comp->wcs; 2144 int i, n, completed = 0; 2145 2146 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 2147 for (i = 0; i < n; i++) 2148 isert_handle_wc(&wcs[i]); 2149 2150 completed += n; 2151 if (completed >= isert_poll_budget) 2152 break; 2153 } 2154 2155 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2156 } 2157 2158 static void 2159 isert_cq_callback(struct ib_cq *cq, void *context) 2160 { 2161 struct isert_comp *comp = context; 2162 2163 queue_work(isert_comp_wq, &comp->work); 2164 } 2165 2166 static int 2167 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2168 { 2169 struct ib_send_wr *wr_failed; 2170 int ret; 2171 2172 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2173 if (ret) { 2174 isert_err("ib_post_recv failed with %d\n", ret); 2175 return ret; 2176 } 2177 2178 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2179 &wr_failed); 2180 if (ret) { 2181 isert_err("ib_post_send failed with %d\n", ret); 2182 return ret; 2183 } 2184 return ret; 2185 } 2186 2187 static int 2188 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2189 { 2190 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2191 struct isert_conn *isert_conn = conn->context; 2192 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2193 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2194 &isert_cmd->tx_desc.iscsi_header; 2195 2196 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2197 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2198 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2199 /* 2200 * Attach SENSE DATA payload to iSCSI Response PDU 2201 */ 2202 if (cmd->se_cmd.sense_buffer && 2203 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2204 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2205 struct isert_device *device = isert_conn->device; 2206 struct ib_device *ib_dev = device->ib_device; 2207 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2208 u32 padding, pdu_len; 2209 2210 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2211 cmd->sense_buffer); 2212 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2213 2214 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2215 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2216 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2217 2218 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2219 (void *)cmd->sense_buffer, pdu_len, 2220 DMA_TO_DEVICE); 2221 2222 isert_cmd->pdu_buf_len = pdu_len; 2223 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2224 tx_dsg->length = pdu_len; 2225 tx_dsg->lkey = device->pd->local_dma_lkey; 2226 isert_cmd->tx_desc.num_sge = 2; 2227 } 2228 2229 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2230 2231 isert_dbg("Posting SCSI Response\n"); 2232 2233 return isert_post_response(isert_conn, isert_cmd); 2234 } 2235 2236 static void 2237 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2238 { 2239 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2240 struct isert_conn *isert_conn = conn->context; 2241 struct isert_device *device = isert_conn->device; 2242 2243 spin_lock_bh(&conn->cmd_lock); 2244 if (!list_empty(&cmd->i_conn_node)) 2245 list_del_init(&cmd->i_conn_node); 2246 spin_unlock_bh(&conn->cmd_lock); 2247 2248 if (cmd->data_direction == DMA_TO_DEVICE) 2249 iscsit_stop_dataout_timer(cmd); 2250 2251 device->unreg_rdma_mem(isert_cmd, isert_conn); 2252 } 2253 2254 static enum target_prot_op 2255 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2256 { 2257 struct isert_conn *isert_conn = conn->context; 2258 struct isert_device *device = isert_conn->device; 2259 2260 if (conn->tpg->tpg_attrib.t10_pi) { 2261 if (device->pi_capable) { 2262 isert_info("conn %p PI offload enabled\n", isert_conn); 2263 isert_conn->pi_support = true; 2264 return TARGET_PROT_ALL; 2265 } 2266 } 2267 2268 isert_info("conn %p PI offload disabled\n", isert_conn); 2269 isert_conn->pi_support = false; 2270 2271 return TARGET_PROT_NORMAL; 2272 } 2273 2274 static int 2275 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2276 bool nopout_response) 2277 { 2278 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2279 struct isert_conn *isert_conn = conn->context; 2280 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2281 2282 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2283 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2284 &isert_cmd->tx_desc.iscsi_header, 2285 nopout_response); 2286 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2287 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2288 2289 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2290 2291 return isert_post_response(isert_conn, isert_cmd); 2292 } 2293 2294 static int 2295 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2296 { 2297 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2298 struct isert_conn *isert_conn = conn->context; 2299 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2300 2301 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2302 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2303 &isert_cmd->tx_desc.iscsi_header); 2304 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2305 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2306 2307 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2308 2309 return isert_post_response(isert_conn, isert_cmd); 2310 } 2311 2312 static int 2313 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2314 { 2315 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2316 struct isert_conn *isert_conn = conn->context; 2317 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2318 2319 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2320 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2321 &isert_cmd->tx_desc.iscsi_header); 2322 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2323 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2324 2325 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2326 2327 return isert_post_response(isert_conn, isert_cmd); 2328 } 2329 2330 static int 2331 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2332 { 2333 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2334 struct isert_conn *isert_conn = conn->context; 2335 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2336 struct isert_device *device = isert_conn->device; 2337 struct ib_device *ib_dev = device->ib_device; 2338 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2339 struct iscsi_reject *hdr = 2340 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2341 2342 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2343 iscsit_build_reject(cmd, conn, hdr); 2344 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2345 2346 hton24(hdr->dlength, ISCSI_HDR_LEN); 2347 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2348 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2349 DMA_TO_DEVICE); 2350 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2351 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2352 tx_dsg->length = ISCSI_HDR_LEN; 2353 tx_dsg->lkey = device->pd->local_dma_lkey; 2354 isert_cmd->tx_desc.num_sge = 2; 2355 2356 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2357 2358 isert_dbg("conn %p Posting Reject\n", isert_conn); 2359 2360 return isert_post_response(isert_conn, isert_cmd); 2361 } 2362 2363 static int 2364 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2365 { 2366 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2367 struct isert_conn *isert_conn = conn->context; 2368 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2369 struct iscsi_text_rsp *hdr = 2370 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2371 u32 txt_rsp_len; 2372 int rc; 2373 2374 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2375 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2376 if (rc < 0) 2377 return rc; 2378 2379 txt_rsp_len = rc; 2380 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2381 2382 if (txt_rsp_len) { 2383 struct isert_device *device = isert_conn->device; 2384 struct ib_device *ib_dev = device->ib_device; 2385 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2386 void *txt_rsp_buf = cmd->buf_ptr; 2387 2388 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2389 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2390 2391 isert_cmd->pdu_buf_len = txt_rsp_len; 2392 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2393 tx_dsg->length = txt_rsp_len; 2394 tx_dsg->lkey = device->pd->local_dma_lkey; 2395 isert_cmd->tx_desc.num_sge = 2; 2396 } 2397 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2398 2399 isert_dbg("conn %p Text Response\n", isert_conn); 2400 2401 return isert_post_response(isert_conn, isert_cmd); 2402 } 2403 2404 static int 2405 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2406 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2407 u32 data_left, u32 offset) 2408 { 2409 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2410 struct scatterlist *sg_start, *tmp_sg; 2411 struct isert_device *device = isert_conn->device; 2412 struct ib_device *ib_dev = device->ib_device; 2413 u32 sg_off, page_off; 2414 int i = 0, sg_nents; 2415 2416 sg_off = offset / PAGE_SIZE; 2417 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2418 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2419 page_off = offset % PAGE_SIZE; 2420 2421 send_wr->sg_list = ib_sge; 2422 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2423 /* 2424 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2425 */ 2426 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2427 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 2428 "page_off: %u\n", 2429 (unsigned long long)tmp_sg->dma_address, 2430 tmp_sg->length, page_off); 2431 2432 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2433 ib_sge->length = min_t(u32, data_left, 2434 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2435 ib_sge->lkey = device->pd->local_dma_lkey; 2436 2437 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 2438 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2439 page_off = 0; 2440 data_left -= ib_sge->length; 2441 if (!data_left) 2442 break; 2443 ib_sge++; 2444 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2445 } 2446 2447 send_wr->num_sge = ++i; 2448 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2449 send_wr->sg_list, send_wr->num_sge); 2450 2451 return send_wr->num_sge; 2452 } 2453 2454 static int 2455 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2456 struct isert_rdma_wr *wr) 2457 { 2458 struct se_cmd *se_cmd = &cmd->se_cmd; 2459 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2460 struct isert_conn *isert_conn = conn->context; 2461 struct isert_data_buf *data = &wr->data; 2462 struct ib_send_wr *send_wr; 2463 struct ib_sge *ib_sge; 2464 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2465 int ret = 0, i, ib_sge_cnt; 2466 2467 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2468 2469 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2470 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2471 se_cmd->t_data_nents, se_cmd->data_length, 2472 offset, wr->iser_ib_op, &wr->data); 2473 if (ret) 2474 return ret; 2475 2476 data_left = data->len; 2477 offset = data->offset; 2478 2479 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2480 if (!ib_sge) { 2481 isert_warn("Unable to allocate ib_sge\n"); 2482 ret = -ENOMEM; 2483 goto unmap_cmd; 2484 } 2485 wr->ib_sge = ib_sge; 2486 2487 wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2488 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2489 GFP_KERNEL); 2490 if (!wr->send_wr) { 2491 isert_dbg("Unable to allocate wr->send_wr\n"); 2492 ret = -ENOMEM; 2493 goto unmap_cmd; 2494 } 2495 2496 wr->isert_cmd = isert_cmd; 2497 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2498 2499 for (i = 0; i < wr->send_wr_num; i++) { 2500 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2501 data_len = min(data_left, rdma_write_max); 2502 2503 send_wr->send_flags = 0; 2504 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2505 send_wr->opcode = IB_WR_RDMA_WRITE; 2506 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2507 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2508 if (i + 1 == wr->send_wr_num) 2509 send_wr->next = &isert_cmd->tx_desc.send_wr; 2510 else 2511 send_wr->next = &wr->send_wr[i + 1]; 2512 } else { 2513 send_wr->opcode = IB_WR_RDMA_READ; 2514 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2515 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2516 if (i + 1 == wr->send_wr_num) 2517 send_wr->send_flags = IB_SEND_SIGNALED; 2518 else 2519 send_wr->next = &wr->send_wr[i + 1]; 2520 } 2521 2522 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2523 send_wr, data_len, offset); 2524 ib_sge += ib_sge_cnt; 2525 2526 offset += data_len; 2527 va_offset += data_len; 2528 data_left -= data_len; 2529 } 2530 2531 return 0; 2532 unmap_cmd: 2533 isert_unmap_data_buf(isert_conn, data); 2534 2535 return ret; 2536 } 2537 2538 static int 2539 isert_map_fr_pagelist(struct ib_device *ib_dev, 2540 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2541 { 2542 u64 start_addr, end_addr, page, chunk_start = 0; 2543 struct scatterlist *tmp_sg; 2544 int i = 0, new_chunk, last_ent, n_pages; 2545 2546 n_pages = 0; 2547 new_chunk = 1; 2548 last_ent = sg_nents - 1; 2549 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2550 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2551 if (new_chunk) 2552 chunk_start = start_addr; 2553 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2554 2555 isert_dbg("SGL[%d] dma_addr: 0x%llx len: %u\n", 2556 i, (unsigned long long)tmp_sg->dma_address, 2557 tmp_sg->length); 2558 2559 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2560 new_chunk = 0; 2561 continue; 2562 } 2563 new_chunk = 1; 2564 2565 page = chunk_start & PAGE_MASK; 2566 do { 2567 fr_pl[n_pages++] = page; 2568 isert_dbg("Mapped page_list[%d] page_addr: 0x%llx\n", 2569 n_pages - 1, page); 2570 page += PAGE_SIZE; 2571 } while (page < end_addr); 2572 } 2573 2574 return n_pages; 2575 } 2576 2577 static inline void 2578 isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 2579 { 2580 u32 rkey; 2581 2582 memset(inv_wr, 0, sizeof(*inv_wr)); 2583 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 2584 inv_wr->opcode = IB_WR_LOCAL_INV; 2585 inv_wr->ex.invalidate_rkey = mr->rkey; 2586 2587 /* Bump the key */ 2588 rkey = ib_inc_rkey(mr->rkey); 2589 ib_update_fast_reg_key(mr, rkey); 2590 } 2591 2592 static int 2593 isert_fast_reg_mr(struct isert_conn *isert_conn, 2594 struct fast_reg_descriptor *fr_desc, 2595 struct isert_data_buf *mem, 2596 enum isert_indicator ind, 2597 struct ib_sge *sge) 2598 { 2599 struct isert_device *device = isert_conn->device; 2600 struct ib_device *ib_dev = device->ib_device; 2601 struct ib_mr *mr; 2602 struct ib_fast_reg_page_list *frpl; 2603 struct ib_send_wr fr_wr, inv_wr; 2604 struct ib_send_wr *bad_wr, *wr = NULL; 2605 int ret, pagelist_len; 2606 u32 page_off; 2607 2608 if (mem->dma_nents == 1) { 2609 sge->lkey = device->pd->local_dma_lkey; 2610 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2611 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2612 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2613 sge->addr, sge->length, sge->lkey); 2614 return 0; 2615 } 2616 2617 if (ind == ISERT_DATA_KEY_VALID) { 2618 /* Registering data buffer */ 2619 mr = fr_desc->data_mr; 2620 frpl = fr_desc->data_frpl; 2621 } else { 2622 /* Registering protection buffer */ 2623 mr = fr_desc->pi_ctx->prot_mr; 2624 frpl = fr_desc->pi_ctx->prot_frpl; 2625 } 2626 2627 page_off = mem->offset % PAGE_SIZE; 2628 2629 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2630 fr_desc, mem->nents, mem->offset); 2631 2632 pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 2633 &frpl->page_list[0]); 2634 2635 if (!(fr_desc->ind & ind)) { 2636 isert_inv_rkey(&inv_wr, mr); 2637 wr = &inv_wr; 2638 } 2639 2640 /* Prepare FASTREG WR */ 2641 memset(&fr_wr, 0, sizeof(fr_wr)); 2642 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2643 fr_wr.opcode = IB_WR_FAST_REG_MR; 2644 fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 2645 fr_wr.wr.fast_reg.page_list = frpl; 2646 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2647 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2648 fr_wr.wr.fast_reg.length = mem->len; 2649 fr_wr.wr.fast_reg.rkey = mr->rkey; 2650 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2651 2652 if (!wr) 2653 wr = &fr_wr; 2654 else 2655 wr->next = &fr_wr; 2656 2657 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2658 if (ret) { 2659 isert_err("fast registration failed, ret:%d\n", ret); 2660 return ret; 2661 } 2662 fr_desc->ind &= ~ind; 2663 2664 sge->lkey = mr->lkey; 2665 sge->addr = frpl->page_list[0] + page_off; 2666 sge->length = mem->len; 2667 2668 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2669 sge->addr, sge->length, sge->lkey); 2670 2671 return ret; 2672 } 2673 2674 static inline void 2675 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2676 struct ib_sig_domain *domain) 2677 { 2678 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2679 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2680 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2681 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2682 /* 2683 * At the moment we hard code those, but if in the future 2684 * the target core would like to use it, we will take it 2685 * from se_cmd. 2686 */ 2687 domain->sig.dif.apptag_check_mask = 0xffff; 2688 domain->sig.dif.app_escape = true; 2689 domain->sig.dif.ref_escape = true; 2690 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2691 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2692 domain->sig.dif.ref_remap = true; 2693 }; 2694 2695 static int 2696 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2697 { 2698 switch (se_cmd->prot_op) { 2699 case TARGET_PROT_DIN_INSERT: 2700 case TARGET_PROT_DOUT_STRIP: 2701 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2702 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2703 break; 2704 case TARGET_PROT_DOUT_INSERT: 2705 case TARGET_PROT_DIN_STRIP: 2706 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2707 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2708 break; 2709 case TARGET_PROT_DIN_PASS: 2710 case TARGET_PROT_DOUT_PASS: 2711 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2712 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2713 break; 2714 default: 2715 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2716 return -EINVAL; 2717 } 2718 2719 return 0; 2720 } 2721 2722 static inline u8 2723 isert_set_prot_checks(u8 prot_checks) 2724 { 2725 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2726 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2727 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2728 } 2729 2730 static int 2731 isert_reg_sig_mr(struct isert_conn *isert_conn, 2732 struct se_cmd *se_cmd, 2733 struct isert_rdma_wr *rdma_wr, 2734 struct fast_reg_descriptor *fr_desc) 2735 { 2736 struct ib_send_wr sig_wr, inv_wr; 2737 struct ib_send_wr *bad_wr, *wr = NULL; 2738 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2739 struct ib_sig_attrs sig_attrs; 2740 int ret; 2741 2742 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2743 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2744 if (ret) 2745 goto err; 2746 2747 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2748 2749 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2750 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 2751 wr = &inv_wr; 2752 } 2753 2754 memset(&sig_wr, 0, sizeof(sig_wr)); 2755 sig_wr.opcode = IB_WR_REG_SIG_MR; 2756 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2757 sig_wr.sg_list = &rdma_wr->ib_sg[DATA]; 2758 sig_wr.num_sge = 1; 2759 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 2760 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 2761 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 2762 if (se_cmd->t_prot_sg) 2763 sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT]; 2764 2765 if (!wr) 2766 wr = &sig_wr; 2767 else 2768 wr->next = &sig_wr; 2769 2770 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2771 if (ret) { 2772 isert_err("fast registration failed, ret:%d\n", ret); 2773 goto err; 2774 } 2775 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2776 2777 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2778 rdma_wr->ib_sg[SIG].addr = 0; 2779 rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 2780 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2781 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2782 /* 2783 * We have protection guards on the wire 2784 * so we need to set a larget transfer 2785 */ 2786 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 2787 2788 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2789 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2790 rdma_wr->ib_sg[SIG].lkey); 2791 err: 2792 return ret; 2793 } 2794 2795 static int 2796 isert_handle_prot_cmd(struct isert_conn *isert_conn, 2797 struct isert_cmd *isert_cmd, 2798 struct isert_rdma_wr *wr) 2799 { 2800 struct isert_device *device = isert_conn->device; 2801 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2802 int ret; 2803 2804 if (!wr->fr_desc->pi_ctx) { 2805 ret = isert_create_pi_ctx(wr->fr_desc, 2806 device->ib_device, 2807 device->pd); 2808 if (ret) { 2809 isert_err("conn %p failed to allocate pi_ctx\n", 2810 isert_conn); 2811 return ret; 2812 } 2813 } 2814 2815 if (se_cmd->t_prot_sg) { 2816 ret = isert_map_data_buf(isert_conn, isert_cmd, 2817 se_cmd->t_prot_sg, 2818 se_cmd->t_prot_nents, 2819 se_cmd->prot_length, 2820 0, wr->iser_ib_op, &wr->prot); 2821 if (ret) { 2822 isert_err("conn %p failed to map protection buffer\n", 2823 isert_conn); 2824 return ret; 2825 } 2826 2827 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2828 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2829 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2830 if (ret) { 2831 isert_err("conn %p failed to fast reg mr\n", 2832 isert_conn); 2833 goto unmap_prot_cmd; 2834 } 2835 } 2836 2837 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2838 if (ret) { 2839 isert_err("conn %p failed to fast reg mr\n", 2840 isert_conn); 2841 goto unmap_prot_cmd; 2842 } 2843 wr->fr_desc->ind |= ISERT_PROTECTED; 2844 2845 return 0; 2846 2847 unmap_prot_cmd: 2848 if (se_cmd->t_prot_sg) 2849 isert_unmap_data_buf(isert_conn, &wr->prot); 2850 2851 return ret; 2852 } 2853 2854 static int 2855 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2856 struct isert_rdma_wr *wr) 2857 { 2858 struct se_cmd *se_cmd = &cmd->se_cmd; 2859 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2860 struct isert_conn *isert_conn = conn->context; 2861 struct fast_reg_descriptor *fr_desc = NULL; 2862 struct ib_send_wr *send_wr; 2863 struct ib_sge *ib_sg; 2864 u32 offset; 2865 int ret = 0; 2866 unsigned long flags; 2867 2868 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2869 2870 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2871 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2872 se_cmd->t_data_nents, se_cmd->data_length, 2873 offset, wr->iser_ib_op, &wr->data); 2874 if (ret) 2875 return ret; 2876 2877 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2878 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2879 fr_desc = list_first_entry(&isert_conn->fr_pool, 2880 struct fast_reg_descriptor, list); 2881 list_del(&fr_desc->list); 2882 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2883 wr->fr_desc = fr_desc; 2884 } 2885 2886 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2887 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2888 if (ret) 2889 goto unmap_cmd; 2890 2891 if (isert_prot_cmd(isert_conn, se_cmd)) { 2892 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 2893 if (ret) 2894 goto unmap_cmd; 2895 2896 ib_sg = &wr->ib_sg[SIG]; 2897 } else { 2898 ib_sg = &wr->ib_sg[DATA]; 2899 } 2900 2901 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2902 wr->ib_sge = &wr->s_ib_sge; 2903 wr->send_wr_num = 1; 2904 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2905 wr->send_wr = &wr->s_send_wr; 2906 wr->isert_cmd = isert_cmd; 2907 2908 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2909 send_wr->sg_list = &wr->s_ib_sge; 2910 send_wr->num_sge = 1; 2911 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2912 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2913 send_wr->opcode = IB_WR_RDMA_WRITE; 2914 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2915 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2916 send_wr->send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 2917 0 : IB_SEND_SIGNALED; 2918 } else { 2919 send_wr->opcode = IB_WR_RDMA_READ; 2920 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2921 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2922 send_wr->send_flags = IB_SEND_SIGNALED; 2923 } 2924 2925 return 0; 2926 2927 unmap_cmd: 2928 if (fr_desc) { 2929 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2930 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2931 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2932 } 2933 isert_unmap_data_buf(isert_conn, &wr->data); 2934 2935 return ret; 2936 } 2937 2938 static int 2939 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2940 { 2941 struct se_cmd *se_cmd = &cmd->se_cmd; 2942 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2943 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2944 struct isert_conn *isert_conn = conn->context; 2945 struct isert_device *device = isert_conn->device; 2946 struct ib_send_wr *wr_failed; 2947 int rc; 2948 2949 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2950 isert_cmd, se_cmd->data_length); 2951 2952 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2953 rc = device->reg_rdma_mem(conn, cmd, wr); 2954 if (rc) { 2955 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2956 return rc; 2957 } 2958 2959 if (!isert_prot_cmd(isert_conn, se_cmd)) { 2960 /* 2961 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2962 */ 2963 isert_create_send_desc(isert_conn, isert_cmd, 2964 &isert_cmd->tx_desc); 2965 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2966 &isert_cmd->tx_desc.iscsi_header); 2967 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2968 isert_init_send_wr(isert_conn, isert_cmd, 2969 &isert_cmd->tx_desc.send_wr); 2970 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2971 wr->send_wr_num += 1; 2972 2973 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2974 if (rc) { 2975 isert_err("ib_post_recv failed with %d\n", rc); 2976 return rc; 2977 } 2978 } 2979 2980 rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed); 2981 if (rc) 2982 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2983 2984 if (!isert_prot_cmd(isert_conn, se_cmd)) 2985 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2986 "READ\n", isert_cmd); 2987 else 2988 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2989 isert_cmd); 2990 2991 return 1; 2992 } 2993 2994 static int 2995 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2996 { 2997 struct se_cmd *se_cmd = &cmd->se_cmd; 2998 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2999 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 3000 struct isert_conn *isert_conn = conn->context; 3001 struct isert_device *device = isert_conn->device; 3002 struct ib_send_wr *wr_failed; 3003 int rc; 3004 3005 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 3006 isert_cmd, se_cmd->data_length, cmd->write_data_done); 3007 wr->iser_ib_op = ISER_IB_RDMA_READ; 3008 rc = device->reg_rdma_mem(conn, cmd, wr); 3009 if (rc) { 3010 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 3011 return rc; 3012 } 3013 3014 rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed); 3015 if (rc) 3016 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 3017 3018 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 3019 isert_cmd); 3020 3021 return 0; 3022 } 3023 3024 static int 3025 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3026 { 3027 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 3028 int ret = 0; 3029 3030 switch (state) { 3031 case ISTATE_REMOVE: 3032 spin_lock_bh(&conn->cmd_lock); 3033 list_del_init(&cmd->i_conn_node); 3034 spin_unlock_bh(&conn->cmd_lock); 3035 isert_put_cmd(isert_cmd, true); 3036 break; 3037 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3038 ret = isert_put_nopin(cmd, conn, false); 3039 break; 3040 default: 3041 isert_err("Unknown immediate state: 0x%02x\n", state); 3042 ret = -EINVAL; 3043 break; 3044 } 3045 3046 return ret; 3047 } 3048 3049 static int 3050 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3051 { 3052 struct isert_conn *isert_conn = conn->context; 3053 int ret; 3054 3055 switch (state) { 3056 case ISTATE_SEND_LOGOUTRSP: 3057 ret = isert_put_logout_rsp(cmd, conn); 3058 if (!ret) 3059 isert_conn->logout_posted = true; 3060 break; 3061 case ISTATE_SEND_NOPIN: 3062 ret = isert_put_nopin(cmd, conn, true); 3063 break; 3064 case ISTATE_SEND_TASKMGTRSP: 3065 ret = isert_put_tm_rsp(cmd, conn); 3066 break; 3067 case ISTATE_SEND_REJECT: 3068 ret = isert_put_reject(cmd, conn); 3069 break; 3070 case ISTATE_SEND_TEXTRSP: 3071 ret = isert_put_text_rsp(cmd, conn); 3072 break; 3073 case ISTATE_SEND_STATUS: 3074 /* 3075 * Special case for sending non GOOD SCSI status from TX thread 3076 * context during pre se_cmd excecution failure. 3077 */ 3078 ret = isert_put_response(conn, cmd); 3079 break; 3080 default: 3081 isert_err("Unknown response state: 0x%02x\n", state); 3082 ret = -EINVAL; 3083 break; 3084 } 3085 3086 return ret; 3087 } 3088 3089 struct rdma_cm_id * 3090 isert_setup_id(struct isert_np *isert_np) 3091 { 3092 struct iscsi_np *np = isert_np->np; 3093 struct rdma_cm_id *id; 3094 struct sockaddr *sa; 3095 int ret; 3096 3097 sa = (struct sockaddr *)&np->np_sockaddr; 3098 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3099 3100 id = rdma_create_id(isert_cma_handler, isert_np, 3101 RDMA_PS_TCP, IB_QPT_RC); 3102 if (IS_ERR(id)) { 3103 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3104 ret = PTR_ERR(id); 3105 goto out; 3106 } 3107 isert_dbg("id %p context %p\n", id, id->context); 3108 3109 ret = rdma_bind_addr(id, sa); 3110 if (ret) { 3111 isert_err("rdma_bind_addr() failed: %d\n", ret); 3112 goto out_id; 3113 } 3114 3115 ret = rdma_listen(id, 0); 3116 if (ret) { 3117 isert_err("rdma_listen() failed: %d\n", ret); 3118 goto out_id; 3119 } 3120 3121 return id; 3122 out_id: 3123 rdma_destroy_id(id); 3124 out: 3125 return ERR_PTR(ret); 3126 } 3127 3128 static int 3129 isert_setup_np(struct iscsi_np *np, 3130 struct sockaddr_storage *ksockaddr) 3131 { 3132 struct isert_np *isert_np; 3133 struct rdma_cm_id *isert_lid; 3134 int ret; 3135 3136 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3137 if (!isert_np) { 3138 isert_err("Unable to allocate struct isert_np\n"); 3139 return -ENOMEM; 3140 } 3141 sema_init(&isert_np->sem, 0); 3142 mutex_init(&isert_np->mutex); 3143 INIT_LIST_HEAD(&isert_np->accepted); 3144 INIT_LIST_HEAD(&isert_np->pending); 3145 isert_np->np = np; 3146 3147 /* 3148 * Setup the np->np_sockaddr from the passed sockaddr setup 3149 * in iscsi_target_configfs.c code.. 3150 */ 3151 memcpy(&np->np_sockaddr, ksockaddr, 3152 sizeof(struct sockaddr_storage)); 3153 3154 isert_lid = isert_setup_id(isert_np); 3155 if (IS_ERR(isert_lid)) { 3156 ret = PTR_ERR(isert_lid); 3157 goto out; 3158 } 3159 3160 isert_np->cm_id = isert_lid; 3161 np->np_context = isert_np; 3162 3163 return 0; 3164 3165 out: 3166 kfree(isert_np); 3167 3168 return ret; 3169 } 3170 3171 static int 3172 isert_rdma_accept(struct isert_conn *isert_conn) 3173 { 3174 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3175 struct rdma_conn_param cp; 3176 int ret; 3177 3178 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3179 cp.initiator_depth = isert_conn->initiator_depth; 3180 cp.retry_count = 7; 3181 cp.rnr_retry_count = 7; 3182 3183 ret = rdma_accept(cm_id, &cp); 3184 if (ret) { 3185 isert_err("rdma_accept() failed with: %d\n", ret); 3186 return ret; 3187 } 3188 3189 return 0; 3190 } 3191 3192 static int 3193 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3194 { 3195 struct isert_conn *isert_conn = conn->context; 3196 int ret; 3197 3198 isert_info("before login_req comp conn: %p\n", isert_conn); 3199 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 3200 if (ret) { 3201 isert_err("isert_conn %p interrupted before got login req\n", 3202 isert_conn); 3203 return ret; 3204 } 3205 reinit_completion(&isert_conn->login_req_comp); 3206 3207 /* 3208 * For login requests after the first PDU, isert_rx_login_req() will 3209 * kick schedule_delayed_work(&conn->login_work) as the packet is 3210 * received, which turns this callback from iscsi_target_do_login_rx() 3211 * into a NOP. 3212 */ 3213 if (!login->first_request) 3214 return 0; 3215 3216 isert_rx_login_req(isert_conn); 3217 3218 isert_info("before login_comp conn: %p\n", conn); 3219 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3220 if (ret) 3221 return ret; 3222 3223 isert_info("processing login->req: %p\n", login->req); 3224 3225 return 0; 3226 } 3227 3228 static void 3229 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3230 struct isert_conn *isert_conn) 3231 { 3232 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3233 struct rdma_route *cm_route = &cm_id->route; 3234 3235 conn->login_family = np->np_sockaddr.ss_family; 3236 3237 conn->login_sockaddr = cm_route->addr.dst_addr; 3238 conn->local_sockaddr = cm_route->addr.src_addr; 3239 } 3240 3241 static int 3242 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3243 { 3244 struct isert_np *isert_np = np->np_context; 3245 struct isert_conn *isert_conn; 3246 int ret; 3247 3248 accept_wait: 3249 ret = down_interruptible(&isert_np->sem); 3250 if (ret) 3251 return -ENODEV; 3252 3253 spin_lock_bh(&np->np_thread_lock); 3254 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3255 spin_unlock_bh(&np->np_thread_lock); 3256 isert_dbg("np_thread_state %d\n", 3257 np->np_thread_state); 3258 /** 3259 * No point in stalling here when np_thread 3260 * is in state RESET/SHUTDOWN/EXIT - bail 3261 **/ 3262 return -ENODEV; 3263 } 3264 spin_unlock_bh(&np->np_thread_lock); 3265 3266 mutex_lock(&isert_np->mutex); 3267 if (list_empty(&isert_np->pending)) { 3268 mutex_unlock(&isert_np->mutex); 3269 goto accept_wait; 3270 } 3271 isert_conn = list_first_entry(&isert_np->pending, 3272 struct isert_conn, node); 3273 list_del_init(&isert_conn->node); 3274 mutex_unlock(&isert_np->mutex); 3275 3276 conn->context = isert_conn; 3277 isert_conn->conn = conn; 3278 3279 isert_set_conn_info(np, conn, isert_conn); 3280 3281 isert_dbg("Processing isert_conn: %p\n", isert_conn); 3282 3283 return 0; 3284 } 3285 3286 static void 3287 isert_free_np(struct iscsi_np *np) 3288 { 3289 struct isert_np *isert_np = np->np_context; 3290 struct isert_conn *isert_conn, *n; 3291 3292 if (isert_np->cm_id) 3293 rdma_destroy_id(isert_np->cm_id); 3294 3295 /* 3296 * FIXME: At this point we don't have a good way to insure 3297 * that at this point we don't have hanging connections that 3298 * completed RDMA establishment but didn't start iscsi login 3299 * process. So work-around this by cleaning up what ever piled 3300 * up in accepted and pending lists. 3301 */ 3302 mutex_lock(&isert_np->mutex); 3303 if (!list_empty(&isert_np->pending)) { 3304 isert_info("Still have isert pending connections\n"); 3305 list_for_each_entry_safe(isert_conn, n, 3306 &isert_np->pending, 3307 node) { 3308 isert_info("cleaning isert_conn %p state (%d)\n", 3309 isert_conn, isert_conn->state); 3310 isert_connect_release(isert_conn); 3311 } 3312 } 3313 3314 if (!list_empty(&isert_np->accepted)) { 3315 isert_info("Still have isert accepted connections\n"); 3316 list_for_each_entry_safe(isert_conn, n, 3317 &isert_np->accepted, 3318 node) { 3319 isert_info("cleaning isert_conn %p state (%d)\n", 3320 isert_conn, isert_conn->state); 3321 isert_connect_release(isert_conn); 3322 } 3323 } 3324 mutex_unlock(&isert_np->mutex); 3325 3326 np->np_context = NULL; 3327 kfree(isert_np); 3328 } 3329 3330 static void isert_release_work(struct work_struct *work) 3331 { 3332 struct isert_conn *isert_conn = container_of(work, 3333 struct isert_conn, 3334 release_work); 3335 3336 isert_info("Starting release conn %p\n", isert_conn); 3337 3338 wait_for_completion(&isert_conn->wait); 3339 3340 mutex_lock(&isert_conn->mutex); 3341 isert_conn->state = ISER_CONN_DOWN; 3342 mutex_unlock(&isert_conn->mutex); 3343 3344 isert_info("Destroying conn %p\n", isert_conn); 3345 isert_put_conn(isert_conn); 3346 } 3347 3348 static void 3349 isert_wait4logout(struct isert_conn *isert_conn) 3350 { 3351 struct iscsi_conn *conn = isert_conn->conn; 3352 3353 isert_info("conn %p\n", isert_conn); 3354 3355 if (isert_conn->logout_posted) { 3356 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3357 wait_for_completion_timeout(&conn->conn_logout_comp, 3358 SECONDS_FOR_LOGOUT_COMP * HZ); 3359 } 3360 } 3361 3362 static void 3363 isert_wait4cmds(struct iscsi_conn *conn) 3364 { 3365 isert_info("iscsi_conn %p\n", conn); 3366 3367 if (conn->sess) { 3368 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3369 target_wait_for_sess_cmds(conn->sess->se_sess); 3370 } 3371 } 3372 3373 static void 3374 isert_wait4flush(struct isert_conn *isert_conn) 3375 { 3376 struct ib_recv_wr *bad_wr; 3377 3378 isert_info("conn %p\n", isert_conn); 3379 3380 init_completion(&isert_conn->wait_comp_err); 3381 isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3382 /* post an indication that all flush errors were consumed */ 3383 if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { 3384 isert_err("conn %p failed to post beacon", isert_conn); 3385 return; 3386 } 3387 3388 wait_for_completion(&isert_conn->wait_comp_err); 3389 } 3390 3391 /** 3392 * isert_put_unsol_pending_cmds() - Drop commands waiting for 3393 * unsolicitate dataout 3394 * @conn: iscsi connection 3395 * 3396 * We might still have commands that are waiting for unsolicited 3397 * dataouts messages. We must put the extra reference on those 3398 * before blocking on the target_wait_for_session_cmds 3399 */ 3400 static void 3401 isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 3402 { 3403 struct iscsi_cmd *cmd, *tmp; 3404 static LIST_HEAD(drop_cmd_list); 3405 3406 spin_lock_bh(&conn->cmd_lock); 3407 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 3408 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 3409 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 3410 (cmd->write_data_done < cmd->se_cmd.data_length)) 3411 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 3412 } 3413 spin_unlock_bh(&conn->cmd_lock); 3414 3415 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 3416 list_del_init(&cmd->i_conn_node); 3417 if (cmd->i_state != ISTATE_REMOVE) { 3418 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 3419 3420 isert_info("conn %p dropping cmd %p\n", conn, cmd); 3421 isert_put_cmd(isert_cmd, true); 3422 } 3423 } 3424 } 3425 3426 static void isert_wait_conn(struct iscsi_conn *conn) 3427 { 3428 struct isert_conn *isert_conn = conn->context; 3429 3430 isert_info("Starting conn %p\n", isert_conn); 3431 3432 mutex_lock(&isert_conn->mutex); 3433 /* 3434 * Only wait for wait_comp_err if the isert_conn made it 3435 * into full feature phase.. 3436 */ 3437 if (isert_conn->state == ISER_CONN_INIT) { 3438 mutex_unlock(&isert_conn->mutex); 3439 return; 3440 } 3441 isert_conn_terminate(isert_conn); 3442 mutex_unlock(&isert_conn->mutex); 3443 3444 isert_wait4flush(isert_conn); 3445 isert_put_unsol_pending_cmds(conn); 3446 isert_wait4cmds(conn); 3447 isert_wait4logout(isert_conn); 3448 3449 queue_work(isert_release_wq, &isert_conn->release_work); 3450 } 3451 3452 static void isert_free_conn(struct iscsi_conn *conn) 3453 { 3454 struct isert_conn *isert_conn = conn->context; 3455 3456 isert_wait4flush(isert_conn); 3457 isert_put_conn(isert_conn); 3458 } 3459 3460 static struct iscsit_transport iser_target_transport = { 3461 .name = "IB/iSER", 3462 .transport_type = ISCSI_INFINIBAND, 3463 .priv_size = sizeof(struct isert_cmd), 3464 .owner = THIS_MODULE, 3465 .iscsit_setup_np = isert_setup_np, 3466 .iscsit_accept_np = isert_accept_np, 3467 .iscsit_free_np = isert_free_np, 3468 .iscsit_wait_conn = isert_wait_conn, 3469 .iscsit_free_conn = isert_free_conn, 3470 .iscsit_get_login_rx = isert_get_login_rx, 3471 .iscsit_put_login_tx = isert_put_login_tx, 3472 .iscsit_immediate_queue = isert_immediate_queue, 3473 .iscsit_response_queue = isert_response_queue, 3474 .iscsit_get_dataout = isert_get_dataout, 3475 .iscsit_queue_data_in = isert_put_datain, 3476 .iscsit_queue_status = isert_put_response, 3477 .iscsit_aborted_task = isert_aborted_task, 3478 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3479 }; 3480 3481 static int __init isert_init(void) 3482 { 3483 int ret; 3484 3485 isert_comp_wq = alloc_workqueue("isert_comp_wq", 3486 WQ_UNBOUND | WQ_HIGHPRI, 0); 3487 if (!isert_comp_wq) { 3488 isert_err("Unable to allocate isert_comp_wq\n"); 3489 ret = -ENOMEM; 3490 return -ENOMEM; 3491 } 3492 3493 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3494 WQ_UNBOUND_MAX_ACTIVE); 3495 if (!isert_release_wq) { 3496 isert_err("Unable to allocate isert_release_wq\n"); 3497 ret = -ENOMEM; 3498 goto destroy_comp_wq; 3499 } 3500 3501 iscsit_register_transport(&iser_target_transport); 3502 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3503 3504 return 0; 3505 3506 destroy_comp_wq: 3507 destroy_workqueue(isert_comp_wq); 3508 3509 return ret; 3510 } 3511 3512 static void __exit isert_exit(void) 3513 { 3514 flush_scheduled_work(); 3515 destroy_workqueue(isert_release_wq); 3516 destroy_workqueue(isert_comp_wq); 3517 iscsit_unregister_transport(&iser_target_transport); 3518 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3519 } 3520 3521 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3522 MODULE_VERSION("1.0"); 3523 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3524 MODULE_LICENSE("GPL"); 3525 3526 module_init(isert_init); 3527 module_exit(isert_exit); 3528