1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 int isert_debug_level = 0; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static void 51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static void 56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 57 static int 58 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 59 struct isert_rdma_wr *wr); 60 static int 61 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 62 static int 63 isert_rdma_post_recvl(struct isert_conn *isert_conn); 64 static int 65 isert_rdma_accept(struct isert_conn *isert_conn); 66 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 67 68 static inline bool 69 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 70 { 71 return (conn->pi_support && 72 cmd->prot_op != TARGET_PROT_NORMAL); 73 } 74 75 76 static void 77 isert_qp_event_callback(struct ib_event *e, void *context) 78 { 79 struct isert_conn *isert_conn = (struct isert_conn *)context; 80 81 isert_err("conn %p event: %d\n", isert_conn, e->event); 82 switch (e->event) { 83 case IB_EVENT_COMM_EST: 84 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 85 break; 86 case IB_EVENT_QP_LAST_WQE_REACHED: 87 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 88 break; 89 default: 90 break; 91 } 92 } 93 94 static int 95 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 96 { 97 int ret; 98 99 ret = ib_query_device(ib_dev, devattr); 100 if (ret) { 101 isert_err("ib_query_device() failed: %d\n", ret); 102 return ret; 103 } 104 isert_dbg("devattr->max_sge: %d\n", devattr->max_sge); 105 isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 106 107 return 0; 108 } 109 110 static int 111 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 112 { 113 struct isert_device *device = isert_conn->conn_device; 114 struct ib_qp_init_attr attr; 115 struct isert_comp *comp; 116 int ret, i, min = 0; 117 118 mutex_lock(&device_list_mutex); 119 for (i = 0; i < device->comps_used; i++) 120 if (device->comps[i].active_qps < 121 device->comps[min].active_qps) 122 min = i; 123 comp = &device->comps[min]; 124 comp->active_qps++; 125 isert_info("conn %p, using comp %p min_index: %d\n", 126 isert_conn, comp, min); 127 mutex_unlock(&device_list_mutex); 128 129 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 130 attr.event_handler = isert_qp_event_callback; 131 attr.qp_context = isert_conn; 132 attr.send_cq = comp->cq; 133 attr.recv_cq = comp->cq; 134 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 135 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 136 /* 137 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 138 * work-around for RDMA_READs with ConnectX-2. 139 * 140 * Also, still make sure to have at least two SGEs for 141 * outgoing control PDU responses. 142 */ 143 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 144 isert_conn->max_sge = attr.cap.max_send_sge; 145 146 attr.cap.max_recv_sge = 1; 147 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 148 attr.qp_type = IB_QPT_RC; 149 if (device->pi_capable) 150 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 151 152 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 153 if (ret) { 154 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 155 goto err; 156 } 157 isert_conn->conn_qp = cma_id->qp; 158 159 return 0; 160 err: 161 mutex_lock(&device_list_mutex); 162 comp->active_qps--; 163 mutex_unlock(&device_list_mutex); 164 165 return ret; 166 } 167 168 static void 169 isert_cq_event_callback(struct ib_event *e, void *context) 170 { 171 isert_dbg("event: %d\n", e->event); 172 } 173 174 static int 175 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 176 { 177 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 178 struct iser_rx_desc *rx_desc; 179 struct ib_sge *rx_sg; 180 u64 dma_addr; 181 int i, j; 182 183 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 184 sizeof(struct iser_rx_desc), GFP_KERNEL); 185 if (!isert_conn->conn_rx_descs) 186 goto fail; 187 188 rx_desc = isert_conn->conn_rx_descs; 189 190 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 191 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 192 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 193 if (ib_dma_mapping_error(ib_dev, dma_addr)) 194 goto dma_map_fail; 195 196 rx_desc->dma_addr = dma_addr; 197 198 rx_sg = &rx_desc->rx_sg; 199 rx_sg->addr = rx_desc->dma_addr; 200 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 201 rx_sg->lkey = isert_conn->conn_mr->lkey; 202 } 203 204 isert_conn->conn_rx_desc_head = 0; 205 206 return 0; 207 208 dma_map_fail: 209 rx_desc = isert_conn->conn_rx_descs; 210 for (j = 0; j < i; j++, rx_desc++) { 211 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 212 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 213 } 214 kfree(isert_conn->conn_rx_descs); 215 isert_conn->conn_rx_descs = NULL; 216 fail: 217 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 218 219 return -ENOMEM; 220 } 221 222 static void 223 isert_free_rx_descriptors(struct isert_conn *isert_conn) 224 { 225 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 226 struct iser_rx_desc *rx_desc; 227 int i; 228 229 if (!isert_conn->conn_rx_descs) 230 return; 231 232 rx_desc = isert_conn->conn_rx_descs; 233 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 234 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 235 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 236 } 237 238 kfree(isert_conn->conn_rx_descs); 239 isert_conn->conn_rx_descs = NULL; 240 } 241 242 static void isert_cq_work(struct work_struct *); 243 static void isert_cq_callback(struct ib_cq *, void *); 244 245 static int 246 isert_create_device_ib_res(struct isert_device *device) 247 { 248 struct ib_device *ib_dev = device->ib_device; 249 struct ib_device_attr *dev_attr; 250 int ret = 0, i; 251 int max_cqe; 252 253 dev_attr = &device->dev_attr; 254 ret = isert_query_device(ib_dev, dev_attr); 255 if (ret) 256 return ret; 257 258 max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe); 259 260 /* asign function handlers */ 261 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 262 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 263 device->use_fastreg = 1; 264 device->reg_rdma_mem = isert_reg_rdma; 265 device->unreg_rdma_mem = isert_unreg_rdma; 266 } else { 267 device->use_fastreg = 0; 268 device->reg_rdma_mem = isert_map_rdma; 269 device->unreg_rdma_mem = isert_unmap_cmd; 270 } 271 272 /* Check signature cap */ 273 device->pi_capable = dev_attr->device_cap_flags & 274 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 275 276 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 277 device->ib_device->num_comp_vectors)); 278 isert_info("Using %d CQs, %s supports %d vectors support " 279 "Fast registration %d pi_capable %d\n", 280 device->comps_used, device->ib_device->name, 281 device->ib_device->num_comp_vectors, device->use_fastreg, 282 device->pi_capable); 283 284 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 285 GFP_KERNEL); 286 if (!device->comps) { 287 isert_err("Unable to allocate completion contexts\n"); 288 return -ENOMEM; 289 } 290 291 for (i = 0; i < device->comps_used; i++) { 292 struct isert_comp *comp = &device->comps[i]; 293 294 comp->device = device; 295 INIT_WORK(&comp->work, isert_cq_work); 296 comp->cq = ib_create_cq(device->ib_device, 297 isert_cq_callback, 298 isert_cq_event_callback, 299 (void *)comp, 300 max_cqe, i); 301 if (IS_ERR(comp->cq)) { 302 ret = PTR_ERR(comp->cq); 303 comp->cq = NULL; 304 goto out_cq; 305 } 306 307 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 308 if (ret) 309 goto out_cq; 310 } 311 312 return 0; 313 314 out_cq: 315 for (i = 0; i < device->comps_used; i++) { 316 struct isert_comp *comp = &device->comps[i]; 317 318 if (comp->cq) { 319 cancel_work_sync(&comp->work); 320 ib_destroy_cq(comp->cq); 321 } 322 } 323 kfree(device->comps); 324 325 return ret; 326 } 327 328 static void 329 isert_free_device_ib_res(struct isert_device *device) 330 { 331 int i; 332 333 isert_info("device %p\n", device); 334 335 for (i = 0; i < device->comps_used; i++) { 336 struct isert_comp *comp = &device->comps[i]; 337 338 cancel_work_sync(&comp->work); 339 ib_destroy_cq(comp->cq); 340 comp->cq = NULL; 341 } 342 kfree(device->comps); 343 } 344 345 static void 346 isert_device_try_release(struct isert_device *device) 347 { 348 mutex_lock(&device_list_mutex); 349 device->refcount--; 350 isert_info("device %p refcount %d\n", device, device->refcount); 351 if (!device->refcount) { 352 isert_free_device_ib_res(device); 353 list_del(&device->dev_node); 354 kfree(device); 355 } 356 mutex_unlock(&device_list_mutex); 357 } 358 359 static struct isert_device * 360 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 361 { 362 struct isert_device *device; 363 int ret; 364 365 mutex_lock(&device_list_mutex); 366 list_for_each_entry(device, &device_list, dev_node) { 367 if (device->ib_device->node_guid == cma_id->device->node_guid) { 368 device->refcount++; 369 isert_info("Found iser device %p refcount %d\n", 370 device, device->refcount); 371 mutex_unlock(&device_list_mutex); 372 return device; 373 } 374 } 375 376 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 377 if (!device) { 378 mutex_unlock(&device_list_mutex); 379 return ERR_PTR(-ENOMEM); 380 } 381 382 INIT_LIST_HEAD(&device->dev_node); 383 384 device->ib_device = cma_id->device; 385 ret = isert_create_device_ib_res(device); 386 if (ret) { 387 kfree(device); 388 mutex_unlock(&device_list_mutex); 389 return ERR_PTR(ret); 390 } 391 392 device->refcount++; 393 list_add_tail(&device->dev_node, &device_list); 394 isert_info("Created a new iser device %p refcount %d\n", 395 device, device->refcount); 396 mutex_unlock(&device_list_mutex); 397 398 return device; 399 } 400 401 static void 402 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 403 { 404 struct fast_reg_descriptor *fr_desc, *tmp; 405 int i = 0; 406 407 if (list_empty(&isert_conn->conn_fr_pool)) 408 return; 409 410 isert_info("Freeing conn %p fastreg pool", isert_conn); 411 412 list_for_each_entry_safe(fr_desc, tmp, 413 &isert_conn->conn_fr_pool, list) { 414 list_del(&fr_desc->list); 415 ib_free_fast_reg_page_list(fr_desc->data_frpl); 416 ib_dereg_mr(fr_desc->data_mr); 417 if (fr_desc->pi_ctx) { 418 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 419 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 420 ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 421 kfree(fr_desc->pi_ctx); 422 } 423 kfree(fr_desc); 424 ++i; 425 } 426 427 if (i < isert_conn->conn_fr_pool_size) 428 isert_warn("Pool still has %d regions registered\n", 429 isert_conn->conn_fr_pool_size - i); 430 } 431 432 static int 433 isert_create_pi_ctx(struct fast_reg_descriptor *desc, 434 struct ib_device *device, 435 struct ib_pd *pd) 436 { 437 struct ib_mr_init_attr mr_init_attr; 438 struct pi_context *pi_ctx; 439 int ret; 440 441 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 442 if (!pi_ctx) { 443 isert_err("Failed to allocate pi context\n"); 444 return -ENOMEM; 445 } 446 447 pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(device, 448 ISCSI_ISER_SG_TABLESIZE); 449 if (IS_ERR(pi_ctx->prot_frpl)) { 450 isert_err("Failed to allocate prot frpl err=%ld\n", 451 PTR_ERR(pi_ctx->prot_frpl)); 452 ret = PTR_ERR(pi_ctx->prot_frpl); 453 goto err_pi_ctx; 454 } 455 456 pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 457 if (IS_ERR(pi_ctx->prot_mr)) { 458 isert_err("Failed to allocate prot frmr err=%ld\n", 459 PTR_ERR(pi_ctx->prot_mr)); 460 ret = PTR_ERR(pi_ctx->prot_mr); 461 goto err_prot_frpl; 462 } 463 desc->ind |= ISERT_PROT_KEY_VALID; 464 465 memset(&mr_init_attr, 0, sizeof(mr_init_attr)); 466 mr_init_attr.max_reg_descriptors = 2; 467 mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 468 pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 469 if (IS_ERR(pi_ctx->sig_mr)) { 470 isert_err("Failed to allocate signature enabled mr err=%ld\n", 471 PTR_ERR(pi_ctx->sig_mr)); 472 ret = PTR_ERR(pi_ctx->sig_mr); 473 goto err_prot_mr; 474 } 475 476 desc->pi_ctx = pi_ctx; 477 desc->ind |= ISERT_SIG_KEY_VALID; 478 desc->ind &= ~ISERT_PROTECTED; 479 480 return 0; 481 482 err_prot_mr: 483 ib_dereg_mr(desc->pi_ctx->prot_mr); 484 err_prot_frpl: 485 ib_free_fast_reg_page_list(desc->pi_ctx->prot_frpl); 486 err_pi_ctx: 487 kfree(desc->pi_ctx); 488 489 return ret; 490 } 491 492 static int 493 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 494 struct fast_reg_descriptor *fr_desc) 495 { 496 int ret; 497 498 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 499 ISCSI_ISER_SG_TABLESIZE); 500 if (IS_ERR(fr_desc->data_frpl)) { 501 isert_err("Failed to allocate data frpl err=%ld\n", 502 PTR_ERR(fr_desc->data_frpl)); 503 return PTR_ERR(fr_desc->data_frpl); 504 } 505 506 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 507 if (IS_ERR(fr_desc->data_mr)) { 508 isert_err("Failed to allocate data frmr err=%ld\n", 509 PTR_ERR(fr_desc->data_mr)); 510 ret = PTR_ERR(fr_desc->data_mr); 511 goto err_data_frpl; 512 } 513 fr_desc->ind |= ISERT_DATA_KEY_VALID; 514 515 isert_dbg("Created fr_desc %p\n", fr_desc); 516 517 return 0; 518 519 err_data_frpl: 520 ib_free_fast_reg_page_list(fr_desc->data_frpl); 521 522 return ret; 523 } 524 525 static int 526 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 527 { 528 struct fast_reg_descriptor *fr_desc; 529 struct isert_device *device = isert_conn->conn_device; 530 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 531 struct se_node_acl *se_nacl = se_sess->se_node_acl; 532 int i, ret, tag_num; 533 /* 534 * Setup the number of FRMRs based upon the number of tags 535 * available to session in iscsi_target_locate_portal(). 536 */ 537 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 538 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 539 540 isert_conn->conn_fr_pool_size = 0; 541 for (i = 0; i < tag_num; i++) { 542 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 543 if (!fr_desc) { 544 isert_err("Failed to allocate fast_reg descriptor\n"); 545 ret = -ENOMEM; 546 goto err; 547 } 548 549 ret = isert_create_fr_desc(device->ib_device, 550 isert_conn->conn_pd, fr_desc); 551 if (ret) { 552 isert_err("Failed to create fastreg descriptor err=%d\n", 553 ret); 554 kfree(fr_desc); 555 goto err; 556 } 557 558 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 559 isert_conn->conn_fr_pool_size++; 560 } 561 562 isert_dbg("Creating conn %p fastreg pool size=%d", 563 isert_conn, isert_conn->conn_fr_pool_size); 564 565 return 0; 566 567 err: 568 isert_conn_free_fastreg_pool(isert_conn); 569 return ret; 570 } 571 572 static int 573 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 574 { 575 struct isert_np *isert_np = cma_id->context; 576 struct iscsi_np *np = isert_np->np; 577 struct isert_conn *isert_conn; 578 struct isert_device *device; 579 struct ib_device *ib_dev = cma_id->device; 580 int ret = 0; 581 582 spin_lock_bh(&np->np_thread_lock); 583 if (!np->enabled) { 584 spin_unlock_bh(&np->np_thread_lock); 585 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 586 return rdma_reject(cma_id, NULL, 0); 587 } 588 spin_unlock_bh(&np->np_thread_lock); 589 590 isert_dbg("cma_id: %p, portal: %p\n", 591 cma_id, cma_id->context); 592 593 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 594 if (!isert_conn) { 595 isert_err("Unable to allocate isert_conn\n"); 596 return -ENOMEM; 597 } 598 isert_conn->state = ISER_CONN_INIT; 599 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 600 init_completion(&isert_conn->conn_login_comp); 601 init_completion(&isert_conn->login_req_comp); 602 init_completion(&isert_conn->conn_wait); 603 kref_init(&isert_conn->conn_kref); 604 mutex_init(&isert_conn->conn_mutex); 605 spin_lock_init(&isert_conn->conn_lock); 606 INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 607 608 isert_conn->conn_cm_id = cma_id; 609 610 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 611 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 612 if (!isert_conn->login_buf) { 613 isert_err("Unable to allocate isert_conn->login_buf\n"); 614 ret = -ENOMEM; 615 goto out; 616 } 617 618 isert_conn->login_req_buf = isert_conn->login_buf; 619 isert_conn->login_rsp_buf = isert_conn->login_buf + 620 ISCSI_DEF_MAX_RECV_SEG_LEN; 621 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 622 isert_conn->login_buf, isert_conn->login_req_buf, 623 isert_conn->login_rsp_buf); 624 625 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 626 (void *)isert_conn->login_req_buf, 627 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 628 629 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 630 if (ret) { 631 isert_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 632 ret); 633 isert_conn->login_req_dma = 0; 634 goto out_login_buf; 635 } 636 637 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 638 (void *)isert_conn->login_rsp_buf, 639 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 640 641 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 642 if (ret) { 643 isert_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 644 ret); 645 isert_conn->login_rsp_dma = 0; 646 goto out_req_dma_map; 647 } 648 649 device = isert_device_find_by_ib_dev(cma_id); 650 if (IS_ERR(device)) { 651 ret = PTR_ERR(device); 652 goto out_rsp_dma_map; 653 } 654 655 /* Set max inflight RDMA READ requests */ 656 isert_conn->initiator_depth = min_t(u8, 657 event->param.conn.initiator_depth, 658 device->dev_attr.max_qp_init_rd_atom); 659 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 660 661 isert_conn->conn_device = device; 662 isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 663 if (IS_ERR(isert_conn->conn_pd)) { 664 ret = PTR_ERR(isert_conn->conn_pd); 665 isert_err("ib_alloc_pd failed for conn %p: ret=%d\n", 666 isert_conn, ret); 667 goto out_pd; 668 } 669 670 isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 671 IB_ACCESS_LOCAL_WRITE); 672 if (IS_ERR(isert_conn->conn_mr)) { 673 ret = PTR_ERR(isert_conn->conn_mr); 674 isert_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 675 isert_conn, ret); 676 goto out_mr; 677 } 678 679 ret = isert_conn_setup_qp(isert_conn, cma_id); 680 if (ret) 681 goto out_conn_dev; 682 683 ret = isert_rdma_post_recvl(isert_conn); 684 if (ret) 685 goto out_conn_dev; 686 687 ret = isert_rdma_accept(isert_conn); 688 if (ret) 689 goto out_conn_dev; 690 691 mutex_lock(&isert_np->np_accept_mutex); 692 list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list); 693 mutex_unlock(&isert_np->np_accept_mutex); 694 695 isert_info("np %p: Allow accept_np to continue\n", np); 696 up(&isert_np->np_sem); 697 return 0; 698 699 out_conn_dev: 700 ib_dereg_mr(isert_conn->conn_mr); 701 out_mr: 702 ib_dealloc_pd(isert_conn->conn_pd); 703 out_pd: 704 isert_device_try_release(device); 705 out_rsp_dma_map: 706 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 707 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 708 out_req_dma_map: 709 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 710 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 711 out_login_buf: 712 kfree(isert_conn->login_buf); 713 out: 714 kfree(isert_conn); 715 rdma_reject(cma_id, NULL, 0); 716 return ret; 717 } 718 719 static void 720 isert_connect_release(struct isert_conn *isert_conn) 721 { 722 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 723 struct isert_device *device = isert_conn->conn_device; 724 725 isert_dbg("conn %p\n", isert_conn); 726 727 if (device && device->use_fastreg) 728 isert_conn_free_fastreg_pool(isert_conn); 729 730 isert_free_rx_descriptors(isert_conn); 731 rdma_destroy_id(isert_conn->conn_cm_id); 732 733 if (isert_conn->conn_qp) { 734 struct isert_comp *comp = isert_conn->conn_qp->recv_cq->cq_context; 735 736 isert_dbg("dec completion context %p active_qps\n", comp); 737 mutex_lock(&device_list_mutex); 738 comp->active_qps--; 739 mutex_unlock(&device_list_mutex); 740 741 ib_destroy_qp(isert_conn->conn_qp); 742 } 743 744 ib_dereg_mr(isert_conn->conn_mr); 745 ib_dealloc_pd(isert_conn->conn_pd); 746 747 if (isert_conn->login_buf) { 748 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 749 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 750 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 751 ISCSI_DEF_MAX_RECV_SEG_LEN, 752 DMA_FROM_DEVICE); 753 kfree(isert_conn->login_buf); 754 } 755 kfree(isert_conn); 756 757 if (device) 758 isert_device_try_release(device); 759 } 760 761 static void 762 isert_connected_handler(struct rdma_cm_id *cma_id) 763 { 764 struct isert_conn *isert_conn = cma_id->qp->qp_context; 765 766 isert_info("conn %p\n", isert_conn); 767 768 if (!kref_get_unless_zero(&isert_conn->conn_kref)) { 769 isert_warn("conn %p connect_release is running\n", isert_conn); 770 return; 771 } 772 773 mutex_lock(&isert_conn->conn_mutex); 774 if (isert_conn->state != ISER_CONN_FULL_FEATURE) 775 isert_conn->state = ISER_CONN_UP; 776 mutex_unlock(&isert_conn->conn_mutex); 777 } 778 779 static void 780 isert_release_conn_kref(struct kref *kref) 781 { 782 struct isert_conn *isert_conn = container_of(kref, 783 struct isert_conn, conn_kref); 784 785 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 786 current->pid); 787 788 isert_connect_release(isert_conn); 789 } 790 791 static void 792 isert_put_conn(struct isert_conn *isert_conn) 793 { 794 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 795 } 796 797 /** 798 * isert_conn_terminate() - Initiate connection termination 799 * @isert_conn: isert connection struct 800 * 801 * Notes: 802 * In case the connection state is FULL_FEATURE, move state 803 * to TEMINATING and start teardown sequence (rdma_disconnect). 804 * In case the connection state is UP, complete flush as well. 805 * 806 * This routine must be called with conn_mutex held. Thus it is 807 * safe to call multiple times. 808 */ 809 static void 810 isert_conn_terminate(struct isert_conn *isert_conn) 811 { 812 int err; 813 814 switch (isert_conn->state) { 815 case ISER_CONN_TERMINATING: 816 break; 817 case ISER_CONN_UP: 818 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 819 isert_info("Terminating conn %p state %d\n", 820 isert_conn, isert_conn->state); 821 isert_conn->state = ISER_CONN_TERMINATING; 822 err = rdma_disconnect(isert_conn->conn_cm_id); 823 if (err) 824 isert_warn("Failed rdma_disconnect isert_conn %p\n", 825 isert_conn); 826 break; 827 default: 828 isert_warn("conn %p teminating in state %d\n", 829 isert_conn, isert_conn->state); 830 } 831 } 832 833 static int 834 isert_np_cma_handler(struct isert_np *isert_np, 835 enum rdma_cm_event_type event) 836 { 837 isert_dbg("isert np %p, handling event %d\n", isert_np, event); 838 839 switch (event) { 840 case RDMA_CM_EVENT_DEVICE_REMOVAL: 841 isert_np->np_cm_id = NULL; 842 break; 843 case RDMA_CM_EVENT_ADDR_CHANGE: 844 isert_np->np_cm_id = isert_setup_id(isert_np); 845 if (IS_ERR(isert_np->np_cm_id)) { 846 isert_err("isert np %p setup id failed: %ld\n", 847 isert_np, PTR_ERR(isert_np->np_cm_id)); 848 isert_np->np_cm_id = NULL; 849 } 850 break; 851 default: 852 isert_err("isert np %p Unexpected event %d\n", 853 isert_np, event); 854 } 855 856 return -1; 857 } 858 859 static int 860 isert_disconnected_handler(struct rdma_cm_id *cma_id, 861 enum rdma_cm_event_type event) 862 { 863 struct isert_np *isert_np = cma_id->context; 864 struct isert_conn *isert_conn; 865 866 if (isert_np->np_cm_id == cma_id) 867 return isert_np_cma_handler(cma_id->context, event); 868 869 isert_conn = cma_id->qp->qp_context; 870 871 mutex_lock(&isert_conn->conn_mutex); 872 isert_conn_terminate(isert_conn); 873 mutex_unlock(&isert_conn->conn_mutex); 874 875 isert_info("conn %p completing conn_wait\n", isert_conn); 876 complete(&isert_conn->conn_wait); 877 878 return 0; 879 } 880 881 static void 882 isert_connect_error(struct rdma_cm_id *cma_id) 883 { 884 struct isert_conn *isert_conn = cma_id->qp->qp_context; 885 886 isert_put_conn(isert_conn); 887 } 888 889 static int 890 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 891 { 892 int ret = 0; 893 894 isert_info("event %d status %d id %p np %p\n", event->event, 895 event->status, cma_id, cma_id->context); 896 897 switch (event->event) { 898 case RDMA_CM_EVENT_CONNECT_REQUEST: 899 ret = isert_connect_request(cma_id, event); 900 if (ret) 901 isert_err("failed handle connect request %d\n", ret); 902 break; 903 case RDMA_CM_EVENT_ESTABLISHED: 904 isert_connected_handler(cma_id); 905 break; 906 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 907 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 908 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 909 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 910 ret = isert_disconnected_handler(cma_id, event->event); 911 break; 912 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 913 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 914 case RDMA_CM_EVENT_CONNECT_ERROR: 915 isert_connect_error(cma_id); 916 break; 917 default: 918 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 919 break; 920 } 921 922 return ret; 923 } 924 925 static int 926 isert_post_recv(struct isert_conn *isert_conn, u32 count) 927 { 928 struct ib_recv_wr *rx_wr, *rx_wr_failed; 929 int i, ret; 930 unsigned int rx_head = isert_conn->conn_rx_desc_head; 931 struct iser_rx_desc *rx_desc; 932 933 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 934 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 935 rx_wr->wr_id = (uintptr_t)rx_desc; 936 rx_wr->sg_list = &rx_desc->rx_sg; 937 rx_wr->num_sge = 1; 938 rx_wr->next = rx_wr + 1; 939 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 940 } 941 942 rx_wr--; 943 rx_wr->next = NULL; /* mark end of work requests list */ 944 945 isert_conn->post_recv_buf_count += count; 946 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 947 &rx_wr_failed); 948 if (ret) { 949 isert_err("ib_post_recv() failed with ret: %d\n", ret); 950 isert_conn->post_recv_buf_count -= count; 951 } else { 952 isert_dbg("isert_post_recv(): Posted %d RX buffers\n", count); 953 isert_conn->conn_rx_desc_head = rx_head; 954 } 955 return ret; 956 } 957 958 static int 959 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 960 { 961 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 962 struct ib_send_wr send_wr, *send_wr_failed; 963 int ret; 964 965 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 966 ISER_HEADERS_LEN, DMA_TO_DEVICE); 967 968 send_wr.next = NULL; 969 send_wr.wr_id = (uintptr_t)tx_desc; 970 send_wr.sg_list = tx_desc->tx_sg; 971 send_wr.num_sge = tx_desc->num_sge; 972 send_wr.opcode = IB_WR_SEND; 973 send_wr.send_flags = IB_SEND_SIGNALED; 974 975 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 976 if (ret) 977 isert_err("ib_post_send() failed, ret: %d\n", ret); 978 979 return ret; 980 } 981 982 static void 983 isert_create_send_desc(struct isert_conn *isert_conn, 984 struct isert_cmd *isert_cmd, 985 struct iser_tx_desc *tx_desc) 986 { 987 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 988 989 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 990 ISER_HEADERS_LEN, DMA_TO_DEVICE); 991 992 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 993 tx_desc->iser_header.flags = ISER_VER; 994 995 tx_desc->num_sge = 1; 996 tx_desc->isert_cmd = isert_cmd; 997 998 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 999 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 1000 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1001 } 1002 } 1003 1004 static int 1005 isert_init_tx_hdrs(struct isert_conn *isert_conn, 1006 struct iser_tx_desc *tx_desc) 1007 { 1008 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1009 u64 dma_addr; 1010 1011 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1012 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1013 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1014 isert_err("ib_dma_mapping_error() failed\n"); 1015 return -ENOMEM; 1016 } 1017 1018 tx_desc->dma_addr = dma_addr; 1019 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1020 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1021 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 1022 1023 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 1024 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 1025 tx_desc->tx_sg[0].lkey); 1026 1027 return 0; 1028 } 1029 1030 static void 1031 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1032 struct ib_send_wr *send_wr) 1033 { 1034 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 1035 1036 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1037 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1038 send_wr->opcode = IB_WR_SEND; 1039 send_wr->sg_list = &tx_desc->tx_sg[0]; 1040 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 1041 send_wr->send_flags = IB_SEND_SIGNALED; 1042 } 1043 1044 static int 1045 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1046 { 1047 struct ib_recv_wr rx_wr, *rx_wr_fail; 1048 struct ib_sge sge; 1049 int ret; 1050 1051 memset(&sge, 0, sizeof(struct ib_sge)); 1052 sge.addr = isert_conn->login_req_dma; 1053 sge.length = ISER_RX_LOGIN_SIZE; 1054 sge.lkey = isert_conn->conn_mr->lkey; 1055 1056 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1057 sge.addr, sge.length, sge.lkey); 1058 1059 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1060 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1061 rx_wr.sg_list = &sge; 1062 rx_wr.num_sge = 1; 1063 1064 isert_conn->post_recv_buf_count++; 1065 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 1066 if (ret) { 1067 isert_err("ib_post_recv() failed: %d\n", ret); 1068 isert_conn->post_recv_buf_count--; 1069 } 1070 1071 return ret; 1072 } 1073 1074 static int 1075 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1076 u32 length) 1077 { 1078 struct isert_conn *isert_conn = conn->context; 1079 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1080 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 1081 int ret; 1082 1083 isert_create_send_desc(isert_conn, NULL, tx_desc); 1084 1085 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1086 sizeof(struct iscsi_hdr)); 1087 1088 isert_init_tx_hdrs(isert_conn, tx_desc); 1089 1090 if (length > 0) { 1091 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1092 1093 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1094 length, DMA_TO_DEVICE); 1095 1096 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1097 1098 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1099 length, DMA_TO_DEVICE); 1100 1101 tx_dsg->addr = isert_conn->login_rsp_dma; 1102 tx_dsg->length = length; 1103 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1104 tx_desc->num_sge = 2; 1105 } 1106 if (!login->login_failed) { 1107 if (login->login_complete) { 1108 if (!conn->sess->sess_ops->SessionType && 1109 isert_conn->conn_device->use_fastreg) { 1110 ret = isert_conn_create_fastreg_pool(isert_conn); 1111 if (ret) { 1112 isert_err("Conn: %p failed to create" 1113 " fastreg pool\n", isert_conn); 1114 return ret; 1115 } 1116 } 1117 1118 ret = isert_alloc_rx_descriptors(isert_conn); 1119 if (ret) 1120 return ret; 1121 1122 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1123 if (ret) 1124 return ret; 1125 1126 /* Now we are in FULL_FEATURE phase */ 1127 mutex_lock(&isert_conn->conn_mutex); 1128 isert_conn->state = ISER_CONN_FULL_FEATURE; 1129 mutex_unlock(&isert_conn->conn_mutex); 1130 goto post_send; 1131 } 1132 1133 ret = isert_rdma_post_recvl(isert_conn); 1134 if (ret) 1135 return ret; 1136 } 1137 post_send: 1138 ret = isert_post_send(isert_conn, tx_desc); 1139 if (ret) 1140 return ret; 1141 1142 return 0; 1143 } 1144 1145 static void 1146 isert_rx_login_req(struct isert_conn *isert_conn) 1147 { 1148 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 1149 int rx_buflen = isert_conn->login_req_len; 1150 struct iscsi_conn *conn = isert_conn->conn; 1151 struct iscsi_login *login = conn->conn_login; 1152 int size; 1153 1154 isert_info("conn %p\n", isert_conn); 1155 1156 WARN_ON_ONCE(!login); 1157 1158 if (login->first_request) { 1159 struct iscsi_login_req *login_req = 1160 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1161 /* 1162 * Setup the initial iscsi_login values from the leading 1163 * login request PDU. 1164 */ 1165 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1166 login->current_stage = 1167 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1168 >> 2; 1169 login->version_min = login_req->min_version; 1170 login->version_max = login_req->max_version; 1171 memcpy(login->isid, login_req->isid, 6); 1172 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1173 login->init_task_tag = login_req->itt; 1174 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1175 login->cid = be16_to_cpu(login_req->cid); 1176 login->tsih = be16_to_cpu(login_req->tsih); 1177 } 1178 1179 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1180 1181 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1182 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1183 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1184 MAX_KEY_VALUE_PAIRS); 1185 memcpy(login->req_buf, &rx_desc->data[0], size); 1186 1187 if (login->first_request) { 1188 complete(&isert_conn->conn_login_comp); 1189 return; 1190 } 1191 schedule_delayed_work(&conn->login_work, 0); 1192 } 1193 1194 static struct iscsi_cmd 1195 *isert_allocate_cmd(struct iscsi_conn *conn) 1196 { 1197 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1198 struct isert_cmd *isert_cmd; 1199 struct iscsi_cmd *cmd; 1200 1201 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1202 if (!cmd) { 1203 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1204 return NULL; 1205 } 1206 isert_cmd = iscsit_priv_cmd(cmd); 1207 isert_cmd->conn = isert_conn; 1208 isert_cmd->iscsi_cmd = cmd; 1209 1210 return cmd; 1211 } 1212 1213 static int 1214 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1215 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1216 struct iser_rx_desc *rx_desc, unsigned char *buf) 1217 { 1218 struct iscsi_conn *conn = isert_conn->conn; 1219 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1220 struct scatterlist *sg; 1221 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1222 bool dump_payload = false; 1223 1224 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1225 if (rc < 0) 1226 return rc; 1227 1228 imm_data = cmd->immediate_data; 1229 imm_data_len = cmd->first_burst_len; 1230 unsol_data = cmd->unsolicited_data; 1231 1232 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1233 if (rc < 0) { 1234 return 0; 1235 } else if (rc > 0) { 1236 dump_payload = true; 1237 goto sequence_cmd; 1238 } 1239 1240 if (!imm_data) 1241 return 0; 1242 1243 sg = &cmd->se_cmd.t_data_sg[0]; 1244 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1245 1246 isert_dbg("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1247 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1248 1249 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1250 1251 cmd->write_data_done += imm_data_len; 1252 1253 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1254 spin_lock_bh(&cmd->istate_lock); 1255 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1256 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1257 spin_unlock_bh(&cmd->istate_lock); 1258 } 1259 1260 sequence_cmd: 1261 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1262 1263 if (!rc && dump_payload == false && unsol_data) 1264 iscsit_set_unsoliticed_dataout(cmd); 1265 else if (dump_payload && imm_data) 1266 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); 1267 1268 return 0; 1269 } 1270 1271 static int 1272 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1273 struct iser_rx_desc *rx_desc, unsigned char *buf) 1274 { 1275 struct scatterlist *sg_start; 1276 struct iscsi_conn *conn = isert_conn->conn; 1277 struct iscsi_cmd *cmd = NULL; 1278 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1279 u32 unsol_data_len = ntoh24(hdr->dlength); 1280 int rc, sg_nents, sg_off, page_off; 1281 1282 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1283 if (rc < 0) 1284 return rc; 1285 else if (!cmd) 1286 return 0; 1287 /* 1288 * FIXME: Unexpected unsolicited_data out 1289 */ 1290 if (!cmd->unsolicited_data) { 1291 isert_err("Received unexpected solicited data payload\n"); 1292 dump_stack(); 1293 return -1; 1294 } 1295 1296 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1297 "write_data_done: %u, data_length: %u\n", 1298 unsol_data_len, cmd->write_data_done, 1299 cmd->se_cmd.data_length); 1300 1301 sg_off = cmd->write_data_done / PAGE_SIZE; 1302 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1303 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1304 page_off = cmd->write_data_done % PAGE_SIZE; 1305 /* 1306 * FIXME: Non page-aligned unsolicited_data out 1307 */ 1308 if (page_off) { 1309 isert_err("unexpected non-page aligned data payload\n"); 1310 dump_stack(); 1311 return -1; 1312 } 1313 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1314 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1315 sg_nents, &rx_desc->data[0], unsol_data_len); 1316 1317 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1318 unsol_data_len); 1319 1320 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1321 if (rc < 0) 1322 return rc; 1323 1324 return 0; 1325 } 1326 1327 static int 1328 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1329 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1330 unsigned char *buf) 1331 { 1332 struct iscsi_conn *conn = isert_conn->conn; 1333 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1334 int rc; 1335 1336 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1337 if (rc < 0) 1338 return rc; 1339 /* 1340 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1341 */ 1342 1343 return iscsit_process_nop_out(conn, cmd, hdr); 1344 } 1345 1346 static int 1347 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1348 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1349 struct iscsi_text *hdr) 1350 { 1351 struct iscsi_conn *conn = isert_conn->conn; 1352 u32 payload_length = ntoh24(hdr->dlength); 1353 int rc; 1354 unsigned char *text_in; 1355 1356 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1357 if (rc < 0) 1358 return rc; 1359 1360 text_in = kzalloc(payload_length, GFP_KERNEL); 1361 if (!text_in) { 1362 isert_err("Unable to allocate text_in of payload_length: %u\n", 1363 payload_length); 1364 return -ENOMEM; 1365 } 1366 cmd->text_in_ptr = text_in; 1367 1368 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1369 1370 return iscsit_process_text_cmd(conn, cmd, hdr); 1371 } 1372 1373 static int 1374 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1375 uint32_t read_stag, uint64_t read_va, 1376 uint32_t write_stag, uint64_t write_va) 1377 { 1378 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1379 struct iscsi_conn *conn = isert_conn->conn; 1380 struct iscsi_session *sess = conn->sess; 1381 struct iscsi_cmd *cmd; 1382 struct isert_cmd *isert_cmd; 1383 int ret = -EINVAL; 1384 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1385 1386 if (sess->sess_ops->SessionType && 1387 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1388 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1389 " ignoring\n", opcode); 1390 return 0; 1391 } 1392 1393 switch (opcode) { 1394 case ISCSI_OP_SCSI_CMD: 1395 cmd = isert_allocate_cmd(conn); 1396 if (!cmd) 1397 break; 1398 1399 isert_cmd = iscsit_priv_cmd(cmd); 1400 isert_cmd->read_stag = read_stag; 1401 isert_cmd->read_va = read_va; 1402 isert_cmd->write_stag = write_stag; 1403 isert_cmd->write_va = write_va; 1404 1405 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1406 rx_desc, (unsigned char *)hdr); 1407 break; 1408 case ISCSI_OP_NOOP_OUT: 1409 cmd = isert_allocate_cmd(conn); 1410 if (!cmd) 1411 break; 1412 1413 isert_cmd = iscsit_priv_cmd(cmd); 1414 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1415 rx_desc, (unsigned char *)hdr); 1416 break; 1417 case ISCSI_OP_SCSI_DATA_OUT: 1418 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1419 (unsigned char *)hdr); 1420 break; 1421 case ISCSI_OP_SCSI_TMFUNC: 1422 cmd = isert_allocate_cmd(conn); 1423 if (!cmd) 1424 break; 1425 1426 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1427 (unsigned char *)hdr); 1428 break; 1429 case ISCSI_OP_LOGOUT: 1430 cmd = isert_allocate_cmd(conn); 1431 if (!cmd) 1432 break; 1433 1434 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1435 break; 1436 case ISCSI_OP_TEXT: 1437 cmd = isert_allocate_cmd(conn); 1438 if (!cmd) 1439 break; 1440 1441 isert_cmd = iscsit_priv_cmd(cmd); 1442 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1443 rx_desc, (struct iscsi_text *)hdr); 1444 break; 1445 default: 1446 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1447 dump_stack(); 1448 break; 1449 } 1450 1451 return ret; 1452 } 1453 1454 static void 1455 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1456 { 1457 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1458 uint64_t read_va = 0, write_va = 0; 1459 uint32_t read_stag = 0, write_stag = 0; 1460 int rc; 1461 1462 switch (iser_hdr->flags & 0xF0) { 1463 case ISCSI_CTRL: 1464 if (iser_hdr->flags & ISER_RSV) { 1465 read_stag = be32_to_cpu(iser_hdr->read_stag); 1466 read_va = be64_to_cpu(iser_hdr->read_va); 1467 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1468 read_stag, (unsigned long long)read_va); 1469 } 1470 if (iser_hdr->flags & ISER_WSV) { 1471 write_stag = be32_to_cpu(iser_hdr->write_stag); 1472 write_va = be64_to_cpu(iser_hdr->write_va); 1473 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1474 write_stag, (unsigned long long)write_va); 1475 } 1476 1477 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1478 break; 1479 case ISER_HELLO: 1480 isert_err("iSER Hello message\n"); 1481 break; 1482 default: 1483 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1484 break; 1485 } 1486 1487 rc = isert_rx_opcode(isert_conn, rx_desc, 1488 read_stag, read_va, write_stag, write_va); 1489 } 1490 1491 static void 1492 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1493 u32 xfer_len) 1494 { 1495 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1496 struct iscsi_hdr *hdr; 1497 u64 rx_dma; 1498 int rx_buflen, outstanding; 1499 1500 if ((char *)desc == isert_conn->login_req_buf) { 1501 rx_dma = isert_conn->login_req_dma; 1502 rx_buflen = ISER_RX_LOGIN_SIZE; 1503 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1504 rx_dma, rx_buflen); 1505 } else { 1506 rx_dma = desc->dma_addr; 1507 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1508 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1509 rx_dma, rx_buflen); 1510 } 1511 1512 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1513 1514 hdr = &desc->iscsi_header; 1515 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1516 hdr->opcode, hdr->itt, hdr->flags, 1517 (int)(xfer_len - ISER_HEADERS_LEN)); 1518 1519 if ((char *)desc == isert_conn->login_req_buf) { 1520 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 1521 if (isert_conn->conn) { 1522 struct iscsi_login *login = isert_conn->conn->conn_login; 1523 1524 if (login && !login->first_request) 1525 isert_rx_login_req(isert_conn); 1526 } 1527 mutex_lock(&isert_conn->conn_mutex); 1528 complete(&isert_conn->login_req_comp); 1529 mutex_unlock(&isert_conn->conn_mutex); 1530 } else { 1531 isert_rx_do_work(desc, isert_conn); 1532 } 1533 1534 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1535 DMA_FROM_DEVICE); 1536 1537 isert_conn->post_recv_buf_count--; 1538 isert_dbg("Decremented post_recv_buf_count: %d\n", 1539 isert_conn->post_recv_buf_count); 1540 1541 if ((char *)desc == isert_conn->login_req_buf) 1542 return; 1543 1544 outstanding = isert_conn->post_recv_buf_count; 1545 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1546 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1547 ISERT_MIN_POSTED_RX); 1548 err = isert_post_recv(isert_conn, count); 1549 if (err) { 1550 isert_err("isert_post_recv() count: %d failed, %d\n", 1551 count, err); 1552 } 1553 } 1554 } 1555 1556 static int 1557 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1558 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1559 enum iser_ib_op_code op, struct isert_data_buf *data) 1560 { 1561 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1562 1563 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1564 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1565 1566 data->len = length - offset; 1567 data->offset = offset; 1568 data->sg_off = data->offset / PAGE_SIZE; 1569 1570 data->sg = &sg[data->sg_off]; 1571 data->nents = min_t(unsigned int, nents - data->sg_off, 1572 ISCSI_ISER_SG_TABLESIZE); 1573 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1574 PAGE_SIZE); 1575 1576 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1577 data->dma_dir); 1578 if (unlikely(!data->dma_nents)) { 1579 isert_err("Cmd: unable to dma map SGs %p\n", sg); 1580 return -EINVAL; 1581 } 1582 1583 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1584 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1585 1586 return 0; 1587 } 1588 1589 static void 1590 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1591 { 1592 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1593 1594 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1595 memset(data, 0, sizeof(*data)); 1596 } 1597 1598 1599 1600 static void 1601 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1602 { 1603 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1604 1605 isert_dbg("Cmd %p\n", isert_cmd); 1606 1607 if (wr->data.sg) { 1608 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1609 isert_unmap_data_buf(isert_conn, &wr->data); 1610 } 1611 1612 if (wr->send_wr) { 1613 isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1614 kfree(wr->send_wr); 1615 wr->send_wr = NULL; 1616 } 1617 1618 if (wr->ib_sge) { 1619 isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1620 kfree(wr->ib_sge); 1621 wr->ib_sge = NULL; 1622 } 1623 } 1624 1625 static void 1626 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1627 { 1628 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1629 LIST_HEAD(unmap_list); 1630 1631 isert_dbg("Cmd %p\n", isert_cmd); 1632 1633 if (wr->fr_desc) { 1634 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 1635 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1636 isert_unmap_data_buf(isert_conn, &wr->prot); 1637 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1638 } 1639 spin_lock_bh(&isert_conn->conn_lock); 1640 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 1641 spin_unlock_bh(&isert_conn->conn_lock); 1642 wr->fr_desc = NULL; 1643 } 1644 1645 if (wr->data.sg) { 1646 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1647 isert_unmap_data_buf(isert_conn, &wr->data); 1648 } 1649 1650 wr->ib_sge = NULL; 1651 wr->send_wr = NULL; 1652 } 1653 1654 static void 1655 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1656 { 1657 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1658 struct isert_conn *isert_conn = isert_cmd->conn; 1659 struct iscsi_conn *conn = isert_conn->conn; 1660 struct isert_device *device = isert_conn->conn_device; 1661 1662 isert_dbg("Cmd %p\n", isert_cmd); 1663 1664 switch (cmd->iscsi_opcode) { 1665 case ISCSI_OP_SCSI_CMD: 1666 spin_lock_bh(&conn->cmd_lock); 1667 if (!list_empty(&cmd->i_conn_node)) 1668 list_del_init(&cmd->i_conn_node); 1669 spin_unlock_bh(&conn->cmd_lock); 1670 1671 if (cmd->data_direction == DMA_TO_DEVICE) { 1672 iscsit_stop_dataout_timer(cmd); 1673 /* 1674 * Check for special case during comp_err where 1675 * WRITE_PENDING has been handed off from core, 1676 * but requires an extra target_put_sess_cmd() 1677 * before transport_generic_free_cmd() below. 1678 */ 1679 if (comp_err && 1680 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1681 struct se_cmd *se_cmd = &cmd->se_cmd; 1682 1683 target_put_sess_cmd(se_cmd->se_sess, se_cmd); 1684 } 1685 } 1686 1687 device->unreg_rdma_mem(isert_cmd, isert_conn); 1688 transport_generic_free_cmd(&cmd->se_cmd, 0); 1689 break; 1690 case ISCSI_OP_SCSI_TMFUNC: 1691 spin_lock_bh(&conn->cmd_lock); 1692 if (!list_empty(&cmd->i_conn_node)) 1693 list_del_init(&cmd->i_conn_node); 1694 spin_unlock_bh(&conn->cmd_lock); 1695 1696 transport_generic_free_cmd(&cmd->se_cmd, 0); 1697 break; 1698 case ISCSI_OP_REJECT: 1699 case ISCSI_OP_NOOP_OUT: 1700 case ISCSI_OP_TEXT: 1701 spin_lock_bh(&conn->cmd_lock); 1702 if (!list_empty(&cmd->i_conn_node)) 1703 list_del_init(&cmd->i_conn_node); 1704 spin_unlock_bh(&conn->cmd_lock); 1705 1706 /* 1707 * Handle special case for REJECT when iscsi_add_reject*() has 1708 * overwritten the original iscsi_opcode assignment, and the 1709 * associated cmd->se_cmd needs to be released. 1710 */ 1711 if (cmd->se_cmd.se_tfo != NULL) { 1712 isert_dbg("Calling transport_generic_free_cmd from" 1713 " isert_put_cmd for 0x%02x\n", 1714 cmd->iscsi_opcode); 1715 transport_generic_free_cmd(&cmd->se_cmd, 0); 1716 break; 1717 } 1718 /* 1719 * Fall-through 1720 */ 1721 default: 1722 iscsit_release_cmd(cmd); 1723 break; 1724 } 1725 } 1726 1727 static void 1728 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1729 { 1730 if (tx_desc->dma_addr != 0) { 1731 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1732 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1733 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1734 tx_desc->dma_addr = 0; 1735 } 1736 } 1737 1738 static void 1739 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1740 struct ib_device *ib_dev, bool comp_err) 1741 { 1742 if (isert_cmd->pdu_buf_dma != 0) { 1743 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1744 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1745 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1746 isert_cmd->pdu_buf_dma = 0; 1747 } 1748 1749 isert_unmap_tx_desc(tx_desc, ib_dev); 1750 isert_put_cmd(isert_cmd, comp_err); 1751 } 1752 1753 static int 1754 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1755 { 1756 struct ib_mr_status mr_status; 1757 int ret; 1758 1759 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1760 if (ret) { 1761 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1762 goto fail_mr_status; 1763 } 1764 1765 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1766 u64 sec_offset_err; 1767 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1768 1769 switch (mr_status.sig_err.err_type) { 1770 case IB_SIG_BAD_GUARD: 1771 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1772 break; 1773 case IB_SIG_BAD_REFTAG: 1774 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1775 break; 1776 case IB_SIG_BAD_APPTAG: 1777 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1778 break; 1779 } 1780 sec_offset_err = mr_status.sig_err.sig_err_offset; 1781 do_div(sec_offset_err, block_size); 1782 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1783 1784 isert_err("PI error found type %d at sector 0x%llx " 1785 "expected 0x%x vs actual 0x%x\n", 1786 mr_status.sig_err.err_type, 1787 (unsigned long long)se_cmd->bad_sector, 1788 mr_status.sig_err.expected, 1789 mr_status.sig_err.actual); 1790 ret = 1; 1791 } 1792 1793 fail_mr_status: 1794 return ret; 1795 } 1796 1797 static void 1798 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1799 struct isert_cmd *isert_cmd) 1800 { 1801 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1802 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1803 struct se_cmd *se_cmd = &cmd->se_cmd; 1804 struct isert_conn *isert_conn = isert_cmd->conn; 1805 struct isert_device *device = isert_conn->conn_device; 1806 int ret = 0; 1807 1808 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1809 ret = isert_check_pi_status(se_cmd, 1810 wr->fr_desc->pi_ctx->sig_mr); 1811 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1812 } 1813 1814 device->unreg_rdma_mem(isert_cmd, isert_conn); 1815 wr->send_wr_num = 0; 1816 if (ret) 1817 transport_send_check_condition_and_sense(se_cmd, 1818 se_cmd->pi_err, 0); 1819 else 1820 isert_put_response(isert_conn->conn, cmd); 1821 } 1822 1823 static void 1824 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1825 struct isert_cmd *isert_cmd) 1826 { 1827 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1828 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1829 struct se_cmd *se_cmd = &cmd->se_cmd; 1830 struct isert_conn *isert_conn = isert_cmd->conn; 1831 struct isert_device *device = isert_conn->conn_device; 1832 int ret = 0; 1833 1834 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1835 ret = isert_check_pi_status(se_cmd, 1836 wr->fr_desc->pi_ctx->sig_mr); 1837 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1838 } 1839 1840 iscsit_stop_dataout_timer(cmd); 1841 device->unreg_rdma_mem(isert_cmd, isert_conn); 1842 cmd->write_data_done = wr->data.len; 1843 wr->send_wr_num = 0; 1844 1845 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1846 spin_lock_bh(&cmd->istate_lock); 1847 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1848 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1849 spin_unlock_bh(&cmd->istate_lock); 1850 1851 if (ret) 1852 transport_send_check_condition_and_sense(se_cmd, 1853 se_cmd->pi_err, 0); 1854 else 1855 target_execute_cmd(se_cmd); 1856 } 1857 1858 static void 1859 isert_do_control_comp(struct work_struct *work) 1860 { 1861 struct isert_cmd *isert_cmd = container_of(work, 1862 struct isert_cmd, comp_work); 1863 struct isert_conn *isert_conn = isert_cmd->conn; 1864 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1865 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1866 1867 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1868 1869 switch (cmd->i_state) { 1870 case ISTATE_SEND_TASKMGTRSP: 1871 iscsit_tmr_post_handler(cmd, cmd->conn); 1872 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1873 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1874 cmd->i_state = ISTATE_SENT_STATUS; 1875 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1876 ib_dev, false); 1877 break; 1878 case ISTATE_SEND_LOGOUTRSP: 1879 iscsit_logout_post_handler(cmd, cmd->conn); 1880 break; 1881 default: 1882 isert_err("Unknown i_state %d\n", cmd->i_state); 1883 dump_stack(); 1884 break; 1885 } 1886 } 1887 1888 static void 1889 isert_response_completion(struct iser_tx_desc *tx_desc, 1890 struct isert_cmd *isert_cmd, 1891 struct isert_conn *isert_conn, 1892 struct ib_device *ib_dev) 1893 { 1894 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1895 1896 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1897 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1898 cmd->i_state == ISTATE_SEND_REJECT || 1899 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1900 isert_unmap_tx_desc(tx_desc, ib_dev); 1901 1902 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1903 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1904 return; 1905 } 1906 1907 cmd->i_state = ISTATE_SENT_STATUS; 1908 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1909 } 1910 1911 static void 1912 isert_send_completion(struct iser_tx_desc *tx_desc, 1913 struct isert_conn *isert_conn) 1914 { 1915 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1916 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1917 struct isert_rdma_wr *wr; 1918 1919 if (!isert_cmd) { 1920 isert_unmap_tx_desc(tx_desc, ib_dev); 1921 return; 1922 } 1923 wr = &isert_cmd->rdma_wr; 1924 1925 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 1926 1927 switch (wr->iser_ib_op) { 1928 case ISER_IB_RECV: 1929 isert_err("Got ISER_IB_RECV\n"); 1930 dump_stack(); 1931 break; 1932 case ISER_IB_SEND: 1933 isert_response_completion(tx_desc, isert_cmd, 1934 isert_conn, ib_dev); 1935 break; 1936 case ISER_IB_RDMA_WRITE: 1937 isert_completion_rdma_write(tx_desc, isert_cmd); 1938 break; 1939 case ISER_IB_RDMA_READ: 1940 isert_completion_rdma_read(tx_desc, isert_cmd); 1941 break; 1942 default: 1943 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 1944 dump_stack(); 1945 break; 1946 } 1947 } 1948 1949 /** 1950 * is_isert_tx_desc() - Indicate if the completion wr_id 1951 * is a TX descriptor or not. 1952 * @isert_conn: iser connection 1953 * @wr_id: completion WR identifier 1954 * 1955 * Since we cannot rely on wc opcode in FLUSH errors 1956 * we must work around it by checking if the wr_id address 1957 * falls in the iser connection rx_descs buffer. If so 1958 * it is an RX descriptor, otherwize it is a TX. 1959 */ 1960 static inline bool 1961 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 1962 { 1963 void *start = isert_conn->conn_rx_descs; 1964 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->conn_rx_descs); 1965 1966 if (wr_id >= start && wr_id < start + len) 1967 return false; 1968 1969 return true; 1970 } 1971 1972 static void 1973 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 1974 { 1975 if (wc->wr_id == ISER_BEACON_WRID) { 1976 isert_info("conn %p completing conn_wait_comp_err\n", 1977 isert_conn); 1978 complete(&isert_conn->conn_wait_comp_err); 1979 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 1980 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1981 struct isert_cmd *isert_cmd; 1982 struct iser_tx_desc *desc; 1983 1984 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 1985 isert_cmd = desc->isert_cmd; 1986 if (!isert_cmd) 1987 isert_unmap_tx_desc(desc, ib_dev); 1988 else 1989 isert_completion_put(desc, isert_cmd, ib_dev, true); 1990 } else { 1991 isert_conn->post_recv_buf_count--; 1992 if (!isert_conn->post_recv_buf_count) 1993 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1994 } 1995 } 1996 1997 static void 1998 isert_handle_wc(struct ib_wc *wc) 1999 { 2000 struct isert_conn *isert_conn; 2001 struct iser_tx_desc *tx_desc; 2002 struct iser_rx_desc *rx_desc; 2003 2004 isert_conn = wc->qp->qp_context; 2005 if (likely(wc->status == IB_WC_SUCCESS)) { 2006 if (wc->opcode == IB_WC_RECV) { 2007 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 2008 isert_rx_completion(rx_desc, isert_conn, wc->byte_len); 2009 } else { 2010 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2011 isert_send_completion(tx_desc, isert_conn); 2012 } 2013 } else { 2014 if (wc->status != IB_WC_WR_FLUSH_ERR) 2015 isert_err("wr id %llx status %d vend_err %x\n", 2016 wc->wr_id, wc->status, wc->vendor_err); 2017 else 2018 isert_dbg("flush error: wr id %llx\n", wc->wr_id); 2019 2020 if (wc->wr_id != ISER_FASTREG_LI_WRID) 2021 isert_cq_comp_err(isert_conn, wc); 2022 } 2023 } 2024 2025 static void 2026 isert_cq_work(struct work_struct *work) 2027 { 2028 enum { isert_poll_budget = 65536 }; 2029 struct isert_comp *comp = container_of(work, struct isert_comp, 2030 work); 2031 struct ib_wc *const wcs = comp->wcs; 2032 int i, n, completed = 0; 2033 2034 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 2035 for (i = 0; i < n; i++) 2036 isert_handle_wc(&wcs[i]); 2037 2038 completed += n; 2039 if (completed >= isert_poll_budget) 2040 break; 2041 } 2042 2043 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2044 } 2045 2046 static void 2047 isert_cq_callback(struct ib_cq *cq, void *context) 2048 { 2049 struct isert_comp *comp = context; 2050 2051 queue_work(isert_comp_wq, &comp->work); 2052 } 2053 2054 static int 2055 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2056 { 2057 struct ib_send_wr *wr_failed; 2058 int ret; 2059 2060 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 2061 &wr_failed); 2062 if (ret) { 2063 isert_err("ib_post_send failed with %d\n", ret); 2064 return ret; 2065 } 2066 return ret; 2067 } 2068 2069 static int 2070 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2071 { 2072 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2073 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2074 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2075 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2076 &isert_cmd->tx_desc.iscsi_header; 2077 2078 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2079 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2080 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2081 /* 2082 * Attach SENSE DATA payload to iSCSI Response PDU 2083 */ 2084 if (cmd->se_cmd.sense_buffer && 2085 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2086 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2087 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2088 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2089 u32 padding, pdu_len; 2090 2091 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2092 cmd->sense_buffer); 2093 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2094 2095 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2096 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2097 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2098 2099 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2100 (void *)cmd->sense_buffer, pdu_len, 2101 DMA_TO_DEVICE); 2102 2103 isert_cmd->pdu_buf_len = pdu_len; 2104 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2105 tx_dsg->length = pdu_len; 2106 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2107 isert_cmd->tx_desc.num_sge = 2; 2108 } 2109 2110 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2111 2112 isert_dbg("Posting SCSI Response\n"); 2113 2114 return isert_post_response(isert_conn, isert_cmd); 2115 } 2116 2117 static void 2118 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2119 { 2120 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2121 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2122 struct isert_device *device = isert_conn->conn_device; 2123 2124 spin_lock_bh(&conn->cmd_lock); 2125 if (!list_empty(&cmd->i_conn_node)) 2126 list_del_init(&cmd->i_conn_node); 2127 spin_unlock_bh(&conn->cmd_lock); 2128 2129 if (cmd->data_direction == DMA_TO_DEVICE) 2130 iscsit_stop_dataout_timer(cmd); 2131 2132 device->unreg_rdma_mem(isert_cmd, isert_conn); 2133 } 2134 2135 static enum target_prot_op 2136 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2137 { 2138 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2139 struct isert_device *device = isert_conn->conn_device; 2140 2141 if (conn->tpg->tpg_attrib.t10_pi) { 2142 if (device->pi_capable) { 2143 isert_info("conn %p PI offload enabled\n", isert_conn); 2144 isert_conn->pi_support = true; 2145 return TARGET_PROT_ALL; 2146 } 2147 } 2148 2149 isert_info("conn %p PI offload disabled\n", isert_conn); 2150 isert_conn->pi_support = false; 2151 2152 return TARGET_PROT_NORMAL; 2153 } 2154 2155 static int 2156 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2157 bool nopout_response) 2158 { 2159 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2160 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2161 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2162 2163 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2164 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2165 &isert_cmd->tx_desc.iscsi_header, 2166 nopout_response); 2167 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2168 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2169 2170 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2171 2172 return isert_post_response(isert_conn, isert_cmd); 2173 } 2174 2175 static int 2176 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2177 { 2178 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2179 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2180 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2181 2182 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2183 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2184 &isert_cmd->tx_desc.iscsi_header); 2185 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2186 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2187 2188 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2189 2190 return isert_post_response(isert_conn, isert_cmd); 2191 } 2192 2193 static int 2194 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2195 { 2196 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2197 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2198 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2199 2200 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2201 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2202 &isert_cmd->tx_desc.iscsi_header); 2203 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2204 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2205 2206 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2207 2208 return isert_post_response(isert_conn, isert_cmd); 2209 } 2210 2211 static int 2212 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2213 { 2214 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2215 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2216 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2217 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2218 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2219 struct iscsi_reject *hdr = 2220 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2221 2222 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2223 iscsit_build_reject(cmd, conn, hdr); 2224 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2225 2226 hton24(hdr->dlength, ISCSI_HDR_LEN); 2227 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2228 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2229 DMA_TO_DEVICE); 2230 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2231 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2232 tx_dsg->length = ISCSI_HDR_LEN; 2233 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2234 isert_cmd->tx_desc.num_sge = 2; 2235 2236 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2237 2238 isert_dbg("conn %p Posting Reject\n", isert_conn); 2239 2240 return isert_post_response(isert_conn, isert_cmd); 2241 } 2242 2243 static int 2244 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2245 { 2246 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2247 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2248 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2249 struct iscsi_text_rsp *hdr = 2250 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2251 u32 txt_rsp_len; 2252 int rc; 2253 2254 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2255 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2256 if (rc < 0) 2257 return rc; 2258 2259 txt_rsp_len = rc; 2260 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2261 2262 if (txt_rsp_len) { 2263 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2264 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2265 void *txt_rsp_buf = cmd->buf_ptr; 2266 2267 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2268 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2269 2270 isert_cmd->pdu_buf_len = txt_rsp_len; 2271 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2272 tx_dsg->length = txt_rsp_len; 2273 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2274 isert_cmd->tx_desc.num_sge = 2; 2275 } 2276 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2277 2278 isert_dbg("conn %p Text Reject\n", isert_conn); 2279 2280 return isert_post_response(isert_conn, isert_cmd); 2281 } 2282 2283 static int 2284 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2285 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2286 u32 data_left, u32 offset) 2287 { 2288 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2289 struct scatterlist *sg_start, *tmp_sg; 2290 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2291 u32 sg_off, page_off; 2292 int i = 0, sg_nents; 2293 2294 sg_off = offset / PAGE_SIZE; 2295 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2296 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2297 page_off = offset % PAGE_SIZE; 2298 2299 send_wr->sg_list = ib_sge; 2300 send_wr->num_sge = sg_nents; 2301 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2302 /* 2303 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2304 */ 2305 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2306 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 2307 "page_off: %u\n", 2308 (unsigned long long)tmp_sg->dma_address, 2309 tmp_sg->length, page_off); 2310 2311 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2312 ib_sge->length = min_t(u32, data_left, 2313 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2314 ib_sge->lkey = isert_conn->conn_mr->lkey; 2315 2316 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 2317 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2318 page_off = 0; 2319 data_left -= ib_sge->length; 2320 ib_sge++; 2321 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2322 } 2323 2324 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2325 send_wr->sg_list, send_wr->num_sge); 2326 2327 return sg_nents; 2328 } 2329 2330 static int 2331 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2332 struct isert_rdma_wr *wr) 2333 { 2334 struct se_cmd *se_cmd = &cmd->se_cmd; 2335 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2336 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2337 struct isert_data_buf *data = &wr->data; 2338 struct ib_send_wr *send_wr; 2339 struct ib_sge *ib_sge; 2340 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2341 int ret = 0, i, ib_sge_cnt; 2342 2343 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2344 2345 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2346 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2347 se_cmd->t_data_nents, se_cmd->data_length, 2348 offset, wr->iser_ib_op, &wr->data); 2349 if (ret) 2350 return ret; 2351 2352 data_left = data->len; 2353 offset = data->offset; 2354 2355 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2356 if (!ib_sge) { 2357 isert_warn("Unable to allocate ib_sge\n"); 2358 ret = -ENOMEM; 2359 goto unmap_cmd; 2360 } 2361 wr->ib_sge = ib_sge; 2362 2363 wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2364 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2365 GFP_KERNEL); 2366 if (!wr->send_wr) { 2367 isert_dbg("Unable to allocate wr->send_wr\n"); 2368 ret = -ENOMEM; 2369 goto unmap_cmd; 2370 } 2371 2372 wr->isert_cmd = isert_cmd; 2373 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2374 2375 for (i = 0; i < wr->send_wr_num; i++) { 2376 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2377 data_len = min(data_left, rdma_write_max); 2378 2379 send_wr->send_flags = 0; 2380 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2381 send_wr->opcode = IB_WR_RDMA_WRITE; 2382 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2383 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2384 if (i + 1 == wr->send_wr_num) 2385 send_wr->next = &isert_cmd->tx_desc.send_wr; 2386 else 2387 send_wr->next = &wr->send_wr[i + 1]; 2388 } else { 2389 send_wr->opcode = IB_WR_RDMA_READ; 2390 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2391 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2392 if (i + 1 == wr->send_wr_num) 2393 send_wr->send_flags = IB_SEND_SIGNALED; 2394 else 2395 send_wr->next = &wr->send_wr[i + 1]; 2396 } 2397 2398 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2399 send_wr, data_len, offset); 2400 ib_sge += ib_sge_cnt; 2401 2402 offset += data_len; 2403 va_offset += data_len; 2404 data_left -= data_len; 2405 } 2406 2407 return 0; 2408 unmap_cmd: 2409 isert_unmap_data_buf(isert_conn, data); 2410 2411 return ret; 2412 } 2413 2414 static int 2415 isert_map_fr_pagelist(struct ib_device *ib_dev, 2416 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2417 { 2418 u64 start_addr, end_addr, page, chunk_start = 0; 2419 struct scatterlist *tmp_sg; 2420 int i = 0, new_chunk, last_ent, n_pages; 2421 2422 n_pages = 0; 2423 new_chunk = 1; 2424 last_ent = sg_nents - 1; 2425 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2426 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2427 if (new_chunk) 2428 chunk_start = start_addr; 2429 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2430 2431 isert_dbg("SGL[%d] dma_addr: 0x%llx len: %u\n", 2432 i, (unsigned long long)tmp_sg->dma_address, 2433 tmp_sg->length); 2434 2435 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2436 new_chunk = 0; 2437 continue; 2438 } 2439 new_chunk = 1; 2440 2441 page = chunk_start & PAGE_MASK; 2442 do { 2443 fr_pl[n_pages++] = page; 2444 isert_dbg("Mapped page_list[%d] page_addr: 0x%llx\n", 2445 n_pages - 1, page); 2446 page += PAGE_SIZE; 2447 } while (page < end_addr); 2448 } 2449 2450 return n_pages; 2451 } 2452 2453 static inline void 2454 isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 2455 { 2456 u32 rkey; 2457 2458 memset(inv_wr, 0, sizeof(*inv_wr)); 2459 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 2460 inv_wr->opcode = IB_WR_LOCAL_INV; 2461 inv_wr->ex.invalidate_rkey = mr->rkey; 2462 2463 /* Bump the key */ 2464 rkey = ib_inc_rkey(mr->rkey); 2465 ib_update_fast_reg_key(mr, rkey); 2466 } 2467 2468 static int 2469 isert_fast_reg_mr(struct isert_conn *isert_conn, 2470 struct fast_reg_descriptor *fr_desc, 2471 struct isert_data_buf *mem, 2472 enum isert_indicator ind, 2473 struct ib_sge *sge) 2474 { 2475 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2476 struct ib_mr *mr; 2477 struct ib_fast_reg_page_list *frpl; 2478 struct ib_send_wr fr_wr, inv_wr; 2479 struct ib_send_wr *bad_wr, *wr = NULL; 2480 int ret, pagelist_len; 2481 u32 page_off; 2482 2483 if (mem->dma_nents == 1) { 2484 sge->lkey = isert_conn->conn_mr->lkey; 2485 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2486 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2487 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2488 sge->addr, sge->length, sge->lkey); 2489 return 0; 2490 } 2491 2492 if (ind == ISERT_DATA_KEY_VALID) { 2493 /* Registering data buffer */ 2494 mr = fr_desc->data_mr; 2495 frpl = fr_desc->data_frpl; 2496 } else { 2497 /* Registering protection buffer */ 2498 mr = fr_desc->pi_ctx->prot_mr; 2499 frpl = fr_desc->pi_ctx->prot_frpl; 2500 } 2501 2502 page_off = mem->offset % PAGE_SIZE; 2503 2504 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2505 fr_desc, mem->nents, mem->offset); 2506 2507 pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 2508 &frpl->page_list[0]); 2509 2510 if (!(fr_desc->ind & ind)) { 2511 isert_inv_rkey(&inv_wr, mr); 2512 wr = &inv_wr; 2513 } 2514 2515 /* Prepare FASTREG WR */ 2516 memset(&fr_wr, 0, sizeof(fr_wr)); 2517 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2518 fr_wr.opcode = IB_WR_FAST_REG_MR; 2519 fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 2520 fr_wr.wr.fast_reg.page_list = frpl; 2521 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2522 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2523 fr_wr.wr.fast_reg.length = mem->len; 2524 fr_wr.wr.fast_reg.rkey = mr->rkey; 2525 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2526 2527 if (!wr) 2528 wr = &fr_wr; 2529 else 2530 wr->next = &fr_wr; 2531 2532 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2533 if (ret) { 2534 isert_err("fast registration failed, ret:%d\n", ret); 2535 return ret; 2536 } 2537 fr_desc->ind &= ~ind; 2538 2539 sge->lkey = mr->lkey; 2540 sge->addr = frpl->page_list[0] + page_off; 2541 sge->length = mem->len; 2542 2543 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2544 sge->addr, sge->length, sge->lkey); 2545 2546 return ret; 2547 } 2548 2549 static inline void 2550 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2551 struct ib_sig_domain *domain) 2552 { 2553 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2554 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2555 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2556 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2557 /* 2558 * At the moment we hard code those, but if in the future 2559 * the target core would like to use it, we will take it 2560 * from se_cmd. 2561 */ 2562 domain->sig.dif.apptag_check_mask = 0xffff; 2563 domain->sig.dif.app_escape = true; 2564 domain->sig.dif.ref_escape = true; 2565 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2566 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2567 domain->sig.dif.ref_remap = true; 2568 }; 2569 2570 static int 2571 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2572 { 2573 switch (se_cmd->prot_op) { 2574 case TARGET_PROT_DIN_INSERT: 2575 case TARGET_PROT_DOUT_STRIP: 2576 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2577 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2578 break; 2579 case TARGET_PROT_DOUT_INSERT: 2580 case TARGET_PROT_DIN_STRIP: 2581 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2582 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2583 break; 2584 case TARGET_PROT_DIN_PASS: 2585 case TARGET_PROT_DOUT_PASS: 2586 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2587 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2588 break; 2589 default: 2590 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2591 return -EINVAL; 2592 } 2593 2594 return 0; 2595 } 2596 2597 static inline u8 2598 isert_set_prot_checks(u8 prot_checks) 2599 { 2600 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2601 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2602 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2603 } 2604 2605 static int 2606 isert_reg_sig_mr(struct isert_conn *isert_conn, 2607 struct se_cmd *se_cmd, 2608 struct isert_rdma_wr *rdma_wr, 2609 struct fast_reg_descriptor *fr_desc) 2610 { 2611 struct ib_send_wr sig_wr, inv_wr; 2612 struct ib_send_wr *bad_wr, *wr = NULL; 2613 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2614 struct ib_sig_attrs sig_attrs; 2615 int ret; 2616 2617 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2618 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2619 if (ret) 2620 goto err; 2621 2622 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2623 2624 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2625 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 2626 wr = &inv_wr; 2627 } 2628 2629 memset(&sig_wr, 0, sizeof(sig_wr)); 2630 sig_wr.opcode = IB_WR_REG_SIG_MR; 2631 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2632 sig_wr.sg_list = &rdma_wr->ib_sg[DATA]; 2633 sig_wr.num_sge = 1; 2634 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 2635 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 2636 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 2637 if (se_cmd->t_prot_sg) 2638 sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT]; 2639 2640 if (!wr) 2641 wr = &sig_wr; 2642 else 2643 wr->next = &sig_wr; 2644 2645 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2646 if (ret) { 2647 isert_err("fast registration failed, ret:%d\n", ret); 2648 goto err; 2649 } 2650 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2651 2652 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2653 rdma_wr->ib_sg[SIG].addr = 0; 2654 rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 2655 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2656 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2657 /* 2658 * We have protection guards on the wire 2659 * so we need to set a larget transfer 2660 */ 2661 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 2662 2663 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2664 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2665 rdma_wr->ib_sg[SIG].lkey); 2666 err: 2667 return ret; 2668 } 2669 2670 static int 2671 isert_handle_prot_cmd(struct isert_conn *isert_conn, 2672 struct isert_cmd *isert_cmd, 2673 struct isert_rdma_wr *wr) 2674 { 2675 struct isert_device *device = isert_conn->conn_device; 2676 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2677 int ret; 2678 2679 if (!wr->fr_desc->pi_ctx) { 2680 ret = isert_create_pi_ctx(wr->fr_desc, 2681 device->ib_device, 2682 isert_conn->conn_pd); 2683 if (ret) { 2684 isert_err("conn %p failed to allocate pi_ctx\n", 2685 isert_conn); 2686 return ret; 2687 } 2688 } 2689 2690 if (se_cmd->t_prot_sg) { 2691 ret = isert_map_data_buf(isert_conn, isert_cmd, 2692 se_cmd->t_prot_sg, 2693 se_cmd->t_prot_nents, 2694 se_cmd->prot_length, 2695 0, wr->iser_ib_op, &wr->prot); 2696 if (ret) { 2697 isert_err("conn %p failed to map protection buffer\n", 2698 isert_conn); 2699 return ret; 2700 } 2701 2702 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2703 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2704 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2705 if (ret) { 2706 isert_err("conn %p failed to fast reg mr\n", 2707 isert_conn); 2708 goto unmap_prot_cmd; 2709 } 2710 } 2711 2712 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2713 if (ret) { 2714 isert_err("conn %p failed to fast reg mr\n", 2715 isert_conn); 2716 goto unmap_prot_cmd; 2717 } 2718 wr->fr_desc->ind |= ISERT_PROTECTED; 2719 2720 return 0; 2721 2722 unmap_prot_cmd: 2723 if (se_cmd->t_prot_sg) 2724 isert_unmap_data_buf(isert_conn, &wr->prot); 2725 2726 return ret; 2727 } 2728 2729 static int 2730 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2731 struct isert_rdma_wr *wr) 2732 { 2733 struct se_cmd *se_cmd = &cmd->se_cmd; 2734 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2735 struct isert_conn *isert_conn = conn->context; 2736 struct fast_reg_descriptor *fr_desc = NULL; 2737 struct ib_send_wr *send_wr; 2738 struct ib_sge *ib_sg; 2739 u32 offset; 2740 int ret = 0; 2741 unsigned long flags; 2742 2743 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2744 2745 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2746 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2747 se_cmd->t_data_nents, se_cmd->data_length, 2748 offset, wr->iser_ib_op, &wr->data); 2749 if (ret) 2750 return ret; 2751 2752 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2753 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2754 fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2755 struct fast_reg_descriptor, list); 2756 list_del(&fr_desc->list); 2757 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2758 wr->fr_desc = fr_desc; 2759 } 2760 2761 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2762 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2763 if (ret) 2764 goto unmap_cmd; 2765 2766 if (isert_prot_cmd(isert_conn, se_cmd)) { 2767 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 2768 if (ret) 2769 goto unmap_cmd; 2770 2771 ib_sg = &wr->ib_sg[SIG]; 2772 } else { 2773 ib_sg = &wr->ib_sg[DATA]; 2774 } 2775 2776 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2777 wr->ib_sge = &wr->s_ib_sge; 2778 wr->send_wr_num = 1; 2779 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2780 wr->send_wr = &wr->s_send_wr; 2781 wr->isert_cmd = isert_cmd; 2782 2783 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2784 send_wr->sg_list = &wr->s_ib_sge; 2785 send_wr->num_sge = 1; 2786 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2787 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2788 send_wr->opcode = IB_WR_RDMA_WRITE; 2789 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2790 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2791 send_wr->send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 2792 0 : IB_SEND_SIGNALED; 2793 } else { 2794 send_wr->opcode = IB_WR_RDMA_READ; 2795 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2796 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2797 send_wr->send_flags = IB_SEND_SIGNALED; 2798 } 2799 2800 return 0; 2801 2802 unmap_cmd: 2803 if (fr_desc) { 2804 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2805 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2806 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2807 } 2808 isert_unmap_data_buf(isert_conn, &wr->data); 2809 2810 return ret; 2811 } 2812 2813 static int 2814 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2815 { 2816 struct se_cmd *se_cmd = &cmd->se_cmd; 2817 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2818 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2819 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2820 struct isert_device *device = isert_conn->conn_device; 2821 struct ib_send_wr *wr_failed; 2822 int rc; 2823 2824 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2825 isert_cmd, se_cmd->data_length); 2826 2827 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2828 rc = device->reg_rdma_mem(conn, cmd, wr); 2829 if (rc) { 2830 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2831 return rc; 2832 } 2833 2834 if (!isert_prot_cmd(isert_conn, se_cmd)) { 2835 /* 2836 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2837 */ 2838 isert_create_send_desc(isert_conn, isert_cmd, 2839 &isert_cmd->tx_desc); 2840 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2841 &isert_cmd->tx_desc.iscsi_header); 2842 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2843 isert_init_send_wr(isert_conn, isert_cmd, 2844 &isert_cmd->tx_desc.send_wr); 2845 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2846 wr->send_wr_num += 1; 2847 } 2848 2849 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2850 if (rc) 2851 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2852 2853 if (!isert_prot_cmd(isert_conn, se_cmd)) 2854 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2855 "READ\n", isert_cmd); 2856 else 2857 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2858 isert_cmd); 2859 2860 return 1; 2861 } 2862 2863 static int 2864 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2865 { 2866 struct se_cmd *se_cmd = &cmd->se_cmd; 2867 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2868 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2869 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2870 struct isert_device *device = isert_conn->conn_device; 2871 struct ib_send_wr *wr_failed; 2872 int rc; 2873 2874 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2875 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2876 wr->iser_ib_op = ISER_IB_RDMA_READ; 2877 rc = device->reg_rdma_mem(conn, cmd, wr); 2878 if (rc) { 2879 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2880 return rc; 2881 } 2882 2883 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2884 if (rc) 2885 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2886 2887 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2888 isert_cmd); 2889 2890 return 0; 2891 } 2892 2893 static int 2894 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2895 { 2896 int ret; 2897 2898 switch (state) { 2899 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2900 ret = isert_put_nopin(cmd, conn, false); 2901 break; 2902 default: 2903 isert_err("Unknown immediate state: 0x%02x\n", state); 2904 ret = -EINVAL; 2905 break; 2906 } 2907 2908 return ret; 2909 } 2910 2911 static int 2912 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2913 { 2914 struct isert_conn *isert_conn = conn->context; 2915 int ret; 2916 2917 switch (state) { 2918 case ISTATE_SEND_LOGOUTRSP: 2919 ret = isert_put_logout_rsp(cmd, conn); 2920 if (!ret) 2921 isert_conn->logout_posted = true; 2922 break; 2923 case ISTATE_SEND_NOPIN: 2924 ret = isert_put_nopin(cmd, conn, true); 2925 break; 2926 case ISTATE_SEND_TASKMGTRSP: 2927 ret = isert_put_tm_rsp(cmd, conn); 2928 break; 2929 case ISTATE_SEND_REJECT: 2930 ret = isert_put_reject(cmd, conn); 2931 break; 2932 case ISTATE_SEND_TEXTRSP: 2933 ret = isert_put_text_rsp(cmd, conn); 2934 break; 2935 case ISTATE_SEND_STATUS: 2936 /* 2937 * Special case for sending non GOOD SCSI status from TX thread 2938 * context during pre se_cmd excecution failure. 2939 */ 2940 ret = isert_put_response(conn, cmd); 2941 break; 2942 default: 2943 isert_err("Unknown response state: 0x%02x\n", state); 2944 ret = -EINVAL; 2945 break; 2946 } 2947 2948 return ret; 2949 } 2950 2951 struct rdma_cm_id * 2952 isert_setup_id(struct isert_np *isert_np) 2953 { 2954 struct iscsi_np *np = isert_np->np; 2955 struct rdma_cm_id *id; 2956 struct sockaddr *sa; 2957 int ret; 2958 2959 sa = (struct sockaddr *)&np->np_sockaddr; 2960 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2961 2962 id = rdma_create_id(isert_cma_handler, isert_np, 2963 RDMA_PS_TCP, IB_QPT_RC); 2964 if (IS_ERR(id)) { 2965 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2966 ret = PTR_ERR(id); 2967 goto out; 2968 } 2969 isert_dbg("id %p context %p\n", id, id->context); 2970 2971 ret = rdma_bind_addr(id, sa); 2972 if (ret) { 2973 isert_err("rdma_bind_addr() failed: %d\n", ret); 2974 goto out_id; 2975 } 2976 2977 ret = rdma_listen(id, ISERT_RDMA_LISTEN_BACKLOG); 2978 if (ret) { 2979 isert_err("rdma_listen() failed: %d\n", ret); 2980 goto out_id; 2981 } 2982 2983 return id; 2984 out_id: 2985 rdma_destroy_id(id); 2986 out: 2987 return ERR_PTR(ret); 2988 } 2989 2990 static int 2991 isert_setup_np(struct iscsi_np *np, 2992 struct __kernel_sockaddr_storage *ksockaddr) 2993 { 2994 struct isert_np *isert_np; 2995 struct rdma_cm_id *isert_lid; 2996 int ret; 2997 2998 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2999 if (!isert_np) { 3000 isert_err("Unable to allocate struct isert_np\n"); 3001 return -ENOMEM; 3002 } 3003 sema_init(&isert_np->np_sem, 0); 3004 mutex_init(&isert_np->np_accept_mutex); 3005 INIT_LIST_HEAD(&isert_np->np_accept_list); 3006 init_completion(&isert_np->np_login_comp); 3007 isert_np->np = np; 3008 3009 /* 3010 * Setup the np->np_sockaddr from the passed sockaddr setup 3011 * in iscsi_target_configfs.c code.. 3012 */ 3013 memcpy(&np->np_sockaddr, ksockaddr, 3014 sizeof(struct __kernel_sockaddr_storage)); 3015 3016 isert_lid = isert_setup_id(isert_np); 3017 if (IS_ERR(isert_lid)) { 3018 ret = PTR_ERR(isert_lid); 3019 goto out; 3020 } 3021 3022 isert_np->np_cm_id = isert_lid; 3023 np->np_context = isert_np; 3024 3025 return 0; 3026 3027 out: 3028 kfree(isert_np); 3029 3030 return ret; 3031 } 3032 3033 static int 3034 isert_rdma_accept(struct isert_conn *isert_conn) 3035 { 3036 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3037 struct rdma_conn_param cp; 3038 int ret; 3039 3040 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3041 cp.initiator_depth = isert_conn->initiator_depth; 3042 cp.retry_count = 7; 3043 cp.rnr_retry_count = 7; 3044 3045 ret = rdma_accept(cm_id, &cp); 3046 if (ret) { 3047 isert_err("rdma_accept() failed with: %d\n", ret); 3048 return ret; 3049 } 3050 3051 return 0; 3052 } 3053 3054 static int 3055 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3056 { 3057 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3058 int ret; 3059 3060 isert_info("before login_req comp conn: %p\n", isert_conn); 3061 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 3062 if (ret) { 3063 isert_err("isert_conn %p interrupted before got login req\n", 3064 isert_conn); 3065 return ret; 3066 } 3067 reinit_completion(&isert_conn->login_req_comp); 3068 3069 /* 3070 * For login requests after the first PDU, isert_rx_login_req() will 3071 * kick schedule_delayed_work(&conn->login_work) as the packet is 3072 * received, which turns this callback from iscsi_target_do_login_rx() 3073 * into a NOP. 3074 */ 3075 if (!login->first_request) 3076 return 0; 3077 3078 isert_rx_login_req(isert_conn); 3079 3080 isert_info("before conn_login_comp conn: %p\n", conn); 3081 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 3082 if (ret) 3083 return ret; 3084 3085 isert_info("processing login->req: %p\n", login->req); 3086 3087 return 0; 3088 } 3089 3090 static void 3091 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3092 struct isert_conn *isert_conn) 3093 { 3094 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3095 struct rdma_route *cm_route = &cm_id->route; 3096 struct sockaddr_in *sock_in; 3097 struct sockaddr_in6 *sock_in6; 3098 3099 conn->login_family = np->np_sockaddr.ss_family; 3100 3101 if (np->np_sockaddr.ss_family == AF_INET6) { 3102 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3103 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3104 &sock_in6->sin6_addr.in6_u); 3105 conn->login_port = ntohs(sock_in6->sin6_port); 3106 3107 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3108 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3109 &sock_in6->sin6_addr.in6_u); 3110 conn->local_port = ntohs(sock_in6->sin6_port); 3111 } else { 3112 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3113 sprintf(conn->login_ip, "%pI4", 3114 &sock_in->sin_addr.s_addr); 3115 conn->login_port = ntohs(sock_in->sin_port); 3116 3117 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3118 sprintf(conn->local_ip, "%pI4", 3119 &sock_in->sin_addr.s_addr); 3120 conn->local_port = ntohs(sock_in->sin_port); 3121 } 3122 } 3123 3124 static int 3125 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3126 { 3127 struct isert_np *isert_np = (struct isert_np *)np->np_context; 3128 struct isert_conn *isert_conn; 3129 int max_accept = 0, ret; 3130 3131 accept_wait: 3132 ret = down_interruptible(&isert_np->np_sem); 3133 if (ret || max_accept > 5) 3134 return -ENODEV; 3135 3136 spin_lock_bh(&np->np_thread_lock); 3137 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3138 spin_unlock_bh(&np->np_thread_lock); 3139 isert_dbg("np_thread_state %d for isert_accept_np\n", 3140 np->np_thread_state); 3141 /** 3142 * No point in stalling here when np_thread 3143 * is in state RESET/SHUTDOWN/EXIT - bail 3144 **/ 3145 return -ENODEV; 3146 } 3147 spin_unlock_bh(&np->np_thread_lock); 3148 3149 mutex_lock(&isert_np->np_accept_mutex); 3150 if (list_empty(&isert_np->np_accept_list)) { 3151 mutex_unlock(&isert_np->np_accept_mutex); 3152 max_accept++; 3153 goto accept_wait; 3154 } 3155 isert_conn = list_first_entry(&isert_np->np_accept_list, 3156 struct isert_conn, conn_accept_node); 3157 list_del_init(&isert_conn->conn_accept_node); 3158 mutex_unlock(&isert_np->np_accept_mutex); 3159 3160 conn->context = isert_conn; 3161 isert_conn->conn = conn; 3162 max_accept = 0; 3163 3164 isert_set_conn_info(np, conn, isert_conn); 3165 3166 isert_dbg("Processing isert_conn: %p\n", isert_conn); 3167 3168 return 0; 3169 } 3170 3171 static void 3172 isert_free_np(struct iscsi_np *np) 3173 { 3174 struct isert_np *isert_np = (struct isert_np *)np->np_context; 3175 struct isert_conn *isert_conn, *n; 3176 3177 if (isert_np->np_cm_id) 3178 rdma_destroy_id(isert_np->np_cm_id); 3179 3180 /* 3181 * FIXME: At this point we don't have a good way to insure 3182 * that at this point we don't have hanging connections that 3183 * completed RDMA establishment but didn't start iscsi login 3184 * process. So work-around this by cleaning up what ever piled 3185 * up in np_accept_list. 3186 */ 3187 mutex_lock(&isert_np->np_accept_mutex); 3188 if (!list_empty(&isert_np->np_accept_list)) { 3189 isert_info("Still have isert connections, cleaning up...\n"); 3190 list_for_each_entry_safe(isert_conn, n, 3191 &isert_np->np_accept_list, 3192 conn_accept_node) { 3193 isert_info("cleaning isert_conn %p state (%d)\n", 3194 isert_conn, isert_conn->state); 3195 isert_connect_release(isert_conn); 3196 } 3197 } 3198 mutex_unlock(&isert_np->np_accept_mutex); 3199 3200 np->np_context = NULL; 3201 kfree(isert_np); 3202 } 3203 3204 static void isert_release_work(struct work_struct *work) 3205 { 3206 struct isert_conn *isert_conn = container_of(work, 3207 struct isert_conn, 3208 release_work); 3209 3210 isert_info("Starting release conn %p\n", isert_conn); 3211 3212 wait_for_completion(&isert_conn->conn_wait); 3213 3214 mutex_lock(&isert_conn->conn_mutex); 3215 isert_conn->state = ISER_CONN_DOWN; 3216 mutex_unlock(&isert_conn->conn_mutex); 3217 3218 isert_info("Destroying conn %p\n", isert_conn); 3219 isert_put_conn(isert_conn); 3220 } 3221 3222 static void 3223 isert_wait4logout(struct isert_conn *isert_conn) 3224 { 3225 struct iscsi_conn *conn = isert_conn->conn; 3226 3227 isert_info("conn %p\n", isert_conn); 3228 3229 if (isert_conn->logout_posted) { 3230 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3231 wait_for_completion_timeout(&conn->conn_logout_comp, 3232 SECONDS_FOR_LOGOUT_COMP * HZ); 3233 } 3234 } 3235 3236 static void 3237 isert_wait4cmds(struct iscsi_conn *conn) 3238 { 3239 isert_info("iscsi_conn %p\n", conn); 3240 3241 if (conn->sess) { 3242 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3243 target_wait_for_sess_cmds(conn->sess->se_sess); 3244 } 3245 } 3246 3247 static void 3248 isert_wait4flush(struct isert_conn *isert_conn) 3249 { 3250 struct ib_recv_wr *bad_wr; 3251 3252 isert_info("conn %p\n", isert_conn); 3253 3254 init_completion(&isert_conn->conn_wait_comp_err); 3255 isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3256 /* post an indication that all flush errors were consumed */ 3257 if (ib_post_recv(isert_conn->conn_qp, &isert_conn->beacon, &bad_wr)) { 3258 isert_err("conn %p failed to post beacon", isert_conn); 3259 return; 3260 } 3261 3262 wait_for_completion(&isert_conn->conn_wait_comp_err); 3263 } 3264 3265 static void isert_wait_conn(struct iscsi_conn *conn) 3266 { 3267 struct isert_conn *isert_conn = conn->context; 3268 3269 isert_info("Starting conn %p\n", isert_conn); 3270 3271 mutex_lock(&isert_conn->conn_mutex); 3272 /* 3273 * Only wait for conn_wait_comp_err if the isert_conn made it 3274 * into full feature phase.. 3275 */ 3276 if (isert_conn->state == ISER_CONN_INIT) { 3277 mutex_unlock(&isert_conn->conn_mutex); 3278 return; 3279 } 3280 isert_conn_terminate(isert_conn); 3281 mutex_unlock(&isert_conn->conn_mutex); 3282 3283 isert_wait4cmds(conn); 3284 isert_wait4flush(isert_conn); 3285 isert_wait4logout(isert_conn); 3286 3287 INIT_WORK(&isert_conn->release_work, isert_release_work); 3288 queue_work(isert_release_wq, &isert_conn->release_work); 3289 } 3290 3291 static void isert_free_conn(struct iscsi_conn *conn) 3292 { 3293 struct isert_conn *isert_conn = conn->context; 3294 3295 isert_put_conn(isert_conn); 3296 } 3297 3298 static struct iscsit_transport iser_target_transport = { 3299 .name = "IB/iSER", 3300 .transport_type = ISCSI_INFINIBAND, 3301 .priv_size = sizeof(struct isert_cmd), 3302 .owner = THIS_MODULE, 3303 .iscsit_setup_np = isert_setup_np, 3304 .iscsit_accept_np = isert_accept_np, 3305 .iscsit_free_np = isert_free_np, 3306 .iscsit_wait_conn = isert_wait_conn, 3307 .iscsit_free_conn = isert_free_conn, 3308 .iscsit_get_login_rx = isert_get_login_rx, 3309 .iscsit_put_login_tx = isert_put_login_tx, 3310 .iscsit_immediate_queue = isert_immediate_queue, 3311 .iscsit_response_queue = isert_response_queue, 3312 .iscsit_get_dataout = isert_get_dataout, 3313 .iscsit_queue_data_in = isert_put_datain, 3314 .iscsit_queue_status = isert_put_response, 3315 .iscsit_aborted_task = isert_aborted_task, 3316 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3317 }; 3318 3319 static int __init isert_init(void) 3320 { 3321 int ret; 3322 3323 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 3324 if (!isert_comp_wq) { 3325 isert_err("Unable to allocate isert_comp_wq\n"); 3326 ret = -ENOMEM; 3327 return -ENOMEM; 3328 } 3329 3330 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3331 WQ_UNBOUND_MAX_ACTIVE); 3332 if (!isert_release_wq) { 3333 isert_err("Unable to allocate isert_release_wq\n"); 3334 ret = -ENOMEM; 3335 goto destroy_comp_wq; 3336 } 3337 3338 iscsit_register_transport(&iser_target_transport); 3339 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3340 3341 return 0; 3342 3343 destroy_comp_wq: 3344 destroy_workqueue(isert_comp_wq); 3345 3346 return ret; 3347 } 3348 3349 static void __exit isert_exit(void) 3350 { 3351 flush_scheduled_work(); 3352 destroy_workqueue(isert_release_wq); 3353 destroy_workqueue(isert_comp_wq); 3354 iscsit_unregister_transport(&iser_target_transport); 3355 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3356 } 3357 3358 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3359 MODULE_VERSION("0.1"); 3360 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3361 MODULE_LICENSE("GPL"); 3362 3363 module_init(isert_init); 3364 module_exit(isert_exit); 3365