1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains iSCSI extentions for RDMA (iSER) Verbs 4 * 5 * (c) Copyright 2013 Datera, Inc. 6 * 7 * Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ****************************************************************************/ 10 11 #include <linux/string.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/socket.h> 15 #include <linux/in.h> 16 #include <linux/in6.h> 17 #include <rdma/ib_verbs.h> 18 #include <rdma/ib_cm.h> 19 #include <rdma/rdma_cm.h> 20 #include <target/target_core_base.h> 21 #include <target/target_core_fabric.h> 22 #include <target/iscsi/iscsi_transport.h> 23 #include <linux/semaphore.h> 24 25 #include "ib_isert.h" 26 27 static int isert_debug_level; 28 module_param_named(debug_level, isert_debug_level, int, 0644); 29 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 30 31 static int isert_sg_tablesize_set(const char *val, 32 const struct kernel_param *kp); 33 static const struct kernel_param_ops sg_tablesize_ops = { 34 .set = isert_sg_tablesize_set, 35 .get = param_get_int, 36 }; 37 38 static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE; 39 module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644); 40 MODULE_PARM_DESC(sg_tablesize, 41 "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)"); 42 43 static DEFINE_MUTEX(device_list_mutex); 44 static LIST_HEAD(device_list); 45 static struct workqueue_struct *isert_login_wq; 46 static struct workqueue_struct *isert_comp_wq; 47 static struct workqueue_struct *isert_release_wq; 48 49 static int 50 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd); 51 static int 52 isert_login_post_recv(struct isert_conn *isert_conn); 53 static int 54 isert_rdma_accept(struct isert_conn *isert_conn); 55 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 56 57 static void isert_release_work(struct work_struct *work); 58 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 59 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 61 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 62 63 static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) 64 { 65 int n = 0, ret; 66 67 ret = kstrtoint(val, 10, &n); 68 if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE || 69 n > ISCSI_ISER_MAX_SG_TABLESIZE) 70 return -EINVAL; 71 72 return param_set_int(val, kp); 73 } 74 75 static inline bool 76 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 77 { 78 return (conn->pi_support && 79 cmd->prot_op != TARGET_PROT_NORMAL); 80 } 81 82 static void 83 isert_qp_event_callback(struct ib_event *e, void *context) 84 { 85 struct isert_conn *isert_conn = context; 86 87 isert_err("%s (%d): conn %p\n", 88 ib_event_msg(e->event), e->event, isert_conn); 89 90 switch (e->event) { 91 case IB_EVENT_COMM_EST: 92 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 93 break; 94 default: 95 break; 96 } 97 } 98 99 static struct ib_qp * 100 isert_create_qp(struct isert_conn *isert_conn, 101 struct rdma_cm_id *cma_id) 102 { 103 u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; 104 struct isert_device *device = isert_conn->device; 105 struct ib_device *ib_dev = device->ib_device; 106 struct ib_qp_init_attr attr; 107 int ret, factor; 108 109 isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); 110 if (IS_ERR(isert_conn->cq)) { 111 isert_err("Unable to allocate cq\n"); 112 ret = PTR_ERR(isert_conn->cq); 113 return ERR_PTR(ret); 114 } 115 isert_conn->cq_size = cq_size; 116 117 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 118 attr.event_handler = isert_qp_event_callback; 119 attr.qp_context = isert_conn; 120 attr.send_cq = isert_conn->cq; 121 attr.recv_cq = isert_conn->cq; 122 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 123 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 124 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 125 isert_sg_tablesize); 126 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 127 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 128 attr.cap.max_recv_sge = 1; 129 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 130 attr.qp_type = IB_QPT_RC; 131 if (device->pi_capable) 132 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 133 134 ret = rdma_create_qp(cma_id, device->pd, &attr); 135 if (ret) { 136 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 137 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 138 139 return ERR_PTR(ret); 140 } 141 142 return cma_id->qp; 143 } 144 145 static int 146 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 147 { 148 struct isert_device *device = isert_conn->device; 149 struct ib_device *ib_dev = device->ib_device; 150 struct iser_rx_desc *rx_desc; 151 struct ib_sge *rx_sg; 152 u64 dma_addr; 153 int i, j; 154 155 isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 156 sizeof(struct iser_rx_desc), 157 GFP_KERNEL); 158 if (!isert_conn->rx_descs) 159 return -ENOMEM; 160 161 rx_desc = isert_conn->rx_descs; 162 163 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 164 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf, 165 ISER_RX_SIZE, DMA_FROM_DEVICE); 166 if (ib_dma_mapping_error(ib_dev, dma_addr)) 167 goto dma_map_fail; 168 169 rx_desc->dma_addr = dma_addr; 170 171 rx_sg = &rx_desc->rx_sg; 172 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc); 173 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 174 rx_sg->lkey = device->pd->local_dma_lkey; 175 rx_desc->rx_cqe.done = isert_recv_done; 176 } 177 178 return 0; 179 180 dma_map_fail: 181 rx_desc = isert_conn->rx_descs; 182 for (j = 0; j < i; j++, rx_desc++) { 183 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 184 ISER_RX_SIZE, DMA_FROM_DEVICE); 185 } 186 kfree(isert_conn->rx_descs); 187 isert_conn->rx_descs = NULL; 188 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 189 return -ENOMEM; 190 } 191 192 static void 193 isert_free_rx_descriptors(struct isert_conn *isert_conn) 194 { 195 struct ib_device *ib_dev = isert_conn->device->ib_device; 196 struct iser_rx_desc *rx_desc; 197 int i; 198 199 if (!isert_conn->rx_descs) 200 return; 201 202 rx_desc = isert_conn->rx_descs; 203 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 204 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 205 ISER_RX_SIZE, DMA_FROM_DEVICE); 206 } 207 208 kfree(isert_conn->rx_descs); 209 isert_conn->rx_descs = NULL; 210 } 211 212 static int 213 isert_create_device_ib_res(struct isert_device *device) 214 { 215 struct ib_device *ib_dev = device->ib_device; 216 int ret; 217 218 isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 219 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 220 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 221 222 device->pd = ib_alloc_pd(ib_dev, 0); 223 if (IS_ERR(device->pd)) { 224 ret = PTR_ERR(device->pd); 225 isert_err("failed to allocate pd, device %p, ret=%d\n", 226 device, ret); 227 return ret; 228 } 229 230 /* Check signature cap */ 231 if (ib_dev->attrs.kernel_cap_flags & IBK_INTEGRITY_HANDOVER) 232 device->pi_capable = true; 233 else 234 device->pi_capable = false; 235 236 return 0; 237 } 238 239 static void 240 isert_free_device_ib_res(struct isert_device *device) 241 { 242 isert_info("device %p\n", device); 243 244 ib_dealloc_pd(device->pd); 245 } 246 247 static void 248 isert_device_put(struct isert_device *device) 249 { 250 mutex_lock(&device_list_mutex); 251 device->refcount--; 252 isert_info("device %p refcount %d\n", device, device->refcount); 253 if (!device->refcount) { 254 isert_free_device_ib_res(device); 255 list_del(&device->dev_node); 256 kfree(device); 257 } 258 mutex_unlock(&device_list_mutex); 259 } 260 261 static struct isert_device * 262 isert_device_get(struct rdma_cm_id *cma_id) 263 { 264 struct isert_device *device; 265 int ret; 266 267 mutex_lock(&device_list_mutex); 268 list_for_each_entry(device, &device_list, dev_node) { 269 if (device->ib_device->node_guid == cma_id->device->node_guid) { 270 device->refcount++; 271 isert_info("Found iser device %p refcount %d\n", 272 device, device->refcount); 273 mutex_unlock(&device_list_mutex); 274 return device; 275 } 276 } 277 278 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 279 if (!device) { 280 mutex_unlock(&device_list_mutex); 281 return ERR_PTR(-ENOMEM); 282 } 283 284 INIT_LIST_HEAD(&device->dev_node); 285 286 device->ib_device = cma_id->device; 287 ret = isert_create_device_ib_res(device); 288 if (ret) { 289 kfree(device); 290 mutex_unlock(&device_list_mutex); 291 return ERR_PTR(ret); 292 } 293 294 device->refcount++; 295 list_add_tail(&device->dev_node, &device_list); 296 isert_info("Created a new iser device %p refcount %d\n", 297 device, device->refcount); 298 mutex_unlock(&device_list_mutex); 299 300 return device; 301 } 302 303 static void 304 isert_init_conn(struct isert_conn *isert_conn) 305 { 306 isert_conn->state = ISER_CONN_INIT; 307 INIT_LIST_HEAD(&isert_conn->node); 308 init_completion(&isert_conn->login_comp); 309 init_completion(&isert_conn->login_req_comp); 310 init_waitqueue_head(&isert_conn->rem_wait); 311 kref_init(&isert_conn->kref); 312 mutex_init(&isert_conn->mutex); 313 INIT_WORK(&isert_conn->release_work, isert_release_work); 314 } 315 316 static void 317 isert_free_login_buf(struct isert_conn *isert_conn) 318 { 319 struct ib_device *ib_dev = isert_conn->device->ib_device; 320 321 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 322 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 323 kfree(isert_conn->login_rsp_buf); 324 325 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 326 ISER_RX_SIZE, DMA_FROM_DEVICE); 327 kfree(isert_conn->login_desc); 328 } 329 330 static int 331 isert_alloc_login_buf(struct isert_conn *isert_conn, 332 struct ib_device *ib_dev) 333 { 334 int ret; 335 336 isert_conn->login_desc = kzalloc(sizeof(*isert_conn->login_desc), 337 GFP_KERNEL); 338 if (!isert_conn->login_desc) 339 return -ENOMEM; 340 341 isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev, 342 isert_conn->login_desc->buf, 343 ISER_RX_SIZE, DMA_FROM_DEVICE); 344 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr); 345 if (ret) { 346 isert_err("login_desc dma mapping error: %d\n", ret); 347 isert_conn->login_desc->dma_addr = 0; 348 goto out_free_login_desc; 349 } 350 351 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 352 if (!isert_conn->login_rsp_buf) { 353 ret = -ENOMEM; 354 goto out_unmap_login_desc; 355 } 356 357 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 358 isert_conn->login_rsp_buf, 359 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 360 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 361 if (ret) { 362 isert_err("login_rsp_dma mapping error: %d\n", ret); 363 isert_conn->login_rsp_dma = 0; 364 goto out_free_login_rsp_buf; 365 } 366 367 return 0; 368 369 out_free_login_rsp_buf: 370 kfree(isert_conn->login_rsp_buf); 371 out_unmap_login_desc: 372 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 373 ISER_RX_SIZE, DMA_FROM_DEVICE); 374 out_free_login_desc: 375 kfree(isert_conn->login_desc); 376 return ret; 377 } 378 379 static void 380 isert_set_nego_params(struct isert_conn *isert_conn, 381 struct rdma_conn_param *param) 382 { 383 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 384 385 /* Set max inflight RDMA READ requests */ 386 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 387 attr->max_qp_init_rd_atom); 388 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 389 390 if (param->private_data) { 391 u8 flags = *(u8 *)param->private_data; 392 393 /* 394 * use remote invalidation if the both initiator 395 * and the HCA support it 396 */ 397 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 398 (attr->device_cap_flags & 399 IB_DEVICE_MEM_MGT_EXTENSIONS); 400 if (isert_conn->snd_w_inv) 401 isert_info("Using remote invalidation\n"); 402 } 403 } 404 405 static void 406 isert_destroy_qp(struct isert_conn *isert_conn) 407 { 408 ib_destroy_qp(isert_conn->qp); 409 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 410 } 411 412 static int 413 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 414 { 415 struct isert_np *isert_np = cma_id->context; 416 struct iscsi_np *np = isert_np->np; 417 struct isert_conn *isert_conn; 418 struct isert_device *device; 419 int ret = 0; 420 421 spin_lock_bh(&np->np_thread_lock); 422 if (!np->enabled) { 423 spin_unlock_bh(&np->np_thread_lock); 424 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 425 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 426 } 427 spin_unlock_bh(&np->np_thread_lock); 428 429 isert_dbg("cma_id: %p, portal: %p\n", 430 cma_id, cma_id->context); 431 432 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 433 if (!isert_conn) 434 return -ENOMEM; 435 436 isert_init_conn(isert_conn); 437 isert_conn->cm_id = cma_id; 438 439 device = isert_device_get(cma_id); 440 if (IS_ERR(device)) { 441 ret = PTR_ERR(device); 442 goto out; 443 } 444 isert_conn->device = device; 445 446 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 447 if (ret) 448 goto out_conn_dev; 449 450 isert_set_nego_params(isert_conn, &event->param.conn); 451 452 isert_conn->qp = isert_create_qp(isert_conn, cma_id); 453 if (IS_ERR(isert_conn->qp)) { 454 ret = PTR_ERR(isert_conn->qp); 455 goto out_rsp_dma_map; 456 } 457 458 ret = isert_login_post_recv(isert_conn); 459 if (ret) 460 goto out_destroy_qp; 461 462 ret = isert_rdma_accept(isert_conn); 463 if (ret) 464 goto out_destroy_qp; 465 466 mutex_lock(&isert_np->mutex); 467 list_add_tail(&isert_conn->node, &isert_np->accepted); 468 mutex_unlock(&isert_np->mutex); 469 470 return 0; 471 472 out_destroy_qp: 473 isert_destroy_qp(isert_conn); 474 out_rsp_dma_map: 475 isert_free_login_buf(isert_conn); 476 out_conn_dev: 477 isert_device_put(device); 478 out: 479 kfree(isert_conn); 480 rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 481 return ret; 482 } 483 484 static void 485 isert_connect_release(struct isert_conn *isert_conn) 486 { 487 struct isert_device *device = isert_conn->device; 488 489 isert_dbg("conn %p\n", isert_conn); 490 491 BUG_ON(!device); 492 493 isert_free_rx_descriptors(isert_conn); 494 if (isert_conn->cm_id && 495 !isert_conn->dev_removed) 496 rdma_destroy_id(isert_conn->cm_id); 497 498 if (isert_conn->qp) 499 isert_destroy_qp(isert_conn); 500 501 if (isert_conn->login_desc) 502 isert_free_login_buf(isert_conn); 503 504 isert_device_put(device); 505 506 if (isert_conn->dev_removed) 507 wake_up_interruptible(&isert_conn->rem_wait); 508 else 509 kfree(isert_conn); 510 } 511 512 static void 513 isert_connected_handler(struct rdma_cm_id *cma_id) 514 { 515 struct isert_conn *isert_conn = cma_id->qp->qp_context; 516 struct isert_np *isert_np = cma_id->context; 517 518 isert_info("conn %p\n", isert_conn); 519 520 mutex_lock(&isert_conn->mutex); 521 isert_conn->state = ISER_CONN_UP; 522 kref_get(&isert_conn->kref); 523 mutex_unlock(&isert_conn->mutex); 524 525 mutex_lock(&isert_np->mutex); 526 list_move_tail(&isert_conn->node, &isert_np->pending); 527 mutex_unlock(&isert_np->mutex); 528 529 isert_info("np %p: Allow accept_np to continue\n", isert_np); 530 up(&isert_np->sem); 531 } 532 533 static void 534 isert_release_kref(struct kref *kref) 535 { 536 struct isert_conn *isert_conn = container_of(kref, 537 struct isert_conn, kref); 538 539 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 540 current->pid); 541 542 isert_connect_release(isert_conn); 543 } 544 545 static void 546 isert_put_conn(struct isert_conn *isert_conn) 547 { 548 kref_put(&isert_conn->kref, isert_release_kref); 549 } 550 551 static void 552 isert_handle_unbound_conn(struct isert_conn *isert_conn) 553 { 554 struct isert_np *isert_np = isert_conn->cm_id->context; 555 556 mutex_lock(&isert_np->mutex); 557 if (!list_empty(&isert_conn->node)) { 558 /* 559 * This means iscsi doesn't know this connection 560 * so schedule a cleanup ourselves 561 */ 562 list_del_init(&isert_conn->node); 563 isert_put_conn(isert_conn); 564 queue_work(isert_release_wq, &isert_conn->release_work); 565 } 566 mutex_unlock(&isert_np->mutex); 567 } 568 569 /** 570 * isert_conn_terminate() - Initiate connection termination 571 * @isert_conn: isert connection struct 572 * 573 * Notes: 574 * In case the connection state is BOUND, move state 575 * to TEMINATING and start teardown sequence (rdma_disconnect). 576 * In case the connection state is UP, complete flush as well. 577 * 578 * This routine must be called with mutex held. Thus it is 579 * safe to call multiple times. 580 */ 581 static void 582 isert_conn_terminate(struct isert_conn *isert_conn) 583 { 584 int err; 585 586 if (isert_conn->state >= ISER_CONN_TERMINATING) 587 return; 588 589 isert_info("Terminating conn %p state %d\n", 590 isert_conn, isert_conn->state); 591 isert_conn->state = ISER_CONN_TERMINATING; 592 err = rdma_disconnect(isert_conn->cm_id); 593 if (err) 594 isert_warn("Failed rdma_disconnect isert_conn %p\n", 595 isert_conn); 596 } 597 598 static int 599 isert_np_cma_handler(struct isert_np *isert_np, 600 enum rdma_cm_event_type event) 601 { 602 isert_dbg("%s (%d): isert np %p\n", 603 rdma_event_msg(event), event, isert_np); 604 605 switch (event) { 606 case RDMA_CM_EVENT_DEVICE_REMOVAL: 607 isert_np->cm_id = NULL; 608 break; 609 case RDMA_CM_EVENT_ADDR_CHANGE: 610 isert_np->cm_id = isert_setup_id(isert_np); 611 if (IS_ERR(isert_np->cm_id)) { 612 isert_err("isert np %p setup id failed: %ld\n", 613 isert_np, PTR_ERR(isert_np->cm_id)); 614 isert_np->cm_id = NULL; 615 } 616 break; 617 default: 618 isert_err("isert np %p Unexpected event %d\n", 619 isert_np, event); 620 } 621 622 return -1; 623 } 624 625 static int 626 isert_disconnected_handler(struct rdma_cm_id *cma_id, 627 enum rdma_cm_event_type event) 628 { 629 struct isert_conn *isert_conn = cma_id->qp->qp_context; 630 631 mutex_lock(&isert_conn->mutex); 632 switch (isert_conn->state) { 633 case ISER_CONN_TERMINATING: 634 break; 635 case ISER_CONN_UP: 636 isert_conn_terminate(isert_conn); 637 ib_drain_qp(isert_conn->qp); 638 isert_handle_unbound_conn(isert_conn); 639 break; 640 case ISER_CONN_BOUND: 641 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 642 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 643 break; 644 default: 645 isert_warn("conn %p terminating in state %d\n", 646 isert_conn, isert_conn->state); 647 } 648 mutex_unlock(&isert_conn->mutex); 649 650 return 0; 651 } 652 653 static int 654 isert_connect_error(struct rdma_cm_id *cma_id) 655 { 656 struct isert_conn *isert_conn = cma_id->qp->qp_context; 657 struct isert_np *isert_np = cma_id->context; 658 659 ib_drain_qp(isert_conn->qp); 660 661 mutex_lock(&isert_np->mutex); 662 list_del_init(&isert_conn->node); 663 mutex_unlock(&isert_np->mutex); 664 isert_conn->cm_id = NULL; 665 isert_put_conn(isert_conn); 666 667 return -1; 668 } 669 670 static int 671 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 672 { 673 struct isert_np *isert_np = cma_id->context; 674 struct isert_conn *isert_conn; 675 int ret = 0; 676 677 isert_info("%s (%d): status %d id %p np %p\n", 678 rdma_event_msg(event->event), event->event, 679 event->status, cma_id, cma_id->context); 680 681 if (isert_np->cm_id == cma_id) 682 return isert_np_cma_handler(cma_id->context, event->event); 683 684 switch (event->event) { 685 case RDMA_CM_EVENT_CONNECT_REQUEST: 686 ret = isert_connect_request(cma_id, event); 687 if (ret) 688 isert_err("failed handle connect request %d\n", ret); 689 break; 690 case RDMA_CM_EVENT_ESTABLISHED: 691 isert_connected_handler(cma_id); 692 break; 693 case RDMA_CM_EVENT_ADDR_CHANGE: 694 case RDMA_CM_EVENT_DISCONNECTED: 695 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 696 ret = isert_disconnected_handler(cma_id, event->event); 697 break; 698 case RDMA_CM_EVENT_DEVICE_REMOVAL: 699 isert_conn = cma_id->qp->qp_context; 700 isert_conn->dev_removed = true; 701 isert_disconnected_handler(cma_id, event->event); 702 wait_event_interruptible(isert_conn->rem_wait, 703 isert_conn->state == ISER_CONN_DOWN); 704 kfree(isert_conn); 705 /* 706 * return non-zero from the callback to destroy 707 * the rdma cm id 708 */ 709 return 1; 710 case RDMA_CM_EVENT_REJECTED: 711 isert_info("Connection rejected: %s\n", 712 rdma_reject_msg(cma_id, event->status)); 713 fallthrough; 714 case RDMA_CM_EVENT_UNREACHABLE: 715 case RDMA_CM_EVENT_CONNECT_ERROR: 716 ret = isert_connect_error(cma_id); 717 break; 718 default: 719 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 720 break; 721 } 722 723 return ret; 724 } 725 726 static int 727 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 728 { 729 struct ib_recv_wr *rx_wr; 730 int i, ret; 731 struct iser_rx_desc *rx_desc; 732 733 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 734 rx_desc = &isert_conn->rx_descs[i]; 735 736 rx_wr->wr_cqe = &rx_desc->rx_cqe; 737 rx_wr->sg_list = &rx_desc->rx_sg; 738 rx_wr->num_sge = 1; 739 rx_wr->next = rx_wr + 1; 740 rx_desc->in_use = false; 741 } 742 rx_wr--; 743 rx_wr->next = NULL; /* mark end of work requests list */ 744 745 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 746 if (ret) 747 isert_err("ib_post_recv() failed with ret: %d\n", ret); 748 749 return ret; 750 } 751 752 static int 753 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 754 { 755 struct ib_recv_wr rx_wr; 756 int ret; 757 758 if (!rx_desc->in_use) { 759 /* 760 * if the descriptor is not in-use we already reposted it 761 * for recv, so just silently return 762 */ 763 return 0; 764 } 765 766 rx_desc->in_use = false; 767 rx_wr.wr_cqe = &rx_desc->rx_cqe; 768 rx_wr.sg_list = &rx_desc->rx_sg; 769 rx_wr.num_sge = 1; 770 rx_wr.next = NULL; 771 772 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 773 if (ret) 774 isert_err("ib_post_recv() failed with ret: %d\n", ret); 775 776 return ret; 777 } 778 779 static int 780 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 781 { 782 struct ib_device *ib_dev = isert_conn->cm_id->device; 783 struct ib_send_wr send_wr; 784 int ret; 785 786 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 787 ISER_HEADERS_LEN, DMA_TO_DEVICE); 788 789 tx_desc->tx_cqe.done = isert_login_send_done; 790 791 send_wr.next = NULL; 792 send_wr.wr_cqe = &tx_desc->tx_cqe; 793 send_wr.sg_list = tx_desc->tx_sg; 794 send_wr.num_sge = tx_desc->num_sge; 795 send_wr.opcode = IB_WR_SEND; 796 send_wr.send_flags = IB_SEND_SIGNALED; 797 798 ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 799 if (ret) 800 isert_err("ib_post_send() failed, ret: %d\n", ret); 801 802 return ret; 803 } 804 805 static void 806 __isert_create_send_desc(struct isert_device *device, 807 struct iser_tx_desc *tx_desc) 808 { 809 810 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 811 tx_desc->iser_header.flags = ISCSI_CTRL; 812 813 tx_desc->num_sge = 1; 814 815 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 816 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 817 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 818 } 819 } 820 821 static void 822 isert_create_send_desc(struct isert_conn *isert_conn, 823 struct isert_cmd *isert_cmd, 824 struct iser_tx_desc *tx_desc) 825 { 826 struct isert_device *device = isert_conn->device; 827 struct ib_device *ib_dev = device->ib_device; 828 829 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 830 ISER_HEADERS_LEN, DMA_TO_DEVICE); 831 832 __isert_create_send_desc(device, tx_desc); 833 } 834 835 static int 836 isert_init_tx_hdrs(struct isert_conn *isert_conn, 837 struct iser_tx_desc *tx_desc) 838 { 839 struct isert_device *device = isert_conn->device; 840 struct ib_device *ib_dev = device->ib_device; 841 u64 dma_addr; 842 843 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 844 ISER_HEADERS_LEN, DMA_TO_DEVICE); 845 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 846 isert_err("ib_dma_mapping_error() failed\n"); 847 return -ENOMEM; 848 } 849 850 tx_desc->dma_addr = dma_addr; 851 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 852 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 853 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 854 855 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 856 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 857 tx_desc->tx_sg[0].lkey); 858 859 return 0; 860 } 861 862 static void 863 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 864 struct ib_send_wr *send_wr) 865 { 866 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 867 868 tx_desc->tx_cqe.done = isert_send_done; 869 send_wr->wr_cqe = &tx_desc->tx_cqe; 870 871 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 872 send_wr->opcode = IB_WR_SEND_WITH_INV; 873 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 874 } else { 875 send_wr->opcode = IB_WR_SEND; 876 } 877 878 send_wr->sg_list = &tx_desc->tx_sg[0]; 879 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 880 send_wr->send_flags = IB_SEND_SIGNALED; 881 } 882 883 static int 884 isert_login_post_recv(struct isert_conn *isert_conn) 885 { 886 struct ib_recv_wr rx_wr; 887 struct ib_sge sge; 888 int ret; 889 890 memset(&sge, 0, sizeof(struct ib_sge)); 891 sge.addr = isert_conn->login_desc->dma_addr + 892 isert_get_hdr_offset(isert_conn->login_desc); 893 sge.length = ISER_RX_PAYLOAD_SIZE; 894 sge.lkey = isert_conn->device->pd->local_dma_lkey; 895 896 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 897 sge.addr, sge.length, sge.lkey); 898 899 isert_conn->login_desc->rx_cqe.done = isert_login_recv_done; 900 901 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 902 rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe; 903 rx_wr.sg_list = &sge; 904 rx_wr.num_sge = 1; 905 906 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 907 if (ret) 908 isert_err("ib_post_recv() failed: %d\n", ret); 909 910 return ret; 911 } 912 913 static int 914 isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, 915 u32 length) 916 { 917 struct isert_conn *isert_conn = conn->context; 918 struct isert_device *device = isert_conn->device; 919 struct ib_device *ib_dev = device->ib_device; 920 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 921 int ret; 922 923 __isert_create_send_desc(device, tx_desc); 924 925 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 926 sizeof(struct iscsi_hdr)); 927 928 isert_init_tx_hdrs(isert_conn, tx_desc); 929 930 if (length > 0) { 931 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 932 933 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 934 length, DMA_TO_DEVICE); 935 936 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 937 938 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 939 length, DMA_TO_DEVICE); 940 941 tx_dsg->addr = isert_conn->login_rsp_dma; 942 tx_dsg->length = length; 943 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 944 tx_desc->num_sge = 2; 945 } 946 if (!login->login_failed) { 947 if (login->login_complete) { 948 ret = isert_alloc_rx_descriptors(isert_conn); 949 if (ret) 950 return ret; 951 952 ret = isert_post_recvm(isert_conn, 953 ISERT_QP_MAX_RECV_DTOS); 954 if (ret) 955 return ret; 956 957 /* Now we are in FULL_FEATURE phase */ 958 mutex_lock(&isert_conn->mutex); 959 isert_conn->state = ISER_CONN_FULL_FEATURE; 960 mutex_unlock(&isert_conn->mutex); 961 goto post_send; 962 } 963 964 ret = isert_login_post_recv(isert_conn); 965 if (ret) 966 return ret; 967 } 968 post_send: 969 ret = isert_login_post_send(isert_conn, tx_desc); 970 if (ret) 971 return ret; 972 973 return 0; 974 } 975 976 static void 977 isert_rx_login_req(struct isert_conn *isert_conn) 978 { 979 struct iser_rx_desc *rx_desc = isert_conn->login_desc; 980 int rx_buflen = isert_conn->login_req_len; 981 struct iscsit_conn *conn = isert_conn->conn; 982 struct iscsi_login *login = conn->conn_login; 983 int size; 984 985 isert_info("conn %p\n", isert_conn); 986 987 WARN_ON_ONCE(!login); 988 989 if (login->first_request) { 990 struct iscsi_login_req *login_req = 991 (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc); 992 /* 993 * Setup the initial iscsi_login values from the leading 994 * login request PDU. 995 */ 996 login->leading_connection = (!login_req->tsih) ? 1 : 0; 997 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 998 login_req->flags); 999 login->version_min = login_req->min_version; 1000 login->version_max = login_req->max_version; 1001 memcpy(login->isid, login_req->isid, 6); 1002 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1003 login->init_task_tag = login_req->itt; 1004 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1005 login->cid = be16_to_cpu(login_req->cid); 1006 login->tsih = be16_to_cpu(login_req->tsih); 1007 } 1008 1009 memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN); 1010 1011 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1012 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1013 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1014 MAX_KEY_VALUE_PAIRS); 1015 memcpy(login->req_buf, isert_get_data(rx_desc), size); 1016 1017 if (login->first_request) { 1018 complete(&isert_conn->login_comp); 1019 return; 1020 } 1021 queue_delayed_work(isert_login_wq, &conn->login_work, 0); 1022 } 1023 1024 static struct iscsit_cmd 1025 *isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc) 1026 { 1027 struct isert_conn *isert_conn = conn->context; 1028 struct isert_cmd *isert_cmd; 1029 struct iscsit_cmd *cmd; 1030 1031 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1032 if (!cmd) { 1033 isert_err("Unable to allocate iscsit_cmd + isert_cmd\n"); 1034 return NULL; 1035 } 1036 isert_cmd = iscsit_priv_cmd(cmd); 1037 isert_cmd->conn = isert_conn; 1038 isert_cmd->iscsit_cmd = cmd; 1039 isert_cmd->rx_desc = rx_desc; 1040 1041 return cmd; 1042 } 1043 1044 static int 1045 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1046 struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd, 1047 struct iser_rx_desc *rx_desc, unsigned char *buf) 1048 { 1049 struct iscsit_conn *conn = isert_conn->conn; 1050 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1051 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1052 bool dump_payload = false; 1053 unsigned int data_len; 1054 1055 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1056 if (rc < 0) 1057 return rc; 1058 1059 imm_data = cmd->immediate_data; 1060 imm_data_len = cmd->first_burst_len; 1061 unsol_data = cmd->unsolicited_data; 1062 data_len = cmd->se_cmd.data_length; 1063 1064 if (imm_data && imm_data_len == data_len) 1065 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1066 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1067 if (rc < 0) { 1068 return 0; 1069 } else if (rc > 0) { 1070 dump_payload = true; 1071 goto sequence_cmd; 1072 } 1073 1074 if (!imm_data) 1075 return 0; 1076 1077 if (imm_data_len != data_len) { 1078 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1079 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1080 isert_get_data(rx_desc), imm_data_len); 1081 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1082 sg_nents, imm_data_len); 1083 } else { 1084 sg_init_table(&isert_cmd->sg, 1); 1085 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1086 cmd->se_cmd.t_data_nents = 1; 1087 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc), 1088 imm_data_len); 1089 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1090 imm_data_len); 1091 } 1092 1093 cmd->write_data_done += imm_data_len; 1094 1095 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1096 spin_lock_bh(&cmd->istate_lock); 1097 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1098 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1099 spin_unlock_bh(&cmd->istate_lock); 1100 } 1101 1102 sequence_cmd: 1103 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1104 1105 if (!rc && !dump_payload && unsol_data) 1106 iscsit_set_unsolicited_dataout(cmd); 1107 else if (dump_payload && imm_data) 1108 target_put_sess_cmd(&cmd->se_cmd); 1109 1110 return 0; 1111 } 1112 1113 static int 1114 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1115 struct iser_rx_desc *rx_desc, unsigned char *buf) 1116 { 1117 struct scatterlist *sg_start; 1118 struct iscsit_conn *conn = isert_conn->conn; 1119 struct iscsit_cmd *cmd = NULL; 1120 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1121 u32 unsol_data_len = ntoh24(hdr->dlength); 1122 int rc, sg_nents, sg_off, page_off; 1123 1124 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1125 if (rc < 0) 1126 return rc; 1127 else if (!cmd) 1128 return 0; 1129 /* 1130 * FIXME: Unexpected unsolicited_data out 1131 */ 1132 if (!cmd->unsolicited_data) { 1133 isert_err("Received unexpected solicited data payload\n"); 1134 dump_stack(); 1135 return -1; 1136 } 1137 1138 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1139 "write_data_done: %u, data_length: %u\n", 1140 unsol_data_len, cmd->write_data_done, 1141 cmd->se_cmd.data_length); 1142 1143 sg_off = cmd->write_data_done / PAGE_SIZE; 1144 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1145 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1146 page_off = cmd->write_data_done % PAGE_SIZE; 1147 /* 1148 * FIXME: Non page-aligned unsolicited_data out 1149 */ 1150 if (page_off) { 1151 isert_err("unexpected non-page aligned data payload\n"); 1152 dump_stack(); 1153 return -1; 1154 } 1155 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1156 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1157 sg_nents, isert_get_data(rx_desc), unsol_data_len); 1158 1159 sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc), 1160 unsol_data_len); 1161 1162 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1163 if (rc < 0) 1164 return rc; 1165 1166 /* 1167 * multiple data-outs on the same command can arrive - 1168 * so post the buffer before hand 1169 */ 1170 return isert_post_recv(isert_conn, rx_desc); 1171 } 1172 1173 static int 1174 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1175 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1176 unsigned char *buf) 1177 { 1178 struct iscsit_conn *conn = isert_conn->conn; 1179 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1180 int rc; 1181 1182 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1183 if (rc < 0) 1184 return rc; 1185 /* 1186 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1187 */ 1188 1189 return iscsit_process_nop_out(conn, cmd, hdr); 1190 } 1191 1192 static int 1193 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1194 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1195 struct iscsi_text *hdr) 1196 { 1197 struct iscsit_conn *conn = isert_conn->conn; 1198 u32 payload_length = ntoh24(hdr->dlength); 1199 int rc; 1200 unsigned char *text_in = NULL; 1201 1202 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1203 if (rc < 0) 1204 return rc; 1205 1206 if (payload_length) { 1207 text_in = kzalloc(payload_length, GFP_KERNEL); 1208 if (!text_in) 1209 return -ENOMEM; 1210 } 1211 cmd->text_in_ptr = text_in; 1212 1213 memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length); 1214 1215 return iscsit_process_text_cmd(conn, cmd, hdr); 1216 } 1217 1218 static int 1219 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1220 uint32_t read_stag, uint64_t read_va, 1221 uint32_t write_stag, uint64_t write_va) 1222 { 1223 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1224 struct iscsit_conn *conn = isert_conn->conn; 1225 struct iscsit_cmd *cmd; 1226 struct isert_cmd *isert_cmd; 1227 int ret = -EINVAL; 1228 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1229 1230 if (conn->sess->sess_ops->SessionType && 1231 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1232 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1233 " ignoring\n", opcode); 1234 return 0; 1235 } 1236 1237 switch (opcode) { 1238 case ISCSI_OP_SCSI_CMD: 1239 cmd = isert_allocate_cmd(conn, rx_desc); 1240 if (!cmd) 1241 break; 1242 1243 isert_cmd = iscsit_priv_cmd(cmd); 1244 isert_cmd->read_stag = read_stag; 1245 isert_cmd->read_va = read_va; 1246 isert_cmd->write_stag = write_stag; 1247 isert_cmd->write_va = write_va; 1248 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1249 1250 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1251 rx_desc, (unsigned char *)hdr); 1252 break; 1253 case ISCSI_OP_NOOP_OUT: 1254 cmd = isert_allocate_cmd(conn, rx_desc); 1255 if (!cmd) 1256 break; 1257 1258 isert_cmd = iscsit_priv_cmd(cmd); 1259 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1260 rx_desc, (unsigned char *)hdr); 1261 break; 1262 case ISCSI_OP_SCSI_DATA_OUT: 1263 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1264 (unsigned char *)hdr); 1265 break; 1266 case ISCSI_OP_SCSI_TMFUNC: 1267 cmd = isert_allocate_cmd(conn, rx_desc); 1268 if (!cmd) 1269 break; 1270 1271 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1272 (unsigned char *)hdr); 1273 break; 1274 case ISCSI_OP_LOGOUT: 1275 cmd = isert_allocate_cmd(conn, rx_desc); 1276 if (!cmd) 1277 break; 1278 1279 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1280 break; 1281 case ISCSI_OP_TEXT: 1282 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1283 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1284 else 1285 cmd = isert_allocate_cmd(conn, rx_desc); 1286 1287 if (!cmd) 1288 break; 1289 1290 isert_cmd = iscsit_priv_cmd(cmd); 1291 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1292 rx_desc, (struct iscsi_text *)hdr); 1293 break; 1294 default: 1295 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1296 dump_stack(); 1297 break; 1298 } 1299 1300 return ret; 1301 } 1302 1303 static void 1304 isert_print_wc(struct ib_wc *wc, const char *type) 1305 { 1306 if (wc->status != IB_WC_WR_FLUSH_ERR) 1307 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1308 ib_wc_status_msg(wc->status), wc->status, 1309 wc->vendor_err); 1310 else 1311 isert_dbg("%s failure: %s (%d)\n", type, 1312 ib_wc_status_msg(wc->status), wc->status); 1313 } 1314 1315 static void 1316 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1317 { 1318 struct isert_conn *isert_conn = wc->qp->qp_context; 1319 struct ib_device *ib_dev = isert_conn->cm_id->device; 1320 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1321 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1322 struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc); 1323 uint64_t read_va = 0, write_va = 0; 1324 uint32_t read_stag = 0, write_stag = 0; 1325 1326 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1327 isert_print_wc(wc, "recv"); 1328 if (wc->status != IB_WC_WR_FLUSH_ERR) 1329 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1330 return; 1331 } 1332 1333 rx_desc->in_use = true; 1334 1335 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1336 ISER_RX_SIZE, DMA_FROM_DEVICE); 1337 1338 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1339 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1340 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1341 1342 switch (iser_ctrl->flags & 0xF0) { 1343 case ISCSI_CTRL: 1344 if (iser_ctrl->flags & ISER_RSV) { 1345 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1346 read_va = be64_to_cpu(iser_ctrl->read_va); 1347 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1348 read_stag, (unsigned long long)read_va); 1349 } 1350 if (iser_ctrl->flags & ISER_WSV) { 1351 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1352 write_va = be64_to_cpu(iser_ctrl->write_va); 1353 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1354 write_stag, (unsigned long long)write_va); 1355 } 1356 1357 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1358 break; 1359 case ISER_HELLO: 1360 isert_err("iSER Hello message\n"); 1361 break; 1362 default: 1363 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1364 break; 1365 } 1366 1367 isert_rx_opcode(isert_conn, rx_desc, 1368 read_stag, read_va, write_stag, write_va); 1369 1370 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1371 ISER_RX_SIZE, DMA_FROM_DEVICE); 1372 } 1373 1374 static void 1375 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1376 { 1377 struct isert_conn *isert_conn = wc->qp->qp_context; 1378 struct ib_device *ib_dev = isert_conn->device->ib_device; 1379 1380 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1381 isert_print_wc(wc, "login recv"); 1382 return; 1383 } 1384 1385 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr, 1386 ISER_RX_SIZE, DMA_FROM_DEVICE); 1387 1388 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1389 1390 if (isert_conn->conn) { 1391 struct iscsi_login *login = isert_conn->conn->conn_login; 1392 1393 if (login && !login->first_request) 1394 isert_rx_login_req(isert_conn); 1395 } 1396 1397 mutex_lock(&isert_conn->mutex); 1398 complete(&isert_conn->login_req_comp); 1399 mutex_unlock(&isert_conn->mutex); 1400 1401 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr, 1402 ISER_RX_SIZE, DMA_FROM_DEVICE); 1403 } 1404 1405 static void 1406 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1407 { 1408 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 1409 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1410 1411 if (!cmd->rw.nr_ops) 1412 return; 1413 1414 if (isert_prot_cmd(conn, se_cmd)) { 1415 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1416 conn->cm_id->port_num, se_cmd->t_data_sg, 1417 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1418 se_cmd->t_prot_nents, dir); 1419 } else { 1420 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1421 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1422 } 1423 1424 cmd->rw.nr_ops = 0; 1425 } 1426 1427 static void 1428 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1429 { 1430 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1431 struct isert_conn *isert_conn = isert_cmd->conn; 1432 struct iscsit_conn *conn = isert_conn->conn; 1433 struct iscsi_text_rsp *hdr; 1434 1435 isert_dbg("Cmd %p\n", isert_cmd); 1436 1437 switch (cmd->iscsi_opcode) { 1438 case ISCSI_OP_SCSI_CMD: 1439 spin_lock_bh(&conn->cmd_lock); 1440 if (!list_empty(&cmd->i_conn_node)) 1441 list_del_init(&cmd->i_conn_node); 1442 spin_unlock_bh(&conn->cmd_lock); 1443 1444 if (cmd->data_direction == DMA_TO_DEVICE) { 1445 iscsit_stop_dataout_timer(cmd); 1446 /* 1447 * Check for special case during comp_err where 1448 * WRITE_PENDING has been handed off from core, 1449 * but requires an extra target_put_sess_cmd() 1450 * before transport_generic_free_cmd() below. 1451 */ 1452 if (comp_err && 1453 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1454 struct se_cmd *se_cmd = &cmd->se_cmd; 1455 1456 target_put_sess_cmd(se_cmd); 1457 } 1458 } 1459 1460 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1461 transport_generic_free_cmd(&cmd->se_cmd, 0); 1462 break; 1463 case ISCSI_OP_SCSI_TMFUNC: 1464 spin_lock_bh(&conn->cmd_lock); 1465 if (!list_empty(&cmd->i_conn_node)) 1466 list_del_init(&cmd->i_conn_node); 1467 spin_unlock_bh(&conn->cmd_lock); 1468 1469 transport_generic_free_cmd(&cmd->se_cmd, 0); 1470 break; 1471 case ISCSI_OP_REJECT: 1472 case ISCSI_OP_NOOP_OUT: 1473 case ISCSI_OP_TEXT: 1474 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1475 /* If the continue bit is on, keep the command alive */ 1476 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1477 break; 1478 1479 spin_lock_bh(&conn->cmd_lock); 1480 if (!list_empty(&cmd->i_conn_node)) 1481 list_del_init(&cmd->i_conn_node); 1482 spin_unlock_bh(&conn->cmd_lock); 1483 1484 /* 1485 * Handle special case for REJECT when iscsi_add_reject*() has 1486 * overwritten the original iscsi_opcode assignment, and the 1487 * associated cmd->se_cmd needs to be released. 1488 */ 1489 if (cmd->se_cmd.se_tfo != NULL) { 1490 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1491 cmd->iscsi_opcode); 1492 transport_generic_free_cmd(&cmd->se_cmd, 0); 1493 break; 1494 } 1495 fallthrough; 1496 default: 1497 iscsit_release_cmd(cmd); 1498 break; 1499 } 1500 } 1501 1502 static void 1503 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1504 { 1505 if (tx_desc->dma_addr != 0) { 1506 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1507 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1508 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1509 tx_desc->dma_addr = 0; 1510 } 1511 } 1512 1513 static void 1514 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1515 struct ib_device *ib_dev, bool comp_err) 1516 { 1517 if (isert_cmd->pdu_buf_dma != 0) { 1518 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1519 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1520 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1521 isert_cmd->pdu_buf_dma = 0; 1522 } 1523 1524 isert_unmap_tx_desc(tx_desc, ib_dev); 1525 isert_put_cmd(isert_cmd, comp_err); 1526 } 1527 1528 static int 1529 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1530 { 1531 struct ib_mr_status mr_status; 1532 int ret; 1533 1534 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1535 if (ret) { 1536 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1537 goto fail_mr_status; 1538 } 1539 1540 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1541 u64 sec_offset_err; 1542 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1543 1544 switch (mr_status.sig_err.err_type) { 1545 case IB_SIG_BAD_GUARD: 1546 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1547 break; 1548 case IB_SIG_BAD_REFTAG: 1549 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1550 break; 1551 case IB_SIG_BAD_APPTAG: 1552 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1553 break; 1554 } 1555 sec_offset_err = mr_status.sig_err.sig_err_offset; 1556 do_div(sec_offset_err, block_size); 1557 se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba; 1558 1559 isert_err("PI error found type %d at sector 0x%llx " 1560 "expected 0x%x vs actual 0x%x\n", 1561 mr_status.sig_err.err_type, 1562 (unsigned long long)se_cmd->sense_info, 1563 mr_status.sig_err.expected, 1564 mr_status.sig_err.actual); 1565 ret = 1; 1566 } 1567 1568 fail_mr_status: 1569 return ret; 1570 } 1571 1572 static void 1573 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1574 { 1575 struct isert_conn *isert_conn = wc->qp->qp_context; 1576 struct isert_device *device = isert_conn->device; 1577 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1578 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1579 struct se_cmd *cmd = &isert_cmd->iscsit_cmd->se_cmd; 1580 int ret = 0; 1581 1582 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1583 isert_print_wc(wc, "rdma write"); 1584 if (wc->status != IB_WC_WR_FLUSH_ERR) 1585 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1586 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1587 return; 1588 } 1589 1590 isert_dbg("Cmd %p\n", isert_cmd); 1591 1592 ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 1593 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1594 1595 if (ret) { 1596 /* 1597 * transport_generic_request_failure() expects to have 1598 * plus two references to handle queue-full, so re-add 1599 * one here as target-core will have already dropped 1600 * it after the first isert_put_datain() callback. 1601 */ 1602 kref_get(&cmd->cmd_kref); 1603 transport_generic_request_failure(cmd, cmd->pi_err); 1604 } else { 1605 /* 1606 * XXX: isert_put_response() failure is not retried. 1607 */ 1608 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsit_cmd); 1609 if (ret) 1610 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1611 } 1612 } 1613 1614 static void 1615 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1616 { 1617 struct isert_conn *isert_conn = wc->qp->qp_context; 1618 struct isert_device *device = isert_conn->device; 1619 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1620 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1621 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1622 struct se_cmd *se_cmd = &cmd->se_cmd; 1623 int ret = 0; 1624 1625 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1626 isert_print_wc(wc, "rdma read"); 1627 if (wc->status != IB_WC_WR_FLUSH_ERR) 1628 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1629 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1630 return; 1631 } 1632 1633 isert_dbg("Cmd %p\n", isert_cmd); 1634 1635 iscsit_stop_dataout_timer(cmd); 1636 1637 if (isert_prot_cmd(isert_conn, se_cmd)) 1638 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 1639 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1640 cmd->write_data_done = 0; 1641 1642 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1643 spin_lock_bh(&cmd->istate_lock); 1644 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1645 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1646 spin_unlock_bh(&cmd->istate_lock); 1647 1648 /* 1649 * transport_generic_request_failure() will drop the extra 1650 * se_cmd->cmd_kref reference after T10-PI error, and handle 1651 * any non-zero ->queue_status() callback error retries. 1652 */ 1653 if (ret) 1654 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1655 else 1656 target_execute_cmd(se_cmd); 1657 } 1658 1659 static void 1660 isert_do_control_comp(struct work_struct *work) 1661 { 1662 struct isert_cmd *isert_cmd = container_of(work, 1663 struct isert_cmd, comp_work); 1664 struct isert_conn *isert_conn = isert_cmd->conn; 1665 struct ib_device *ib_dev = isert_conn->cm_id->device; 1666 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1667 1668 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1669 1670 switch (cmd->i_state) { 1671 case ISTATE_SEND_TASKMGTRSP: 1672 iscsit_tmr_post_handler(cmd, cmd->conn); 1673 fallthrough; 1674 case ISTATE_SEND_REJECT: 1675 case ISTATE_SEND_TEXTRSP: 1676 cmd->i_state = ISTATE_SENT_STATUS; 1677 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1678 ib_dev, false); 1679 break; 1680 case ISTATE_SEND_LOGOUTRSP: 1681 iscsit_logout_post_handler(cmd, cmd->conn); 1682 break; 1683 default: 1684 isert_err("Unknown i_state %d\n", cmd->i_state); 1685 dump_stack(); 1686 break; 1687 } 1688 } 1689 1690 static void 1691 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1692 { 1693 struct isert_conn *isert_conn = wc->qp->qp_context; 1694 struct ib_device *ib_dev = isert_conn->cm_id->device; 1695 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1696 1697 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1698 isert_print_wc(wc, "login send"); 1699 if (wc->status != IB_WC_WR_FLUSH_ERR) 1700 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1701 } 1702 1703 isert_unmap_tx_desc(tx_desc, ib_dev); 1704 } 1705 1706 static void 1707 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1708 { 1709 struct isert_conn *isert_conn = wc->qp->qp_context; 1710 struct ib_device *ib_dev = isert_conn->cm_id->device; 1711 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1712 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1713 1714 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1715 isert_print_wc(wc, "send"); 1716 if (wc->status != IB_WC_WR_FLUSH_ERR) 1717 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1718 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1719 return; 1720 } 1721 1722 isert_dbg("Cmd %p\n", isert_cmd); 1723 1724 switch (isert_cmd->iscsit_cmd->i_state) { 1725 case ISTATE_SEND_TASKMGTRSP: 1726 case ISTATE_SEND_LOGOUTRSP: 1727 case ISTATE_SEND_REJECT: 1728 case ISTATE_SEND_TEXTRSP: 1729 isert_unmap_tx_desc(tx_desc, ib_dev); 1730 1731 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1732 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1733 return; 1734 default: 1735 isert_cmd->iscsit_cmd->i_state = ISTATE_SENT_STATUS; 1736 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1737 break; 1738 } 1739 } 1740 1741 static int 1742 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1743 { 1744 int ret; 1745 1746 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1747 if (ret) 1748 return ret; 1749 1750 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1751 if (ret) { 1752 isert_err("ib_post_send failed with %d\n", ret); 1753 return ret; 1754 } 1755 return ret; 1756 } 1757 1758 static int 1759 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1760 { 1761 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1762 struct isert_conn *isert_conn = conn->context; 1763 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1764 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1765 &isert_cmd->tx_desc.iscsi_header; 1766 1767 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1768 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1769 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1770 /* 1771 * Attach SENSE DATA payload to iSCSI Response PDU 1772 */ 1773 if (cmd->se_cmd.sense_buffer && 1774 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1775 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1776 struct isert_device *device = isert_conn->device; 1777 struct ib_device *ib_dev = device->ib_device; 1778 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1779 u32 padding, pdu_len; 1780 1781 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1782 cmd->sense_buffer); 1783 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1784 1785 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1786 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1787 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1788 1789 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1790 (void *)cmd->sense_buffer, pdu_len, 1791 DMA_TO_DEVICE); 1792 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1793 return -ENOMEM; 1794 1795 isert_cmd->pdu_buf_len = pdu_len; 1796 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1797 tx_dsg->length = pdu_len; 1798 tx_dsg->lkey = device->pd->local_dma_lkey; 1799 isert_cmd->tx_desc.num_sge = 2; 1800 } 1801 1802 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1803 1804 isert_dbg("Posting SCSI Response\n"); 1805 1806 return isert_post_response(isert_conn, isert_cmd); 1807 } 1808 1809 static void 1810 isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1811 { 1812 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1813 struct isert_conn *isert_conn = conn->context; 1814 1815 spin_lock_bh(&conn->cmd_lock); 1816 if (!list_empty(&cmd->i_conn_node)) 1817 list_del_init(&cmd->i_conn_node); 1818 spin_unlock_bh(&conn->cmd_lock); 1819 1820 if (cmd->data_direction == DMA_TO_DEVICE) 1821 iscsit_stop_dataout_timer(cmd); 1822 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1823 } 1824 1825 static enum target_prot_op 1826 isert_get_sup_prot_ops(struct iscsit_conn *conn) 1827 { 1828 struct isert_conn *isert_conn = conn->context; 1829 struct isert_device *device = isert_conn->device; 1830 1831 if (conn->tpg->tpg_attrib.t10_pi) { 1832 if (device->pi_capable) { 1833 isert_info("conn %p PI offload enabled\n", isert_conn); 1834 isert_conn->pi_support = true; 1835 return TARGET_PROT_ALL; 1836 } 1837 } 1838 1839 isert_info("conn %p PI offload disabled\n", isert_conn); 1840 isert_conn->pi_support = false; 1841 1842 return TARGET_PROT_NORMAL; 1843 } 1844 1845 static int 1846 isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn, 1847 bool nopout_response) 1848 { 1849 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1850 struct isert_conn *isert_conn = conn->context; 1851 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1852 1853 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1854 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1855 &isert_cmd->tx_desc.iscsi_header, 1856 nopout_response); 1857 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1858 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1859 1860 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1861 1862 return isert_post_response(isert_conn, isert_cmd); 1863 } 1864 1865 static int 1866 isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1867 { 1868 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1869 struct isert_conn *isert_conn = conn->context; 1870 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1871 1872 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1873 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1874 &isert_cmd->tx_desc.iscsi_header); 1875 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1876 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1877 1878 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1879 1880 return isert_post_response(isert_conn, isert_cmd); 1881 } 1882 1883 static int 1884 isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1885 { 1886 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1887 struct isert_conn *isert_conn = conn->context; 1888 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1889 1890 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1891 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1892 &isert_cmd->tx_desc.iscsi_header); 1893 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1894 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1895 1896 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1897 1898 return isert_post_response(isert_conn, isert_cmd); 1899 } 1900 1901 static int 1902 isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1903 { 1904 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1905 struct isert_conn *isert_conn = conn->context; 1906 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1907 struct isert_device *device = isert_conn->device; 1908 struct ib_device *ib_dev = device->ib_device; 1909 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1910 struct iscsi_reject *hdr = 1911 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1912 1913 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1914 iscsit_build_reject(cmd, conn, hdr); 1915 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1916 1917 hton24(hdr->dlength, ISCSI_HDR_LEN); 1918 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1919 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1920 DMA_TO_DEVICE); 1921 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1922 return -ENOMEM; 1923 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1924 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1925 tx_dsg->length = ISCSI_HDR_LEN; 1926 tx_dsg->lkey = device->pd->local_dma_lkey; 1927 isert_cmd->tx_desc.num_sge = 2; 1928 1929 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1930 1931 isert_dbg("conn %p Posting Reject\n", isert_conn); 1932 1933 return isert_post_response(isert_conn, isert_cmd); 1934 } 1935 1936 static int 1937 isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1938 { 1939 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1940 struct isert_conn *isert_conn = conn->context; 1941 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1942 struct iscsi_text_rsp *hdr = 1943 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1944 u32 txt_rsp_len; 1945 int rc; 1946 1947 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1948 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1949 if (rc < 0) 1950 return rc; 1951 1952 txt_rsp_len = rc; 1953 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1954 1955 if (txt_rsp_len) { 1956 struct isert_device *device = isert_conn->device; 1957 struct ib_device *ib_dev = device->ib_device; 1958 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1959 void *txt_rsp_buf = cmd->buf_ptr; 1960 1961 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1962 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1963 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1964 return -ENOMEM; 1965 1966 isert_cmd->pdu_buf_len = txt_rsp_len; 1967 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1968 tx_dsg->length = txt_rsp_len; 1969 tx_dsg->lkey = device->pd->local_dma_lkey; 1970 isert_cmd->tx_desc.num_sge = 2; 1971 } 1972 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1973 1974 isert_dbg("conn %p Text Response\n", isert_conn); 1975 1976 return isert_post_response(isert_conn, isert_cmd); 1977 } 1978 1979 static inline void 1980 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 1981 { 1982 domain->sig_type = IB_SIG_TYPE_T10_DIF; 1983 domain->sig.dif.bg_type = IB_T10DIF_CRC; 1984 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 1985 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 1986 /* 1987 * At the moment we hard code those, but if in the future 1988 * the target core would like to use it, we will take it 1989 * from se_cmd. 1990 */ 1991 domain->sig.dif.apptag_check_mask = 0xffff; 1992 domain->sig.dif.app_escape = true; 1993 domain->sig.dif.ref_escape = true; 1994 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 1995 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 1996 domain->sig.dif.ref_remap = true; 1997 } 1998 1999 static int 2000 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2001 { 2002 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2003 2004 switch (se_cmd->prot_op) { 2005 case TARGET_PROT_DIN_INSERT: 2006 case TARGET_PROT_DOUT_STRIP: 2007 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2008 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2009 break; 2010 case TARGET_PROT_DOUT_INSERT: 2011 case TARGET_PROT_DIN_STRIP: 2012 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2013 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2014 break; 2015 case TARGET_PROT_DIN_PASS: 2016 case TARGET_PROT_DOUT_PASS: 2017 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2018 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2019 break; 2020 default: 2021 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2022 return -EINVAL; 2023 } 2024 2025 if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 2026 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 2027 if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 2028 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 2029 if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 2030 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 2031 2032 return 0; 2033 } 2034 2035 static int 2036 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2037 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2038 { 2039 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 2040 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2041 u8 port_num = conn->cm_id->port_num; 2042 u64 addr; 2043 u32 rkey, offset; 2044 int ret; 2045 2046 if (cmd->ctx_init_done) 2047 goto rdma_ctx_post; 2048 2049 if (dir == DMA_FROM_DEVICE) { 2050 addr = cmd->write_va; 2051 rkey = cmd->write_stag; 2052 offset = cmd->iscsit_cmd->write_data_done; 2053 } else { 2054 addr = cmd->read_va; 2055 rkey = cmd->read_stag; 2056 offset = 0; 2057 } 2058 2059 if (isert_prot_cmd(conn, se_cmd)) { 2060 struct ib_sig_attrs sig_attrs; 2061 2062 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2063 if (ret) 2064 return ret; 2065 2066 WARN_ON_ONCE(offset); 2067 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2068 se_cmd->t_data_sg, se_cmd->t_data_nents, 2069 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2070 &sig_attrs, addr, rkey, dir); 2071 } else { 2072 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2073 se_cmd->t_data_sg, se_cmd->t_data_nents, 2074 offset, addr, rkey, dir); 2075 } 2076 2077 if (ret < 0) { 2078 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2079 return ret; 2080 } 2081 2082 cmd->ctx_init_done = true; 2083 2084 rdma_ctx_post: 2085 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2086 if (ret < 0) 2087 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2088 return ret; 2089 } 2090 2091 static int 2092 isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 2093 { 2094 struct se_cmd *se_cmd = &cmd->se_cmd; 2095 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2096 struct isert_conn *isert_conn = conn->context; 2097 struct ib_cqe *cqe = NULL; 2098 struct ib_send_wr *chain_wr = NULL; 2099 int rc; 2100 2101 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2102 isert_cmd, se_cmd->data_length); 2103 2104 if (isert_prot_cmd(isert_conn, se_cmd)) { 2105 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2106 cqe = &isert_cmd->tx_desc.tx_cqe; 2107 } else { 2108 /* 2109 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2110 */ 2111 isert_create_send_desc(isert_conn, isert_cmd, 2112 &isert_cmd->tx_desc); 2113 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2114 &isert_cmd->tx_desc.iscsi_header); 2115 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2116 isert_init_send_wr(isert_conn, isert_cmd, 2117 &isert_cmd->tx_desc.send_wr); 2118 2119 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2120 if (rc) 2121 return rc; 2122 2123 chain_wr = &isert_cmd->tx_desc.send_wr; 2124 } 2125 2126 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2127 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2128 isert_cmd, rc); 2129 return rc; 2130 } 2131 2132 static int 2133 isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery) 2134 { 2135 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2136 int ret; 2137 2138 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2139 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2140 2141 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2142 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2143 &isert_cmd->tx_desc.tx_cqe, NULL); 2144 2145 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2146 isert_cmd, ret); 2147 return ret; 2148 } 2149 2150 static int 2151 isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2152 { 2153 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2154 int ret = 0; 2155 2156 switch (state) { 2157 case ISTATE_REMOVE: 2158 spin_lock_bh(&conn->cmd_lock); 2159 list_del_init(&cmd->i_conn_node); 2160 spin_unlock_bh(&conn->cmd_lock); 2161 isert_put_cmd(isert_cmd, true); 2162 break; 2163 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2164 ret = isert_put_nopin(cmd, conn, false); 2165 break; 2166 default: 2167 isert_err("Unknown immediate state: 0x%02x\n", state); 2168 ret = -EINVAL; 2169 break; 2170 } 2171 2172 return ret; 2173 } 2174 2175 static int 2176 isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2177 { 2178 struct isert_conn *isert_conn = conn->context; 2179 int ret; 2180 2181 switch (state) { 2182 case ISTATE_SEND_LOGOUTRSP: 2183 ret = isert_put_logout_rsp(cmd, conn); 2184 if (!ret) 2185 isert_conn->logout_posted = true; 2186 break; 2187 case ISTATE_SEND_NOPIN: 2188 ret = isert_put_nopin(cmd, conn, true); 2189 break; 2190 case ISTATE_SEND_TASKMGTRSP: 2191 ret = isert_put_tm_rsp(cmd, conn); 2192 break; 2193 case ISTATE_SEND_REJECT: 2194 ret = isert_put_reject(cmd, conn); 2195 break; 2196 case ISTATE_SEND_TEXTRSP: 2197 ret = isert_put_text_rsp(cmd, conn); 2198 break; 2199 case ISTATE_SEND_STATUS: 2200 /* 2201 * Special case for sending non GOOD SCSI status from TX thread 2202 * context during pre se_cmd excecution failure. 2203 */ 2204 ret = isert_put_response(conn, cmd); 2205 break; 2206 default: 2207 isert_err("Unknown response state: 0x%02x\n", state); 2208 ret = -EINVAL; 2209 break; 2210 } 2211 2212 return ret; 2213 } 2214 2215 struct rdma_cm_id * 2216 isert_setup_id(struct isert_np *isert_np) 2217 { 2218 struct iscsi_np *np = isert_np->np; 2219 struct rdma_cm_id *id; 2220 struct sockaddr *sa; 2221 int ret; 2222 2223 sa = (struct sockaddr *)&np->np_sockaddr; 2224 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2225 2226 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2227 RDMA_PS_TCP, IB_QPT_RC); 2228 if (IS_ERR(id)) { 2229 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2230 ret = PTR_ERR(id); 2231 goto out; 2232 } 2233 isert_dbg("id %p context %p\n", id, id->context); 2234 2235 /* 2236 * Allow both IPv4 and IPv6 sockets to bind a single port 2237 * at the same time. 2238 */ 2239 ret = rdma_set_afonly(id, 1); 2240 if (ret) { 2241 isert_err("rdma_set_afonly() failed: %d\n", ret); 2242 goto out_id; 2243 } 2244 2245 ret = rdma_bind_addr(id, sa); 2246 if (ret) { 2247 isert_err("rdma_bind_addr() failed: %d\n", ret); 2248 goto out_id; 2249 } 2250 2251 ret = rdma_listen(id, 0); 2252 if (ret) { 2253 isert_err("rdma_listen() failed: %d\n", ret); 2254 goto out_id; 2255 } 2256 2257 return id; 2258 out_id: 2259 rdma_destroy_id(id); 2260 out: 2261 return ERR_PTR(ret); 2262 } 2263 2264 static int 2265 isert_setup_np(struct iscsi_np *np, 2266 struct sockaddr_storage *ksockaddr) 2267 { 2268 struct isert_np *isert_np; 2269 struct rdma_cm_id *isert_lid; 2270 int ret; 2271 2272 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2273 if (!isert_np) 2274 return -ENOMEM; 2275 2276 sema_init(&isert_np->sem, 0); 2277 mutex_init(&isert_np->mutex); 2278 INIT_LIST_HEAD(&isert_np->accepted); 2279 INIT_LIST_HEAD(&isert_np->pending); 2280 isert_np->np = np; 2281 2282 /* 2283 * Setup the np->np_sockaddr from the passed sockaddr setup 2284 * in iscsi_target_configfs.c code.. 2285 */ 2286 memcpy(&np->np_sockaddr, ksockaddr, 2287 sizeof(struct sockaddr_storage)); 2288 2289 isert_lid = isert_setup_id(isert_np); 2290 if (IS_ERR(isert_lid)) { 2291 ret = PTR_ERR(isert_lid); 2292 goto out; 2293 } 2294 2295 isert_np->cm_id = isert_lid; 2296 np->np_context = isert_np; 2297 2298 return 0; 2299 2300 out: 2301 kfree(isert_np); 2302 2303 return ret; 2304 } 2305 2306 static int 2307 isert_rdma_accept(struct isert_conn *isert_conn) 2308 { 2309 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2310 struct rdma_conn_param cp; 2311 int ret; 2312 struct iser_cm_hdr rsp_hdr; 2313 2314 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2315 cp.initiator_depth = isert_conn->initiator_depth; 2316 cp.retry_count = 7; 2317 cp.rnr_retry_count = 7; 2318 2319 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2320 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2321 if (!isert_conn->snd_w_inv) 2322 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2323 cp.private_data = (void *)&rsp_hdr; 2324 cp.private_data_len = sizeof(rsp_hdr); 2325 2326 ret = rdma_accept(cm_id, &cp); 2327 if (ret) { 2328 isert_err("rdma_accept() failed with: %d\n", ret); 2329 return ret; 2330 } 2331 2332 return 0; 2333 } 2334 2335 static int 2336 isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) 2337 { 2338 struct isert_conn *isert_conn = conn->context; 2339 int ret; 2340 2341 isert_info("before login_req comp conn: %p\n", isert_conn); 2342 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2343 if (ret) { 2344 isert_err("isert_conn %p interrupted before got login req\n", 2345 isert_conn); 2346 return ret; 2347 } 2348 reinit_completion(&isert_conn->login_req_comp); 2349 2350 /* 2351 * For login requests after the first PDU, isert_rx_login_req() will 2352 * kick queue_delayed_work(isert_login_wq, &conn->login_work) as 2353 * the packet is received, which turns this callback from 2354 * iscsi_target_do_login_rx() into a NOP. 2355 */ 2356 if (!login->first_request) 2357 return 0; 2358 2359 isert_rx_login_req(isert_conn); 2360 2361 isert_info("before login_comp conn: %p\n", conn); 2362 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2363 if (ret) 2364 return ret; 2365 2366 isert_info("processing login->req: %p\n", login->req); 2367 2368 return 0; 2369 } 2370 2371 static void 2372 isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn, 2373 struct isert_conn *isert_conn) 2374 { 2375 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2376 struct rdma_route *cm_route = &cm_id->route; 2377 2378 conn->login_family = np->np_sockaddr.ss_family; 2379 2380 conn->login_sockaddr = cm_route->addr.dst_addr; 2381 conn->local_sockaddr = cm_route->addr.src_addr; 2382 } 2383 2384 static int 2385 isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn) 2386 { 2387 struct isert_np *isert_np = np->np_context; 2388 struct isert_conn *isert_conn; 2389 int ret; 2390 2391 accept_wait: 2392 ret = down_interruptible(&isert_np->sem); 2393 if (ret) 2394 return -ENODEV; 2395 2396 spin_lock_bh(&np->np_thread_lock); 2397 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2398 spin_unlock_bh(&np->np_thread_lock); 2399 isert_dbg("np_thread_state %d\n", 2400 np->np_thread_state); 2401 /* 2402 * No point in stalling here when np_thread 2403 * is in state RESET/SHUTDOWN/EXIT - bail 2404 */ 2405 return -ENODEV; 2406 } 2407 spin_unlock_bh(&np->np_thread_lock); 2408 2409 mutex_lock(&isert_np->mutex); 2410 if (list_empty(&isert_np->pending)) { 2411 mutex_unlock(&isert_np->mutex); 2412 goto accept_wait; 2413 } 2414 isert_conn = list_first_entry(&isert_np->pending, 2415 struct isert_conn, node); 2416 list_del_init(&isert_conn->node); 2417 mutex_unlock(&isert_np->mutex); 2418 2419 conn->context = isert_conn; 2420 isert_conn->conn = conn; 2421 isert_conn->state = ISER_CONN_BOUND; 2422 2423 isert_set_conn_info(np, conn, isert_conn); 2424 2425 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2426 2427 return 0; 2428 } 2429 2430 static void 2431 isert_free_np(struct iscsi_np *np) 2432 { 2433 struct isert_np *isert_np = np->np_context; 2434 struct isert_conn *isert_conn, *n; 2435 LIST_HEAD(drop_conn_list); 2436 2437 if (isert_np->cm_id) 2438 rdma_destroy_id(isert_np->cm_id); 2439 2440 /* 2441 * FIXME: At this point we don't have a good way to insure 2442 * that at this point we don't have hanging connections that 2443 * completed RDMA establishment but didn't start iscsi login 2444 * process. So work-around this by cleaning up what ever piled 2445 * up in accepted and pending lists. 2446 */ 2447 mutex_lock(&isert_np->mutex); 2448 if (!list_empty(&isert_np->pending)) { 2449 isert_info("Still have isert pending connections\n"); 2450 list_for_each_entry_safe(isert_conn, n, 2451 &isert_np->pending, 2452 node) { 2453 isert_info("cleaning isert_conn %p state (%d)\n", 2454 isert_conn, isert_conn->state); 2455 list_move_tail(&isert_conn->node, &drop_conn_list); 2456 } 2457 } 2458 2459 if (!list_empty(&isert_np->accepted)) { 2460 isert_info("Still have isert accepted connections\n"); 2461 list_for_each_entry_safe(isert_conn, n, 2462 &isert_np->accepted, 2463 node) { 2464 isert_info("cleaning isert_conn %p state (%d)\n", 2465 isert_conn, isert_conn->state); 2466 list_move_tail(&isert_conn->node, &drop_conn_list); 2467 } 2468 } 2469 mutex_unlock(&isert_np->mutex); 2470 2471 list_for_each_entry_safe(isert_conn, n, &drop_conn_list, node) { 2472 list_del_init(&isert_conn->node); 2473 isert_connect_release(isert_conn); 2474 } 2475 2476 np->np_context = NULL; 2477 kfree(isert_np); 2478 } 2479 2480 static void isert_release_work(struct work_struct *work) 2481 { 2482 struct isert_conn *isert_conn = container_of(work, 2483 struct isert_conn, 2484 release_work); 2485 2486 isert_info("Starting release conn %p\n", isert_conn); 2487 2488 mutex_lock(&isert_conn->mutex); 2489 isert_conn->state = ISER_CONN_DOWN; 2490 mutex_unlock(&isert_conn->mutex); 2491 2492 isert_info("Destroying conn %p\n", isert_conn); 2493 isert_put_conn(isert_conn); 2494 } 2495 2496 static void 2497 isert_wait4logout(struct isert_conn *isert_conn) 2498 { 2499 struct iscsit_conn *conn = isert_conn->conn; 2500 2501 isert_info("conn %p\n", isert_conn); 2502 2503 if (isert_conn->logout_posted) { 2504 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2505 wait_for_completion_timeout(&conn->conn_logout_comp, 2506 SECONDS_FOR_LOGOUT_COMP * HZ); 2507 } 2508 } 2509 2510 static void 2511 isert_wait4cmds(struct iscsit_conn *conn) 2512 { 2513 isert_info("iscsit_conn %p\n", conn); 2514 2515 if (conn->sess) { 2516 target_stop_cmd_counter(conn->cmd_cnt); 2517 target_wait_for_cmds(conn->cmd_cnt); 2518 } 2519 } 2520 2521 /** 2522 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2523 * unsolicitate dataout 2524 * @conn: iscsi connection 2525 * 2526 * We might still have commands that are waiting for unsolicited 2527 * dataouts messages. We must put the extra reference on those 2528 * before blocking on the target_wait_for_session_cmds 2529 */ 2530 static void 2531 isert_put_unsol_pending_cmds(struct iscsit_conn *conn) 2532 { 2533 struct iscsit_cmd *cmd, *tmp; 2534 static LIST_HEAD(drop_cmd_list); 2535 2536 spin_lock_bh(&conn->cmd_lock); 2537 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2538 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2539 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2540 (cmd->write_data_done < cmd->se_cmd.data_length)) 2541 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2542 } 2543 spin_unlock_bh(&conn->cmd_lock); 2544 2545 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2546 list_del_init(&cmd->i_conn_node); 2547 if (cmd->i_state != ISTATE_REMOVE) { 2548 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2549 2550 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2551 isert_put_cmd(isert_cmd, true); 2552 } 2553 } 2554 } 2555 2556 static void isert_wait_conn(struct iscsit_conn *conn) 2557 { 2558 struct isert_conn *isert_conn = conn->context; 2559 2560 isert_info("Starting conn %p\n", isert_conn); 2561 2562 mutex_lock(&isert_conn->mutex); 2563 isert_conn_terminate(isert_conn); 2564 mutex_unlock(&isert_conn->mutex); 2565 2566 ib_drain_qp(isert_conn->qp); 2567 isert_put_unsol_pending_cmds(conn); 2568 isert_wait4cmds(conn); 2569 isert_wait4logout(isert_conn); 2570 2571 queue_work(isert_release_wq, &isert_conn->release_work); 2572 } 2573 2574 static void isert_free_conn(struct iscsit_conn *conn) 2575 { 2576 struct isert_conn *isert_conn = conn->context; 2577 2578 ib_drain_qp(isert_conn->qp); 2579 isert_put_conn(isert_conn); 2580 } 2581 2582 static void isert_get_rx_pdu(struct iscsit_conn *conn) 2583 { 2584 struct completion comp; 2585 2586 init_completion(&comp); 2587 2588 wait_for_completion_interruptible(&comp); 2589 } 2590 2591 static struct iscsit_transport iser_target_transport = { 2592 .name = "IB/iSER", 2593 .transport_type = ISCSI_INFINIBAND, 2594 .rdma_shutdown = true, 2595 .priv_size = sizeof(struct isert_cmd), 2596 .owner = THIS_MODULE, 2597 .iscsit_setup_np = isert_setup_np, 2598 .iscsit_accept_np = isert_accept_np, 2599 .iscsit_free_np = isert_free_np, 2600 .iscsit_wait_conn = isert_wait_conn, 2601 .iscsit_free_conn = isert_free_conn, 2602 .iscsit_get_login_rx = isert_get_login_rx, 2603 .iscsit_put_login_tx = isert_put_login_tx, 2604 .iscsit_immediate_queue = isert_immediate_queue, 2605 .iscsit_response_queue = isert_response_queue, 2606 .iscsit_get_dataout = isert_get_dataout, 2607 .iscsit_queue_data_in = isert_put_datain, 2608 .iscsit_queue_status = isert_put_response, 2609 .iscsit_aborted_task = isert_aborted_task, 2610 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2611 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2612 }; 2613 2614 static int __init isert_init(void) 2615 { 2616 isert_login_wq = alloc_workqueue("isert_login_wq", 0, 0); 2617 if (!isert_login_wq) { 2618 isert_err("Unable to allocate isert_login_wq\n"); 2619 return -ENOMEM; 2620 } 2621 2622 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2623 WQ_UNBOUND | WQ_HIGHPRI, 0); 2624 if (!isert_comp_wq) { 2625 isert_err("Unable to allocate isert_comp_wq\n"); 2626 goto destroy_login_wq; 2627 } 2628 2629 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2630 WQ_UNBOUND_MAX_ACTIVE); 2631 if (!isert_release_wq) { 2632 isert_err("Unable to allocate isert_release_wq\n"); 2633 goto destroy_comp_wq; 2634 } 2635 2636 iscsit_register_transport(&iser_target_transport); 2637 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2638 2639 return 0; 2640 2641 destroy_comp_wq: 2642 destroy_workqueue(isert_comp_wq); 2643 destroy_login_wq: 2644 destroy_workqueue(isert_login_wq); 2645 2646 return -ENOMEM; 2647 } 2648 2649 static void __exit isert_exit(void) 2650 { 2651 flush_workqueue(isert_login_wq); 2652 destroy_workqueue(isert_release_wq); 2653 destroy_workqueue(isert_comp_wq); 2654 iscsit_unregister_transport(&iser_target_transport); 2655 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2656 destroy_workqueue(isert_login_wq); 2657 } 2658 2659 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2660 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2661 MODULE_LICENSE("GPL"); 2662 2663 module_init(isert_init); 2664 module_exit(isert_exit); 2665