1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains iSCSI extentions for RDMA (iSER) Verbs 4 * 5 * (c) Copyright 2013 Datera, Inc. 6 * 7 * Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ****************************************************************************/ 10 11 #include <linux/string.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/socket.h> 15 #include <linux/in.h> 16 #include <linux/in6.h> 17 #include <rdma/ib_verbs.h> 18 #include <rdma/ib_cm.h> 19 #include <rdma/rdma_cm.h> 20 #include <target/target_core_base.h> 21 #include <target/target_core_fabric.h> 22 #include <target/iscsi/iscsi_transport.h> 23 #include <linux/semaphore.h> 24 25 #include "ib_isert.h" 26 27 static int isert_debug_level; 28 module_param_named(debug_level, isert_debug_level, int, 0644); 29 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 30 31 static int isert_sg_tablesize_set(const char *val, 32 const struct kernel_param *kp); 33 static const struct kernel_param_ops sg_tablesize_ops = { 34 .set = isert_sg_tablesize_set, 35 .get = param_get_int, 36 }; 37 38 static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE; 39 module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644); 40 MODULE_PARM_DESC(sg_tablesize, 41 "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)"); 42 43 static DEFINE_MUTEX(device_list_mutex); 44 static LIST_HEAD(device_list); 45 static struct workqueue_struct *isert_login_wq; 46 static struct workqueue_struct *isert_comp_wq; 47 static struct workqueue_struct *isert_release_wq; 48 49 static int 50 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd); 51 static int 52 isert_login_post_recv(struct isert_conn *isert_conn); 53 static int 54 isert_rdma_accept(struct isert_conn *isert_conn); 55 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 56 57 static void isert_release_work(struct work_struct *work); 58 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 59 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 61 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 62 static void isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, 63 struct ib_device *ib_dev); 64 65 static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) 66 { 67 int n = 0, ret; 68 69 ret = kstrtoint(val, 10, &n); 70 if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE || 71 n > ISCSI_ISER_MAX_SG_TABLESIZE) 72 return -EINVAL; 73 74 return param_set_int(val, kp); 75 } 76 77 static inline bool 78 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 79 { 80 return (conn->pi_support && 81 cmd->prot_op != TARGET_PROT_NORMAL); 82 } 83 84 static void 85 isert_qp_event_callback(struct ib_event *e, void *context) 86 { 87 struct isert_conn *isert_conn = context; 88 89 isert_err("%s (%d): conn %p\n", 90 ib_event_msg(e->event), e->event, isert_conn); 91 92 switch (e->event) { 93 case IB_EVENT_COMM_EST: 94 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 95 break; 96 default: 97 break; 98 } 99 } 100 101 static struct ib_qp * 102 isert_create_qp(struct isert_conn *isert_conn, 103 struct rdma_cm_id *cma_id) 104 { 105 u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; 106 struct isert_device *device = isert_conn->device; 107 struct ib_device *ib_dev = device->ib_device; 108 struct ib_qp_init_attr attr; 109 int ret, factor; 110 111 isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); 112 if (IS_ERR(isert_conn->cq)) { 113 isert_err("Unable to allocate cq\n"); 114 ret = PTR_ERR(isert_conn->cq); 115 return ERR_PTR(ret); 116 } 117 isert_conn->cq_size = cq_size; 118 119 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 120 attr.event_handler = isert_qp_event_callback; 121 attr.qp_context = isert_conn; 122 attr.send_cq = isert_conn->cq; 123 attr.recv_cq = isert_conn->cq; 124 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 125 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 126 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 127 isert_sg_tablesize); 128 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 129 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 130 attr.cap.max_recv_sge = 1; 131 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 132 attr.qp_type = IB_QPT_RC; 133 if (device->pi_capable) 134 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 135 136 ret = rdma_create_qp(cma_id, device->pd, &attr); 137 if (ret) { 138 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 139 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 140 141 return ERR_PTR(ret); 142 } 143 144 return cma_id->qp; 145 } 146 147 static int 148 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 149 { 150 struct isert_device *device = isert_conn->device; 151 struct ib_device *ib_dev = device->ib_device; 152 struct iser_rx_desc *rx_desc; 153 struct ib_sge *rx_sg; 154 u64 dma_addr; 155 int i, j; 156 157 isert_conn->rx_descs = kzalloc_objs(struct iser_rx_desc, 158 ISERT_QP_MAX_RECV_DTOS); 159 if (!isert_conn->rx_descs) 160 return -ENOMEM; 161 162 rx_desc = isert_conn->rx_descs; 163 164 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 165 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf, 166 ISER_RX_SIZE, DMA_FROM_DEVICE); 167 if (ib_dma_mapping_error(ib_dev, dma_addr)) 168 goto dma_map_fail; 169 170 rx_desc->dma_addr = dma_addr; 171 172 rx_sg = &rx_desc->rx_sg; 173 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc); 174 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 175 rx_sg->lkey = device->pd->local_dma_lkey; 176 rx_desc->rx_cqe.done = isert_recv_done; 177 } 178 179 return 0; 180 181 dma_map_fail: 182 rx_desc = isert_conn->rx_descs; 183 for (j = 0; j < i; j++, rx_desc++) { 184 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 185 ISER_RX_SIZE, DMA_FROM_DEVICE); 186 } 187 kfree(isert_conn->rx_descs); 188 isert_conn->rx_descs = NULL; 189 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 190 return -ENOMEM; 191 } 192 193 static void 194 isert_free_rx_descriptors(struct isert_conn *isert_conn) 195 { 196 struct ib_device *ib_dev = isert_conn->device->ib_device; 197 struct iser_rx_desc *rx_desc; 198 int i; 199 200 if (!isert_conn->rx_descs) 201 return; 202 203 rx_desc = isert_conn->rx_descs; 204 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 205 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 206 ISER_RX_SIZE, DMA_FROM_DEVICE); 207 } 208 209 kfree(isert_conn->rx_descs); 210 isert_conn->rx_descs = NULL; 211 } 212 213 static int 214 isert_create_device_ib_res(struct isert_device *device) 215 { 216 struct ib_device *ib_dev = device->ib_device; 217 int ret; 218 219 isert_dbg("devattr->max_send_sge: %u devattr->max_recv_sge %u\n", 220 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 221 isert_dbg("devattr->max_sge_rd: %u\n", ib_dev->attrs.max_sge_rd); 222 223 device->pd = ib_alloc_pd(ib_dev, 0); 224 if (IS_ERR(device->pd)) { 225 ret = PTR_ERR(device->pd); 226 isert_err("failed to allocate pd, device %p, ret=%d\n", 227 device, ret); 228 return ret; 229 } 230 231 /* Check signature cap */ 232 if (ib_dev->attrs.kernel_cap_flags & IBK_INTEGRITY_HANDOVER) 233 device->pi_capable = true; 234 else 235 device->pi_capable = false; 236 237 return 0; 238 } 239 240 static void 241 isert_free_device_ib_res(struct isert_device *device) 242 { 243 isert_info("device %p\n", device); 244 245 ib_dealloc_pd(device->pd); 246 } 247 248 static void 249 isert_device_put(struct isert_device *device) 250 { 251 mutex_lock(&device_list_mutex); 252 device->refcount--; 253 isert_info("device %p refcount %d\n", device, device->refcount); 254 if (!device->refcount) { 255 isert_free_device_ib_res(device); 256 list_del(&device->dev_node); 257 kfree(device); 258 } 259 mutex_unlock(&device_list_mutex); 260 } 261 262 static struct isert_device * 263 isert_device_get(struct rdma_cm_id *cma_id) 264 { 265 struct isert_device *device; 266 int ret; 267 268 mutex_lock(&device_list_mutex); 269 list_for_each_entry(device, &device_list, dev_node) { 270 if (device->ib_device->node_guid == cma_id->device->node_guid) { 271 device->refcount++; 272 isert_info("Found iser device %p refcount %d\n", 273 device, device->refcount); 274 mutex_unlock(&device_list_mutex); 275 return device; 276 } 277 } 278 279 device = kzalloc_obj(struct isert_device); 280 if (!device) { 281 mutex_unlock(&device_list_mutex); 282 return ERR_PTR(-ENOMEM); 283 } 284 285 INIT_LIST_HEAD(&device->dev_node); 286 287 device->ib_device = cma_id->device; 288 ret = isert_create_device_ib_res(device); 289 if (ret) { 290 kfree(device); 291 mutex_unlock(&device_list_mutex); 292 return ERR_PTR(ret); 293 } 294 295 device->refcount++; 296 list_add_tail(&device->dev_node, &device_list); 297 isert_info("Created a new iser device %p refcount %d\n", 298 device, device->refcount); 299 mutex_unlock(&device_list_mutex); 300 301 return device; 302 } 303 304 static void 305 isert_init_conn(struct isert_conn *isert_conn) 306 { 307 isert_conn->state = ISER_CONN_INIT; 308 INIT_LIST_HEAD(&isert_conn->node); 309 init_completion(&isert_conn->login_comp); 310 init_completion(&isert_conn->login_req_comp); 311 init_waitqueue_head(&isert_conn->rem_wait); 312 kref_init(&isert_conn->kref); 313 mutex_init(&isert_conn->mutex); 314 INIT_WORK(&isert_conn->release_work, isert_release_work); 315 } 316 317 static void 318 isert_free_login_buf(struct isert_conn *isert_conn) 319 { 320 struct ib_device *ib_dev = isert_conn->device->ib_device; 321 322 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 323 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 324 kfree(isert_conn->login_rsp_buf); 325 326 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 327 ISER_RX_SIZE, DMA_FROM_DEVICE); 328 kfree(isert_conn->login_desc); 329 } 330 331 static int 332 isert_alloc_login_buf(struct isert_conn *isert_conn, 333 struct ib_device *ib_dev) 334 { 335 int ret; 336 337 isert_conn->login_desc = kzalloc_obj(*isert_conn->login_desc); 338 if (!isert_conn->login_desc) 339 return -ENOMEM; 340 341 isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev, 342 isert_conn->login_desc->buf, 343 ISER_RX_SIZE, DMA_FROM_DEVICE); 344 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr); 345 if (ret) { 346 isert_err("login_desc dma mapping error: %d\n", ret); 347 isert_conn->login_desc->dma_addr = 0; 348 goto out_free_login_desc; 349 } 350 351 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 352 if (!isert_conn->login_rsp_buf) { 353 ret = -ENOMEM; 354 goto out_unmap_login_desc; 355 } 356 357 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 358 isert_conn->login_rsp_buf, 359 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 360 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 361 if (ret) { 362 isert_err("login_rsp_dma mapping error: %d\n", ret); 363 isert_conn->login_rsp_dma = 0; 364 goto out_free_login_rsp_buf; 365 } 366 367 return 0; 368 369 out_free_login_rsp_buf: 370 kfree(isert_conn->login_rsp_buf); 371 out_unmap_login_desc: 372 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 373 ISER_RX_SIZE, DMA_FROM_DEVICE); 374 out_free_login_desc: 375 kfree(isert_conn->login_desc); 376 return ret; 377 } 378 379 static void 380 isert_set_nego_params(struct isert_conn *isert_conn, 381 struct rdma_conn_param *param) 382 { 383 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 384 385 /* Set max inflight RDMA READ requests */ 386 isert_conn->initiator_depth = min(param->initiator_depth, attr->max_qp_init_rd_atom); 387 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 388 389 if (param->private_data) { 390 u8 flags = *(u8 *)param->private_data; 391 392 /* 393 * use remote invalidation if the both initiator 394 * and the HCA support it 395 */ 396 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 397 (attr->device_cap_flags & 398 IB_DEVICE_MEM_MGT_EXTENSIONS); 399 if (isert_conn->snd_w_inv) 400 isert_info("Using remote invalidation\n"); 401 } 402 } 403 404 static void 405 isert_destroy_qp(struct isert_conn *isert_conn) 406 { 407 ib_destroy_qp(isert_conn->qp); 408 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 409 } 410 411 static int 412 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 413 { 414 struct isert_np *isert_np = cma_id->context; 415 struct iscsi_np *np = isert_np->np; 416 struct isert_conn *isert_conn; 417 struct isert_device *device; 418 int ret = 0; 419 420 spin_lock_bh(&np->np_thread_lock); 421 if (!np->enabled) { 422 spin_unlock_bh(&np->np_thread_lock); 423 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 424 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 425 } 426 spin_unlock_bh(&np->np_thread_lock); 427 428 isert_dbg("cma_id: %p, portal: %p\n", 429 cma_id, cma_id->context); 430 431 isert_conn = kzalloc_obj(struct isert_conn); 432 if (!isert_conn) 433 return -ENOMEM; 434 435 isert_init_conn(isert_conn); 436 isert_conn->cm_id = cma_id; 437 438 device = isert_device_get(cma_id); 439 if (IS_ERR(device)) { 440 ret = PTR_ERR(device); 441 goto out; 442 } 443 isert_conn->device = device; 444 445 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 446 if (ret) 447 goto out_conn_dev; 448 449 isert_set_nego_params(isert_conn, &event->param.conn); 450 451 isert_conn->qp = isert_create_qp(isert_conn, cma_id); 452 if (IS_ERR(isert_conn->qp)) { 453 ret = PTR_ERR(isert_conn->qp); 454 goto out_rsp_dma_map; 455 } 456 457 ret = isert_login_post_recv(isert_conn); 458 if (ret) 459 goto out_destroy_qp; 460 461 ret = isert_rdma_accept(isert_conn); 462 if (ret) 463 goto out_destroy_qp; 464 465 mutex_lock(&isert_np->mutex); 466 list_add_tail(&isert_conn->node, &isert_np->accepted); 467 mutex_unlock(&isert_np->mutex); 468 469 return 0; 470 471 out_destroy_qp: 472 isert_destroy_qp(isert_conn); 473 out_rsp_dma_map: 474 isert_free_login_buf(isert_conn); 475 out_conn_dev: 476 isert_device_put(device); 477 out: 478 kfree(isert_conn); 479 rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 480 return ret; 481 } 482 483 static void 484 isert_connect_release(struct isert_conn *isert_conn) 485 { 486 struct isert_device *device = isert_conn->device; 487 488 isert_dbg("conn %p\n", isert_conn); 489 490 BUG_ON(!device); 491 492 isert_free_rx_descriptors(isert_conn); 493 if (isert_conn->cm_id && 494 !isert_conn->dev_removed) 495 rdma_destroy_id(isert_conn->cm_id); 496 497 if (isert_conn->qp) 498 isert_destroy_qp(isert_conn); 499 500 isert_unmap_tx_desc(&isert_conn->login_tx_desc, device->ib_device); 501 502 if (isert_conn->login_desc) 503 isert_free_login_buf(isert_conn); 504 505 isert_device_put(device); 506 507 if (isert_conn->dev_removed) 508 wake_up_interruptible(&isert_conn->rem_wait); 509 else 510 kfree(isert_conn); 511 } 512 513 static void 514 isert_connected_handler(struct rdma_cm_id *cma_id) 515 { 516 struct isert_conn *isert_conn = cma_id->qp->qp_context; 517 struct isert_np *isert_np = cma_id->context; 518 519 isert_info("conn %p\n", isert_conn); 520 521 mutex_lock(&isert_conn->mutex); 522 isert_conn->state = ISER_CONN_UP; 523 kref_get(&isert_conn->kref); 524 mutex_unlock(&isert_conn->mutex); 525 526 mutex_lock(&isert_np->mutex); 527 list_move_tail(&isert_conn->node, &isert_np->pending); 528 mutex_unlock(&isert_np->mutex); 529 530 isert_info("np %p: Allow accept_np to continue\n", isert_np); 531 up(&isert_np->sem); 532 } 533 534 static void 535 isert_release_kref(struct kref *kref) 536 { 537 struct isert_conn *isert_conn = container_of(kref, 538 struct isert_conn, kref); 539 540 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 541 current->pid); 542 543 isert_connect_release(isert_conn); 544 } 545 546 static void 547 isert_put_conn(struct isert_conn *isert_conn) 548 { 549 kref_put(&isert_conn->kref, isert_release_kref); 550 } 551 552 static void 553 isert_handle_unbound_conn(struct isert_conn *isert_conn) 554 { 555 struct isert_np *isert_np = isert_conn->cm_id->context; 556 557 mutex_lock(&isert_np->mutex); 558 if (!list_empty(&isert_conn->node)) { 559 /* 560 * This means iscsi doesn't know this connection 561 * so schedule a cleanup ourselves 562 */ 563 list_del_init(&isert_conn->node); 564 isert_put_conn(isert_conn); 565 queue_work(isert_release_wq, &isert_conn->release_work); 566 } 567 mutex_unlock(&isert_np->mutex); 568 } 569 570 /** 571 * isert_conn_terminate() - Initiate connection termination 572 * @isert_conn: isert connection struct 573 * 574 * Notes: 575 * In case the connection state is BOUND, move state 576 * to TEMINATING and start teardown sequence (rdma_disconnect). 577 * In case the connection state is UP, complete flush as well. 578 * 579 * This routine must be called with mutex held. Thus it is 580 * safe to call multiple times. 581 */ 582 static void 583 isert_conn_terminate(struct isert_conn *isert_conn) 584 { 585 int err; 586 587 if (isert_conn->state >= ISER_CONN_TERMINATING) 588 return; 589 590 isert_info("Terminating conn %p state %d\n", 591 isert_conn, isert_conn->state); 592 isert_conn->state = ISER_CONN_TERMINATING; 593 err = rdma_disconnect(isert_conn->cm_id); 594 if (err) 595 isert_warn("Failed rdma_disconnect isert_conn %p\n", 596 isert_conn); 597 } 598 599 static int 600 isert_np_cma_handler(struct isert_np *isert_np, 601 enum rdma_cm_event_type event) 602 { 603 isert_dbg("%s (%d): isert np %p\n", 604 rdma_event_msg(event), event, isert_np); 605 606 switch (event) { 607 case RDMA_CM_EVENT_DEVICE_REMOVAL: 608 isert_np->cm_id = NULL; 609 break; 610 case RDMA_CM_EVENT_ADDR_CHANGE: 611 isert_np->cm_id = isert_setup_id(isert_np); 612 if (IS_ERR(isert_np->cm_id)) { 613 isert_err("isert np %p setup id failed: %ld\n", 614 isert_np, PTR_ERR(isert_np->cm_id)); 615 isert_np->cm_id = NULL; 616 } 617 break; 618 default: 619 isert_err("isert np %p Unexpected event %d\n", 620 isert_np, event); 621 } 622 623 return -1; 624 } 625 626 static int 627 isert_disconnected_handler(struct rdma_cm_id *cma_id, 628 enum rdma_cm_event_type event) 629 { 630 struct isert_conn *isert_conn = cma_id->qp->qp_context; 631 632 mutex_lock(&isert_conn->mutex); 633 switch (isert_conn->state) { 634 case ISER_CONN_TERMINATING: 635 break; 636 case ISER_CONN_UP: 637 isert_conn_terminate(isert_conn); 638 ib_drain_qp(isert_conn->qp); 639 isert_handle_unbound_conn(isert_conn); 640 break; 641 case ISER_CONN_BOUND: 642 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 643 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 644 break; 645 default: 646 isert_warn("conn %p terminating in state %d\n", 647 isert_conn, isert_conn->state); 648 } 649 mutex_unlock(&isert_conn->mutex); 650 651 return 0; 652 } 653 654 static int 655 isert_connect_error(struct rdma_cm_id *cma_id) 656 { 657 struct isert_conn *isert_conn = cma_id->qp->qp_context; 658 struct isert_np *isert_np = cma_id->context; 659 660 ib_drain_qp(isert_conn->qp); 661 662 mutex_lock(&isert_np->mutex); 663 list_del_init(&isert_conn->node); 664 mutex_unlock(&isert_np->mutex); 665 isert_conn->cm_id = NULL; 666 isert_put_conn(isert_conn); 667 668 return -1; 669 } 670 671 static int 672 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 673 { 674 struct isert_np *isert_np = cma_id->context; 675 struct isert_conn *isert_conn; 676 int ret = 0; 677 678 isert_info("%s (%d): status %d id %p np %p\n", 679 rdma_event_msg(event->event), event->event, 680 event->status, cma_id, cma_id->context); 681 682 if (isert_np->cm_id == cma_id) 683 return isert_np_cma_handler(cma_id->context, event->event); 684 685 switch (event->event) { 686 case RDMA_CM_EVENT_CONNECT_REQUEST: 687 ret = isert_connect_request(cma_id, event); 688 if (ret) 689 isert_err("failed handle connect request %d\n", ret); 690 break; 691 case RDMA_CM_EVENT_ESTABLISHED: 692 isert_connected_handler(cma_id); 693 break; 694 case RDMA_CM_EVENT_ADDR_CHANGE: 695 case RDMA_CM_EVENT_DISCONNECTED: 696 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 697 ret = isert_disconnected_handler(cma_id, event->event); 698 break; 699 case RDMA_CM_EVENT_DEVICE_REMOVAL: 700 isert_conn = cma_id->qp->qp_context; 701 isert_conn->dev_removed = true; 702 isert_disconnected_handler(cma_id, event->event); 703 wait_event_interruptible(isert_conn->rem_wait, 704 isert_conn->state == ISER_CONN_DOWN); 705 kfree(isert_conn); 706 /* 707 * return non-zero from the callback to destroy 708 * the rdma cm id 709 */ 710 return 1; 711 case RDMA_CM_EVENT_REJECTED: 712 isert_info("Connection rejected: %s\n", 713 rdma_reject_msg(cma_id, event->status)); 714 fallthrough; 715 case RDMA_CM_EVENT_UNREACHABLE: 716 case RDMA_CM_EVENT_CONNECT_ERROR: 717 ret = isert_connect_error(cma_id); 718 break; 719 default: 720 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 721 break; 722 } 723 724 return ret; 725 } 726 727 static int 728 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 729 { 730 struct ib_recv_wr *rx_wr; 731 int i, ret; 732 struct iser_rx_desc *rx_desc; 733 734 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 735 rx_desc = &isert_conn->rx_descs[i]; 736 737 rx_wr->wr_cqe = &rx_desc->rx_cqe; 738 rx_wr->sg_list = &rx_desc->rx_sg; 739 rx_wr->num_sge = 1; 740 rx_wr->next = rx_wr + 1; 741 rx_desc->in_use = false; 742 } 743 rx_wr--; 744 rx_wr->next = NULL; /* mark end of work requests list */ 745 746 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 747 if (ret) 748 isert_err("ib_post_recv() failed with ret: %d\n", ret); 749 750 return ret; 751 } 752 753 static int 754 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 755 { 756 struct ib_recv_wr rx_wr; 757 int ret; 758 759 if (!rx_desc->in_use) { 760 /* 761 * if the descriptor is not in-use we already reposted it 762 * for recv, so just silently return 763 */ 764 return 0; 765 } 766 767 rx_desc->in_use = false; 768 rx_wr.wr_cqe = &rx_desc->rx_cqe; 769 rx_wr.sg_list = &rx_desc->rx_sg; 770 rx_wr.num_sge = 1; 771 rx_wr.next = NULL; 772 773 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 774 if (ret) 775 isert_err("ib_post_recv() failed with ret: %d\n", ret); 776 777 return ret; 778 } 779 780 static int 781 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 782 { 783 struct ib_device *ib_dev = isert_conn->cm_id->device; 784 struct ib_send_wr send_wr; 785 int ret; 786 787 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 788 ISER_HEADERS_LEN, DMA_TO_DEVICE); 789 790 tx_desc->tx_cqe.done = isert_login_send_done; 791 792 send_wr.next = NULL; 793 send_wr.wr_cqe = &tx_desc->tx_cqe; 794 send_wr.sg_list = tx_desc->tx_sg; 795 send_wr.num_sge = tx_desc->num_sge; 796 send_wr.opcode = IB_WR_SEND; 797 send_wr.send_flags = IB_SEND_SIGNALED; 798 799 ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 800 if (ret) 801 isert_err("ib_post_send() failed, ret: %d\n", ret); 802 803 return ret; 804 } 805 806 static void 807 __isert_create_send_desc(struct isert_device *device, 808 struct iser_tx_desc *tx_desc) 809 { 810 811 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 812 tx_desc->iser_header.flags = ISCSI_CTRL; 813 814 tx_desc->num_sge = 1; 815 816 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 817 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 818 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 819 } 820 } 821 822 static void 823 isert_create_send_desc(struct isert_conn *isert_conn, 824 struct isert_cmd *isert_cmd, 825 struct iser_tx_desc *tx_desc) 826 { 827 struct isert_device *device = isert_conn->device; 828 struct ib_device *ib_dev = device->ib_device; 829 830 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 831 ISER_HEADERS_LEN, DMA_TO_DEVICE); 832 833 __isert_create_send_desc(device, tx_desc); 834 } 835 836 static int 837 isert_init_tx_hdrs(struct isert_conn *isert_conn, 838 struct iser_tx_desc *tx_desc) 839 { 840 struct isert_device *device = isert_conn->device; 841 struct ib_device *ib_dev = device->ib_device; 842 u64 dma_addr; 843 844 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 845 ISER_HEADERS_LEN, DMA_TO_DEVICE); 846 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 847 isert_err("ib_dma_mapping_error() failed\n"); 848 return -ENOMEM; 849 } 850 851 tx_desc->dma_addr = dma_addr; 852 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 853 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 854 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 855 856 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 857 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 858 tx_desc->tx_sg[0].lkey); 859 860 return 0; 861 } 862 863 static void 864 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 865 struct ib_send_wr *send_wr) 866 { 867 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 868 869 tx_desc->tx_cqe.done = isert_send_done; 870 send_wr->wr_cqe = &tx_desc->tx_cqe; 871 872 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 873 send_wr->opcode = IB_WR_SEND_WITH_INV; 874 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 875 } else { 876 send_wr->opcode = IB_WR_SEND; 877 } 878 879 send_wr->sg_list = &tx_desc->tx_sg[0]; 880 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 881 send_wr->send_flags = IB_SEND_SIGNALED; 882 } 883 884 static int 885 isert_login_post_recv(struct isert_conn *isert_conn) 886 { 887 struct ib_recv_wr rx_wr; 888 struct ib_sge sge; 889 int ret; 890 891 memset(&sge, 0, sizeof(struct ib_sge)); 892 sge.addr = isert_conn->login_desc->dma_addr + 893 isert_get_hdr_offset(isert_conn->login_desc); 894 sge.length = ISER_RX_PAYLOAD_SIZE; 895 sge.lkey = isert_conn->device->pd->local_dma_lkey; 896 897 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 898 sge.addr, sge.length, sge.lkey); 899 900 isert_conn->login_desc->rx_cqe.done = isert_login_recv_done; 901 902 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 903 rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe; 904 rx_wr.sg_list = &sge; 905 rx_wr.num_sge = 1; 906 907 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 908 if (ret) 909 isert_err("ib_post_recv() failed: %d\n", ret); 910 911 return ret; 912 } 913 914 static int 915 isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, 916 u32 length) 917 { 918 struct isert_conn *isert_conn = conn->context; 919 struct isert_device *device = isert_conn->device; 920 struct ib_device *ib_dev = device->ib_device; 921 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 922 int ret; 923 924 __isert_create_send_desc(device, tx_desc); 925 926 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 927 sizeof(struct iscsi_hdr)); 928 929 isert_init_tx_hdrs(isert_conn, tx_desc); 930 931 if (length > 0) { 932 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 933 934 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 935 length, DMA_TO_DEVICE); 936 937 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 938 939 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 940 length, DMA_TO_DEVICE); 941 942 tx_dsg->addr = isert_conn->login_rsp_dma; 943 tx_dsg->length = length; 944 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 945 tx_desc->num_sge = 2; 946 } 947 if (!login->login_failed) { 948 if (login->login_complete) { 949 /* Posted and sent from isert_get_rx_pdu(). */ 950 isert_conn->login_rsp_pending = true; 951 return 0; 952 } 953 954 ret = isert_login_post_recv(isert_conn); 955 if (ret) 956 return ret; 957 } 958 959 ret = isert_login_post_send(isert_conn, tx_desc); 960 if (ret) 961 return ret; 962 963 return 0; 964 } 965 966 static int 967 isert_check_login_req(struct isert_conn *isert_conn) 968 { 969 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(isert_conn->login_desc); 970 u32 dlength = ntoh24(hdr->dlength); 971 972 if (unlikely(dlength > (u32)isert_conn->login_req_len)) { 973 isert_dbg("login PDU declares %u data bytes but only %d were received\n", 974 dlength, isert_conn->login_req_len); 975 return -EINVAL; 976 } 977 978 return 0; 979 } 980 981 static void 982 isert_rx_login_req(struct isert_conn *isert_conn) 983 { 984 struct iser_rx_desc *rx_desc = isert_conn->login_desc; 985 int rx_buflen = isert_conn->login_req_len; 986 struct iscsit_conn *conn = isert_conn->conn; 987 struct iscsi_login *login = conn->conn_login; 988 int size; 989 990 isert_info("conn %p\n", isert_conn); 991 992 WARN_ON_ONCE(!login); 993 994 if (login->first_request) { 995 struct iscsi_login_req *login_req = 996 (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc); 997 /* 998 * Setup the initial iscsi_login values from the leading 999 * login request PDU. 1000 */ 1001 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1002 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1003 login_req->flags); 1004 login->version_min = login_req->min_version; 1005 login->version_max = login_req->max_version; 1006 memcpy(login->isid, login_req->isid, 6); 1007 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1008 login->init_task_tag = login_req->itt; 1009 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1010 login->cid = be16_to_cpu(login_req->cid); 1011 login->tsih = be16_to_cpu(login_req->tsih); 1012 } 1013 1014 memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN); 1015 1016 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1017 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1018 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1019 MAX_KEY_VALUE_PAIRS); 1020 memcpy(login->req_buf, isert_get_data(rx_desc), size); 1021 1022 if (login->first_request) { 1023 complete(&isert_conn->login_comp); 1024 return; 1025 } 1026 queue_delayed_work(isert_login_wq, &conn->login_work, 0); 1027 } 1028 1029 static struct iscsit_cmd 1030 *isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc) 1031 { 1032 struct isert_conn *isert_conn = conn->context; 1033 struct isert_cmd *isert_cmd; 1034 struct iscsit_cmd *cmd; 1035 1036 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1037 if (!cmd) { 1038 isert_err("Unable to allocate iscsit_cmd + isert_cmd\n"); 1039 return NULL; 1040 } 1041 isert_cmd = iscsit_priv_cmd(cmd); 1042 isert_cmd->conn = isert_conn; 1043 isert_cmd->iscsit_cmd = cmd; 1044 isert_cmd->rx_desc = rx_desc; 1045 1046 return cmd; 1047 } 1048 1049 static int 1050 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1051 struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd, 1052 struct iser_rx_desc *rx_desc, unsigned char *buf) 1053 { 1054 struct iscsit_conn *conn = isert_conn->conn; 1055 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1056 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1057 bool dump_payload = false; 1058 unsigned int data_len; 1059 1060 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1061 if (rc < 0) 1062 return rc; 1063 1064 imm_data = cmd->immediate_data; 1065 imm_data_len = cmd->first_burst_len; 1066 unsol_data = cmd->unsolicited_data; 1067 data_len = cmd->se_cmd.data_length; 1068 1069 if (imm_data && imm_data_len == data_len) 1070 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1071 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1072 if (rc < 0) { 1073 return 0; 1074 } else if (rc > 0) { 1075 dump_payload = true; 1076 goto sequence_cmd; 1077 } 1078 1079 if (!imm_data) 1080 return 0; 1081 1082 if (imm_data_len != data_len) { 1083 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1084 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1085 isert_get_data(rx_desc), imm_data_len); 1086 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1087 sg_nents, imm_data_len); 1088 } else { 1089 sg_init_table(&isert_cmd->sg, 1); 1090 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1091 cmd->se_cmd.t_data_nents = 1; 1092 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc), 1093 imm_data_len); 1094 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1095 imm_data_len); 1096 } 1097 1098 cmd->write_data_done += imm_data_len; 1099 1100 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1101 spin_lock_bh(&cmd->istate_lock); 1102 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1103 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1104 spin_unlock_bh(&cmd->istate_lock); 1105 } 1106 1107 sequence_cmd: 1108 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1109 1110 if (!rc && !dump_payload && unsol_data) 1111 iscsit_set_unsolicited_dataout(cmd); 1112 else if (dump_payload && imm_data) 1113 target_put_sess_cmd(&cmd->se_cmd); 1114 1115 return 0; 1116 } 1117 1118 static int 1119 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1120 struct iser_rx_desc *rx_desc, unsigned char *buf) 1121 { 1122 struct scatterlist *sg_start; 1123 struct iscsit_conn *conn = isert_conn->conn; 1124 struct iscsit_cmd *cmd = NULL; 1125 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1126 u32 unsol_data_len = ntoh24(hdr->dlength); 1127 int rc, sg_nents, sg_off, page_off; 1128 1129 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1130 if (rc < 0) 1131 return rc; 1132 else if (!cmd) 1133 return 0; 1134 /* 1135 * FIXME: Unexpected unsolicited_data out 1136 */ 1137 if (!cmd->unsolicited_data) { 1138 isert_err("Received unexpected solicited data payload\n"); 1139 dump_stack(); 1140 return -1; 1141 } 1142 1143 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1144 "write_data_done: %u, data_length: %u\n", 1145 unsol_data_len, cmd->write_data_done, 1146 cmd->se_cmd.data_length); 1147 1148 sg_off = cmd->write_data_done / PAGE_SIZE; 1149 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1150 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1151 page_off = cmd->write_data_done % PAGE_SIZE; 1152 /* 1153 * FIXME: Non page-aligned unsolicited_data out 1154 */ 1155 if (page_off) { 1156 isert_err("unexpected non-page aligned data payload\n"); 1157 dump_stack(); 1158 return -1; 1159 } 1160 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1161 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1162 sg_nents, isert_get_data(rx_desc), unsol_data_len); 1163 1164 sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc), 1165 unsol_data_len); 1166 1167 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1168 if (rc < 0) 1169 return rc; 1170 1171 /* 1172 * multiple data-outs on the same command can arrive - 1173 * so post the buffer before hand 1174 */ 1175 return isert_post_recv(isert_conn, rx_desc); 1176 } 1177 1178 static int 1179 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1180 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1181 unsigned char *buf) 1182 { 1183 struct iscsit_conn *conn = isert_conn->conn; 1184 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1185 int rc; 1186 1187 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1188 if (rc < 0) 1189 return rc; 1190 /* 1191 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1192 */ 1193 1194 return iscsit_process_nop_out(conn, cmd, hdr); 1195 } 1196 1197 static int 1198 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1199 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1200 struct iscsi_text *hdr) 1201 { 1202 struct iscsit_conn *conn = isert_conn->conn; 1203 u32 payload_length = ntoh24(hdr->dlength); 1204 int rc; 1205 unsigned char *text_in = NULL; 1206 1207 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1208 if (rc < 0) 1209 return rc; 1210 1211 if (payload_length) { 1212 text_in = kzalloc(payload_length, GFP_KERNEL); 1213 if (!text_in) 1214 return -ENOMEM; 1215 } 1216 cmd->text_in_ptr = text_in; 1217 1218 memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length); 1219 1220 return iscsit_process_text_cmd(conn, cmd, hdr); 1221 } 1222 1223 static int 1224 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1225 uint32_t read_stag, uint64_t read_va, 1226 uint32_t write_stag, uint64_t write_va) 1227 { 1228 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1229 struct iscsit_conn *conn = isert_conn->conn; 1230 struct iscsit_cmd *cmd; 1231 struct isert_cmd *isert_cmd; 1232 int ret = -EINVAL; 1233 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1234 1235 if (conn->sess->sess_ops->SessionType && 1236 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1237 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1238 " ignoring\n", opcode); 1239 return 0; 1240 } 1241 1242 switch (opcode) { 1243 case ISCSI_OP_SCSI_CMD: 1244 cmd = isert_allocate_cmd(conn, rx_desc); 1245 if (!cmd) 1246 break; 1247 1248 isert_cmd = iscsit_priv_cmd(cmd); 1249 isert_cmd->read_stag = read_stag; 1250 isert_cmd->read_va = read_va; 1251 isert_cmd->write_stag = write_stag; 1252 isert_cmd->write_va = write_va; 1253 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1254 1255 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1256 rx_desc, (unsigned char *)hdr); 1257 break; 1258 case ISCSI_OP_NOOP_OUT: 1259 cmd = isert_allocate_cmd(conn, rx_desc); 1260 if (!cmd) 1261 break; 1262 1263 isert_cmd = iscsit_priv_cmd(cmd); 1264 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1265 rx_desc, (unsigned char *)hdr); 1266 break; 1267 case ISCSI_OP_SCSI_DATA_OUT: 1268 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1269 (unsigned char *)hdr); 1270 break; 1271 case ISCSI_OP_SCSI_TMFUNC: 1272 cmd = isert_allocate_cmd(conn, rx_desc); 1273 if (!cmd) 1274 break; 1275 1276 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1277 (unsigned char *)hdr); 1278 break; 1279 case ISCSI_OP_LOGOUT: 1280 cmd = isert_allocate_cmd(conn, rx_desc); 1281 if (!cmd) 1282 break; 1283 1284 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1285 break; 1286 case ISCSI_OP_TEXT: 1287 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1288 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1289 else 1290 cmd = isert_allocate_cmd(conn, rx_desc); 1291 1292 if (!cmd) 1293 break; 1294 1295 isert_cmd = iscsit_priv_cmd(cmd); 1296 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1297 rx_desc, (struct iscsi_text *)hdr); 1298 break; 1299 default: 1300 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1301 dump_stack(); 1302 break; 1303 } 1304 1305 return ret; 1306 } 1307 1308 static void 1309 isert_print_wc(struct ib_wc *wc, const char *type) 1310 { 1311 if (wc->status != IB_WC_WR_FLUSH_ERR) 1312 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1313 ib_wc_status_msg(wc->status), wc->status, 1314 wc->vendor_err); 1315 else 1316 isert_dbg("%s failure: %s (%d)\n", type, 1317 ib_wc_status_msg(wc->status), wc->status); 1318 } 1319 1320 static void 1321 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1322 { 1323 struct isert_conn *isert_conn = wc->qp->qp_context; 1324 struct ib_device *ib_dev = isert_conn->cm_id->device; 1325 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1326 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1327 struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc); 1328 uint64_t read_va = 0, write_va = 0; 1329 uint32_t read_stag = 0, write_stag = 0; 1330 1331 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1332 isert_print_wc(wc, "recv"); 1333 if (wc->status != IB_WC_WR_FLUSH_ERR) 1334 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1335 return; 1336 } 1337 1338 rx_desc->in_use = true; 1339 1340 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1341 ISER_RX_SIZE, DMA_FROM_DEVICE); 1342 1343 /* 1344 * The data segment length declared in the BHS is attacker controlled 1345 * and is used further down to read that many bytes out of the fixed 1346 * size receive descriptor, so it has to be checked against the number 1347 * of bytes that were actually received. Comparing without subtracting 1348 * also rejects PDUs shorter than the iSER and iSCSI headers, which 1349 * would otherwise be parsed out of stale descriptor contents. 1350 */ 1351 if (unlikely(wc->byte_len < ISER_HEADERS_LEN + ntoh24(hdr->dlength))) { 1352 isert_err("PDU declares %u data bytes but only %u bytes were received\n", 1353 ntoh24(hdr->dlength), wc->byte_len); 1354 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1355 return; 1356 } 1357 1358 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1359 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1360 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1361 1362 switch (iser_ctrl->flags & 0xF0) { 1363 case ISCSI_CTRL: 1364 if (iser_ctrl->flags & ISER_RSV) { 1365 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1366 read_va = be64_to_cpu(iser_ctrl->read_va); 1367 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1368 read_stag, (unsigned long long)read_va); 1369 } 1370 if (iser_ctrl->flags & ISER_WSV) { 1371 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1372 write_va = be64_to_cpu(iser_ctrl->write_va); 1373 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1374 write_stag, (unsigned long long)write_va); 1375 } 1376 1377 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1378 break; 1379 case ISER_HELLO: 1380 isert_err("iSER Hello message\n"); 1381 break; 1382 default: 1383 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1384 break; 1385 } 1386 1387 isert_rx_opcode(isert_conn, rx_desc, 1388 read_stag, read_va, write_stag, write_va); 1389 1390 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1391 ISER_RX_SIZE, DMA_FROM_DEVICE); 1392 } 1393 1394 static void 1395 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1396 { 1397 struct isert_conn *isert_conn = wc->qp->qp_context; 1398 struct ib_device *ib_dev = isert_conn->device->ib_device; 1399 1400 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1401 isert_print_wc(wc, "login recv"); 1402 return; 1403 } 1404 1405 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr, 1406 ISER_RX_SIZE, DMA_FROM_DEVICE); 1407 1408 if (unlikely(wc->byte_len < ISER_HEADERS_LEN)) { 1409 isert_dbg("login request length %u is too short\n", 1410 wc->byte_len); 1411 return; 1412 } 1413 1414 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1415 1416 if (isert_conn->conn) { 1417 struct iscsi_login *login = isert_conn->conn->conn_login; 1418 1419 if (login && !login->first_request) { 1420 if (isert_check_login_req(isert_conn)) 1421 return; 1422 1423 isert_rx_login_req(isert_conn); 1424 } 1425 } 1426 1427 mutex_lock(&isert_conn->mutex); 1428 complete(&isert_conn->login_req_comp); 1429 mutex_unlock(&isert_conn->mutex); 1430 1431 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr, 1432 ISER_RX_SIZE, DMA_FROM_DEVICE); 1433 } 1434 1435 static void 1436 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1437 { 1438 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 1439 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1440 1441 if (!cmd->rw.nr_ops) 1442 return; 1443 1444 if (isert_prot_cmd(conn, se_cmd)) { 1445 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1446 conn->cm_id->port_num, se_cmd->t_data_sg, 1447 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1448 se_cmd->t_prot_nents, dir); 1449 } else { 1450 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1451 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1452 } 1453 1454 cmd->rw.nr_ops = 0; 1455 } 1456 1457 static void 1458 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1459 { 1460 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1461 struct isert_conn *isert_conn = isert_cmd->conn; 1462 struct iscsit_conn *conn = isert_conn->conn; 1463 struct iscsi_text_rsp *hdr; 1464 1465 isert_dbg("Cmd %p\n", isert_cmd); 1466 1467 switch (cmd->iscsi_opcode) { 1468 case ISCSI_OP_SCSI_CMD: 1469 spin_lock_bh(&conn->cmd_lock); 1470 if (!list_empty(&cmd->i_conn_node)) 1471 list_del_init(&cmd->i_conn_node); 1472 spin_unlock_bh(&conn->cmd_lock); 1473 1474 if (cmd->data_direction == DMA_TO_DEVICE) { 1475 iscsit_stop_dataout_timer(cmd); 1476 /* 1477 * Check for special case during comp_err where 1478 * WRITE_PENDING has been handed off from core, 1479 * but requires an extra target_put_sess_cmd() 1480 * before transport_generic_free_cmd() below. 1481 */ 1482 if (comp_err && 1483 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1484 struct se_cmd *se_cmd = &cmd->se_cmd; 1485 1486 target_put_sess_cmd(se_cmd); 1487 } 1488 } 1489 1490 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1491 transport_generic_free_cmd(&cmd->se_cmd, 0); 1492 break; 1493 case ISCSI_OP_SCSI_TMFUNC: 1494 spin_lock_bh(&conn->cmd_lock); 1495 if (!list_empty(&cmd->i_conn_node)) 1496 list_del_init(&cmd->i_conn_node); 1497 spin_unlock_bh(&conn->cmd_lock); 1498 1499 transport_generic_free_cmd(&cmd->se_cmd, 0); 1500 break; 1501 case ISCSI_OP_REJECT: 1502 case ISCSI_OP_NOOP_OUT: 1503 case ISCSI_OP_TEXT: 1504 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1505 /* If the continue bit is on, keep the command alive */ 1506 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1507 break; 1508 1509 spin_lock_bh(&conn->cmd_lock); 1510 if (!list_empty(&cmd->i_conn_node)) 1511 list_del_init(&cmd->i_conn_node); 1512 spin_unlock_bh(&conn->cmd_lock); 1513 1514 /* 1515 * Handle special case for REJECT when iscsi_add_reject*() has 1516 * overwritten the original iscsi_opcode assignment, and the 1517 * associated cmd->se_cmd needs to be released. 1518 */ 1519 if (cmd->se_cmd.se_tfo != NULL) { 1520 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1521 cmd->iscsi_opcode); 1522 transport_generic_free_cmd(&cmd->se_cmd, 0); 1523 break; 1524 } 1525 fallthrough; 1526 default: 1527 iscsit_release_cmd(cmd); 1528 break; 1529 } 1530 } 1531 1532 static void 1533 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1534 { 1535 if (tx_desc->dma_addr != 0) { 1536 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1537 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1538 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1539 tx_desc->dma_addr = 0; 1540 } 1541 } 1542 1543 static void 1544 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1545 struct ib_device *ib_dev, bool comp_err) 1546 { 1547 if (isert_cmd->pdu_buf_dma != 0) { 1548 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1549 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1550 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1551 isert_cmd->pdu_buf_dma = 0; 1552 } 1553 1554 isert_unmap_tx_desc(tx_desc, ib_dev); 1555 isert_put_cmd(isert_cmd, comp_err); 1556 } 1557 1558 static int 1559 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1560 { 1561 struct ib_mr_status mr_status; 1562 int ret; 1563 1564 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1565 if (ret) { 1566 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1567 goto fail_mr_status; 1568 } 1569 1570 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1571 u64 sec_offset_err; 1572 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1573 1574 switch (mr_status.sig_err.err_type) { 1575 case IB_SIG_BAD_GUARD: 1576 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1577 break; 1578 case IB_SIG_BAD_REFTAG: 1579 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1580 break; 1581 case IB_SIG_BAD_APPTAG: 1582 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1583 break; 1584 } 1585 sec_offset_err = mr_status.sig_err.sig_err_offset; 1586 do_div(sec_offset_err, block_size); 1587 se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba; 1588 1589 isert_err("PI error found type %d at sector 0x%llx " 1590 "expected 0x%x vs actual 0x%x\n", 1591 mr_status.sig_err.err_type, 1592 (unsigned long long)se_cmd->sense_info, 1593 mr_status.sig_err.expected, 1594 mr_status.sig_err.actual); 1595 ret = 1; 1596 } 1597 1598 fail_mr_status: 1599 return ret; 1600 } 1601 1602 static void 1603 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1604 { 1605 struct isert_conn *isert_conn = wc->qp->qp_context; 1606 struct isert_device *device = isert_conn->device; 1607 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1608 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1609 struct se_cmd *cmd = &isert_cmd->iscsit_cmd->se_cmd; 1610 int ret = 0; 1611 1612 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1613 isert_print_wc(wc, "rdma write"); 1614 if (wc->status != IB_WC_WR_FLUSH_ERR) 1615 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1616 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1617 return; 1618 } 1619 1620 isert_dbg("Cmd %p\n", isert_cmd); 1621 1622 ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 1623 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1624 1625 if (ret) { 1626 /* 1627 * transport_generic_request_failure() expects to have 1628 * plus two references to handle queue-full, so re-add 1629 * one here as target-core will have already dropped 1630 * it after the first isert_put_datain() callback. 1631 */ 1632 kref_get(&cmd->cmd_kref); 1633 transport_generic_request_failure(cmd, cmd->pi_err); 1634 } else { 1635 /* 1636 * XXX: isert_put_response() failure is not retried. 1637 */ 1638 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsit_cmd); 1639 if (ret) 1640 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1641 } 1642 } 1643 1644 static void 1645 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1646 { 1647 struct isert_conn *isert_conn = wc->qp->qp_context; 1648 struct isert_device *device = isert_conn->device; 1649 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1650 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1651 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1652 struct se_cmd *se_cmd = &cmd->se_cmd; 1653 int ret = 0; 1654 1655 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1656 isert_print_wc(wc, "rdma read"); 1657 if (wc->status != IB_WC_WR_FLUSH_ERR) 1658 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1659 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1660 return; 1661 } 1662 1663 isert_dbg("Cmd %p\n", isert_cmd); 1664 1665 iscsit_stop_dataout_timer(cmd); 1666 1667 if (isert_prot_cmd(isert_conn, se_cmd)) 1668 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 1669 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1670 cmd->write_data_done = 0; 1671 1672 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1673 spin_lock_bh(&cmd->istate_lock); 1674 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1675 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1676 spin_unlock_bh(&cmd->istate_lock); 1677 1678 /* 1679 * transport_generic_request_failure() will drop the extra 1680 * se_cmd->cmd_kref reference after T10-PI error, and handle 1681 * any non-zero ->queue_status() callback error retries. 1682 */ 1683 if (ret) 1684 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1685 else 1686 target_execute_cmd(se_cmd); 1687 } 1688 1689 static void 1690 isert_do_control_comp(struct work_struct *work) 1691 { 1692 struct isert_cmd *isert_cmd = container_of(work, 1693 struct isert_cmd, comp_work); 1694 struct isert_conn *isert_conn = isert_cmd->conn; 1695 struct ib_device *ib_dev = isert_conn->cm_id->device; 1696 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1697 1698 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1699 1700 switch (cmd->i_state) { 1701 case ISTATE_SEND_TASKMGTRSP: 1702 iscsit_tmr_post_handler(cmd, cmd->conn); 1703 fallthrough; 1704 case ISTATE_SEND_REJECT: 1705 case ISTATE_SEND_TEXTRSP: 1706 cmd->i_state = ISTATE_SENT_STATUS; 1707 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1708 ib_dev, false); 1709 break; 1710 case ISTATE_SEND_LOGOUTRSP: 1711 iscsit_logout_post_handler(cmd, cmd->conn); 1712 break; 1713 default: 1714 isert_err("Unknown i_state %d\n", cmd->i_state); 1715 dump_stack(); 1716 break; 1717 } 1718 } 1719 1720 static void 1721 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1722 { 1723 struct isert_conn *isert_conn = wc->qp->qp_context; 1724 struct ib_device *ib_dev = isert_conn->cm_id->device; 1725 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1726 1727 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1728 isert_print_wc(wc, "login send"); 1729 if (wc->status != IB_WC_WR_FLUSH_ERR) 1730 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1731 } 1732 1733 isert_unmap_tx_desc(tx_desc, ib_dev); 1734 } 1735 1736 static void 1737 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1738 { 1739 struct isert_conn *isert_conn = wc->qp->qp_context; 1740 struct ib_device *ib_dev = isert_conn->cm_id->device; 1741 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1742 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1743 1744 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1745 isert_print_wc(wc, "send"); 1746 if (wc->status != IB_WC_WR_FLUSH_ERR) 1747 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1748 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1749 return; 1750 } 1751 1752 isert_dbg("Cmd %p\n", isert_cmd); 1753 1754 switch (isert_cmd->iscsit_cmd->i_state) { 1755 case ISTATE_SEND_TASKMGTRSP: 1756 case ISTATE_SEND_LOGOUTRSP: 1757 case ISTATE_SEND_REJECT: 1758 case ISTATE_SEND_TEXTRSP: 1759 isert_unmap_tx_desc(tx_desc, ib_dev); 1760 1761 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1762 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1763 return; 1764 default: 1765 isert_cmd->iscsit_cmd->i_state = ISTATE_SENT_STATUS; 1766 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1767 break; 1768 } 1769 } 1770 1771 static int 1772 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1773 { 1774 int ret; 1775 1776 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1777 if (ret) 1778 return ret; 1779 1780 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1781 if (ret) { 1782 isert_err("ib_post_send failed with %d\n", ret); 1783 return ret; 1784 } 1785 return ret; 1786 } 1787 1788 static int 1789 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1790 { 1791 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1792 struct isert_conn *isert_conn = conn->context; 1793 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1794 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1795 &isert_cmd->tx_desc.iscsi_header; 1796 1797 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1798 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1799 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1800 /* 1801 * Attach SENSE DATA payload to iSCSI Response PDU 1802 */ 1803 if (cmd->se_cmd.sense_buffer && 1804 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1805 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1806 struct isert_device *device = isert_conn->device; 1807 struct ib_device *ib_dev = device->ib_device; 1808 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1809 u32 padding, pdu_len; 1810 1811 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1812 cmd->sense_buffer); 1813 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1814 1815 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1816 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1817 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1818 1819 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1820 (void *)cmd->sense_buffer, pdu_len, 1821 DMA_TO_DEVICE); 1822 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1823 return -ENOMEM; 1824 1825 isert_cmd->pdu_buf_len = pdu_len; 1826 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1827 tx_dsg->length = pdu_len; 1828 tx_dsg->lkey = device->pd->local_dma_lkey; 1829 isert_cmd->tx_desc.num_sge = 2; 1830 } 1831 1832 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1833 1834 isert_dbg("Posting SCSI Response\n"); 1835 1836 return isert_post_response(isert_conn, isert_cmd); 1837 } 1838 1839 static void 1840 isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1841 { 1842 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1843 struct isert_conn *isert_conn = conn->context; 1844 1845 spin_lock_bh(&conn->cmd_lock); 1846 if (!list_empty(&cmd->i_conn_node)) 1847 list_del_init(&cmd->i_conn_node); 1848 spin_unlock_bh(&conn->cmd_lock); 1849 1850 if (cmd->data_direction == DMA_TO_DEVICE) 1851 iscsit_stop_dataout_timer(cmd); 1852 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1853 } 1854 1855 static enum target_prot_op 1856 isert_get_sup_prot_ops(struct iscsit_conn *conn) 1857 { 1858 struct isert_conn *isert_conn = conn->context; 1859 struct isert_device *device = isert_conn->device; 1860 1861 if (conn->tpg->tpg_attrib.t10_pi) { 1862 if (device->pi_capable) { 1863 isert_info("conn %p PI offload enabled\n", isert_conn); 1864 isert_conn->pi_support = true; 1865 return TARGET_PROT_ALL; 1866 } 1867 } 1868 1869 isert_info("conn %p PI offload disabled\n", isert_conn); 1870 isert_conn->pi_support = false; 1871 1872 return TARGET_PROT_NORMAL; 1873 } 1874 1875 static int 1876 isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn, 1877 bool nopout_response) 1878 { 1879 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1880 struct isert_conn *isert_conn = conn->context; 1881 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1882 1883 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1884 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1885 &isert_cmd->tx_desc.iscsi_header, 1886 nopout_response); 1887 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1888 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1889 1890 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1891 1892 return isert_post_response(isert_conn, isert_cmd); 1893 } 1894 1895 static int 1896 isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1897 { 1898 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1899 struct isert_conn *isert_conn = conn->context; 1900 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1901 1902 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1903 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1904 &isert_cmd->tx_desc.iscsi_header); 1905 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1906 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1907 1908 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1909 1910 return isert_post_response(isert_conn, isert_cmd); 1911 } 1912 1913 static int 1914 isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1915 { 1916 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1917 struct isert_conn *isert_conn = conn->context; 1918 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1919 1920 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1921 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1922 &isert_cmd->tx_desc.iscsi_header); 1923 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1924 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1925 1926 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1927 1928 return isert_post_response(isert_conn, isert_cmd); 1929 } 1930 1931 static int 1932 isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1933 { 1934 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1935 struct isert_conn *isert_conn = conn->context; 1936 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1937 struct isert_device *device = isert_conn->device; 1938 struct ib_device *ib_dev = device->ib_device; 1939 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1940 struct iscsi_reject *hdr = 1941 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1942 1943 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1944 iscsit_build_reject(cmd, conn, hdr); 1945 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1946 1947 hton24(hdr->dlength, ISCSI_HDR_LEN); 1948 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1949 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1950 DMA_TO_DEVICE); 1951 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1952 return -ENOMEM; 1953 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1954 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1955 tx_dsg->length = ISCSI_HDR_LEN; 1956 tx_dsg->lkey = device->pd->local_dma_lkey; 1957 isert_cmd->tx_desc.num_sge = 2; 1958 1959 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1960 1961 isert_dbg("conn %p Posting Reject\n", isert_conn); 1962 1963 return isert_post_response(isert_conn, isert_cmd); 1964 } 1965 1966 static int 1967 isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1968 { 1969 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1970 struct isert_conn *isert_conn = conn->context; 1971 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1972 struct iscsi_text_rsp *hdr = 1973 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1974 u32 txt_rsp_len; 1975 int rc; 1976 1977 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1978 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1979 if (rc < 0) 1980 return rc; 1981 1982 txt_rsp_len = rc; 1983 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1984 1985 if (txt_rsp_len) { 1986 struct isert_device *device = isert_conn->device; 1987 struct ib_device *ib_dev = device->ib_device; 1988 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1989 void *txt_rsp_buf = cmd->buf_ptr; 1990 1991 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1992 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1993 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1994 return -ENOMEM; 1995 1996 isert_cmd->pdu_buf_len = txt_rsp_len; 1997 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1998 tx_dsg->length = txt_rsp_len; 1999 tx_dsg->lkey = device->pd->local_dma_lkey; 2000 isert_cmd->tx_desc.num_sge = 2; 2001 } 2002 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2003 2004 isert_dbg("conn %p Text Response\n", isert_conn); 2005 2006 return isert_post_response(isert_conn, isert_cmd); 2007 } 2008 2009 static inline void 2010 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 2011 { 2012 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2013 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2014 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2015 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2016 /* 2017 * At the moment we hard code those, but if in the future 2018 * the target core would like to use it, we will take it 2019 * from se_cmd. 2020 */ 2021 domain->sig.dif.apptag_check_mask = 0xffff; 2022 domain->sig.dif.app_escape = true; 2023 domain->sig.dif.ref_escape = true; 2024 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2025 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2026 domain->sig.dif.ref_remap = true; 2027 } 2028 2029 static int 2030 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2031 { 2032 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2033 2034 switch (se_cmd->prot_op) { 2035 case TARGET_PROT_DIN_INSERT: 2036 case TARGET_PROT_DOUT_STRIP: 2037 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2038 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2039 break; 2040 case TARGET_PROT_DOUT_INSERT: 2041 case TARGET_PROT_DIN_STRIP: 2042 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2043 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2044 break; 2045 case TARGET_PROT_DIN_PASS: 2046 case TARGET_PROT_DOUT_PASS: 2047 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2048 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2049 break; 2050 default: 2051 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2052 return -EINVAL; 2053 } 2054 2055 if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 2056 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 2057 if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 2058 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 2059 if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 2060 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 2061 2062 return 0; 2063 } 2064 2065 static int 2066 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2067 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2068 { 2069 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 2070 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2071 u8 port_num = conn->cm_id->port_num; 2072 u64 addr; 2073 u32 rkey, offset; 2074 int ret; 2075 2076 if (cmd->ctx_init_done) 2077 goto rdma_ctx_post; 2078 2079 if (dir == DMA_FROM_DEVICE) { 2080 addr = cmd->write_va; 2081 rkey = cmd->write_stag; 2082 offset = cmd->iscsit_cmd->write_data_done; 2083 } else { 2084 addr = cmd->read_va; 2085 rkey = cmd->read_stag; 2086 offset = 0; 2087 } 2088 2089 if (isert_prot_cmd(conn, se_cmd)) { 2090 struct ib_sig_attrs sig_attrs; 2091 2092 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2093 if (ret) 2094 return ret; 2095 2096 WARN_ON_ONCE(offset); 2097 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2098 se_cmd->t_data_sg, se_cmd->t_data_nents, 2099 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2100 &sig_attrs, addr, rkey, dir); 2101 } else { 2102 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2103 se_cmd->t_data_sg, se_cmd->t_data_nents, 2104 offset, addr, rkey, dir); 2105 } 2106 2107 if (ret < 0) { 2108 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2109 return ret; 2110 } 2111 2112 cmd->ctx_init_done = true; 2113 2114 rdma_ctx_post: 2115 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2116 if (ret < 0) 2117 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2118 return ret; 2119 } 2120 2121 static int 2122 isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 2123 { 2124 struct se_cmd *se_cmd = &cmd->se_cmd; 2125 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2126 struct isert_conn *isert_conn = conn->context; 2127 struct ib_cqe *cqe = NULL; 2128 struct ib_send_wr *chain_wr = NULL; 2129 int rc; 2130 2131 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2132 isert_cmd, se_cmd->data_length); 2133 2134 if (isert_prot_cmd(isert_conn, se_cmd)) { 2135 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2136 cqe = &isert_cmd->tx_desc.tx_cqe; 2137 } else { 2138 /* 2139 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2140 */ 2141 isert_create_send_desc(isert_conn, isert_cmd, 2142 &isert_cmd->tx_desc); 2143 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2144 &isert_cmd->tx_desc.iscsi_header); 2145 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2146 isert_init_send_wr(isert_conn, isert_cmd, 2147 &isert_cmd->tx_desc.send_wr); 2148 2149 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2150 if (rc) 2151 return rc; 2152 2153 chain_wr = &isert_cmd->tx_desc.send_wr; 2154 } 2155 2156 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2157 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2158 isert_cmd, rc); 2159 return rc; 2160 } 2161 2162 static int 2163 isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery) 2164 { 2165 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2166 int ret; 2167 2168 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2169 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2170 2171 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2172 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2173 &isert_cmd->tx_desc.tx_cqe, NULL); 2174 2175 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2176 isert_cmd, ret); 2177 return ret; 2178 } 2179 2180 static int 2181 isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2182 { 2183 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2184 int ret = 0; 2185 2186 switch (state) { 2187 case ISTATE_REMOVE: 2188 spin_lock_bh(&conn->cmd_lock); 2189 list_del_init(&cmd->i_conn_node); 2190 spin_unlock_bh(&conn->cmd_lock); 2191 isert_put_cmd(isert_cmd, true); 2192 break; 2193 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2194 ret = isert_put_nopin(cmd, conn, false); 2195 break; 2196 default: 2197 isert_err("Unknown immediate state: 0x%02x\n", state); 2198 ret = -EINVAL; 2199 break; 2200 } 2201 2202 return ret; 2203 } 2204 2205 static int 2206 isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2207 { 2208 struct isert_conn *isert_conn = conn->context; 2209 int ret; 2210 2211 switch (state) { 2212 case ISTATE_SEND_LOGOUTRSP: 2213 ret = isert_put_logout_rsp(cmd, conn); 2214 if (!ret) 2215 isert_conn->logout_posted = true; 2216 break; 2217 case ISTATE_SEND_NOPIN: 2218 ret = isert_put_nopin(cmd, conn, true); 2219 break; 2220 case ISTATE_SEND_TASKMGTRSP: 2221 ret = isert_put_tm_rsp(cmd, conn); 2222 break; 2223 case ISTATE_SEND_REJECT: 2224 ret = isert_put_reject(cmd, conn); 2225 break; 2226 case ISTATE_SEND_TEXTRSP: 2227 ret = isert_put_text_rsp(cmd, conn); 2228 break; 2229 case ISTATE_SEND_STATUS: 2230 /* 2231 * Special case for sending non GOOD SCSI status from TX thread 2232 * context during pre se_cmd excecution failure. 2233 */ 2234 ret = isert_put_response(conn, cmd); 2235 break; 2236 default: 2237 isert_err("Unknown response state: 0x%02x\n", state); 2238 ret = -EINVAL; 2239 break; 2240 } 2241 2242 return ret; 2243 } 2244 2245 struct rdma_cm_id * 2246 isert_setup_id(struct isert_np *isert_np) 2247 { 2248 struct iscsi_np *np = isert_np->np; 2249 struct rdma_cm_id *id; 2250 struct sockaddr *sa; 2251 int ret; 2252 2253 sa = (struct sockaddr *)&np->np_sockaddr; 2254 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2255 2256 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2257 RDMA_PS_TCP, IB_QPT_RC); 2258 if (IS_ERR(id)) { 2259 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2260 ret = PTR_ERR(id); 2261 goto out; 2262 } 2263 isert_dbg("id %p context %p\n", id, id->context); 2264 2265 /* 2266 * Allow both IPv4 and IPv6 sockets to bind a single port 2267 * at the same time. 2268 */ 2269 ret = rdma_set_afonly(id, 1); 2270 if (ret) { 2271 isert_err("rdma_set_afonly() failed: %d\n", ret); 2272 goto out_id; 2273 } 2274 2275 ret = rdma_bind_addr(id, sa); 2276 if (ret) { 2277 isert_err("rdma_bind_addr() failed: %d\n", ret); 2278 goto out_id; 2279 } 2280 2281 ret = rdma_listen(id, 0); 2282 if (ret) { 2283 isert_err("rdma_listen() failed: %d\n", ret); 2284 goto out_id; 2285 } 2286 2287 return id; 2288 out_id: 2289 rdma_destroy_id(id); 2290 out: 2291 return ERR_PTR(ret); 2292 } 2293 2294 static int 2295 isert_setup_np(struct iscsi_np *np, 2296 struct sockaddr_storage *ksockaddr) 2297 { 2298 struct isert_np *isert_np; 2299 struct rdma_cm_id *isert_lid; 2300 int ret; 2301 2302 isert_np = kzalloc_obj(struct isert_np); 2303 if (!isert_np) 2304 return -ENOMEM; 2305 2306 sema_init(&isert_np->sem, 0); 2307 mutex_init(&isert_np->mutex); 2308 INIT_LIST_HEAD(&isert_np->accepted); 2309 INIT_LIST_HEAD(&isert_np->pending); 2310 isert_np->np = np; 2311 2312 /* 2313 * Setup the np->np_sockaddr from the passed sockaddr setup 2314 * in iscsi_target_configfs.c code.. 2315 */ 2316 memcpy(&np->np_sockaddr, ksockaddr, 2317 sizeof(struct sockaddr_storage)); 2318 2319 isert_lid = isert_setup_id(isert_np); 2320 if (IS_ERR(isert_lid)) { 2321 ret = PTR_ERR(isert_lid); 2322 goto out; 2323 } 2324 2325 isert_np->cm_id = isert_lid; 2326 np->np_context = isert_np; 2327 2328 return 0; 2329 2330 out: 2331 kfree(isert_np); 2332 2333 return ret; 2334 } 2335 2336 static int 2337 isert_rdma_accept(struct isert_conn *isert_conn) 2338 { 2339 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2340 struct rdma_conn_param cp; 2341 int ret; 2342 struct iser_cm_hdr rsp_hdr; 2343 2344 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2345 cp.initiator_depth = isert_conn->initiator_depth; 2346 cp.retry_count = 7; 2347 cp.rnr_retry_count = 7; 2348 2349 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2350 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2351 if (!isert_conn->snd_w_inv) 2352 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2353 cp.private_data = (void *)&rsp_hdr; 2354 cp.private_data_len = sizeof(rsp_hdr); 2355 2356 ret = rdma_accept(cm_id, &cp); 2357 if (ret) { 2358 isert_err("rdma_accept() failed with: %d\n", ret); 2359 return ret; 2360 } 2361 2362 return 0; 2363 } 2364 2365 static int 2366 isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) 2367 { 2368 struct isert_conn *isert_conn = conn->context; 2369 int ret; 2370 2371 isert_info("before login_req comp conn: %p\n", isert_conn); 2372 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2373 if (ret) { 2374 isert_err("isert_conn %p interrupted before got login req\n", 2375 isert_conn); 2376 return ret; 2377 } 2378 reinit_completion(&isert_conn->login_req_comp); 2379 2380 /* 2381 * For login requests after the first PDU, isert_rx_login_req() will 2382 * kick queue_delayed_work(isert_login_wq, &conn->login_work) as 2383 * the packet is received, which turns this callback from 2384 * iscsi_target_do_login_rx() into a NOP. 2385 */ 2386 if (!login->first_request) 2387 return 0; 2388 2389 ret = isert_check_login_req(isert_conn); 2390 if (ret) 2391 return ret; 2392 2393 isert_rx_login_req(isert_conn); 2394 2395 isert_info("before login_comp conn: %p\n", conn); 2396 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2397 if (ret) 2398 return ret; 2399 2400 isert_info("processing login->req: %p\n", login->req); 2401 2402 return 0; 2403 } 2404 2405 static void 2406 isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn, 2407 struct isert_conn *isert_conn) 2408 { 2409 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2410 struct rdma_route *cm_route = &cm_id->route; 2411 2412 conn->login_family = np->np_sockaddr.ss_family; 2413 2414 conn->login_sockaddr = cm_route->addr.dst_addr; 2415 conn->local_sockaddr = cm_route->addr.src_addr; 2416 } 2417 2418 static int 2419 isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn) 2420 { 2421 struct isert_np *isert_np = np->np_context; 2422 struct isert_conn *isert_conn; 2423 int ret; 2424 2425 accept_wait: 2426 ret = down_interruptible(&isert_np->sem); 2427 if (ret) 2428 return -ENODEV; 2429 2430 spin_lock_bh(&np->np_thread_lock); 2431 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2432 spin_unlock_bh(&np->np_thread_lock); 2433 isert_dbg("np_thread_state %d\n", 2434 np->np_thread_state); 2435 /* 2436 * No point in stalling here when np_thread 2437 * is in state RESET/SHUTDOWN/EXIT - bail 2438 */ 2439 return -ENODEV; 2440 } 2441 spin_unlock_bh(&np->np_thread_lock); 2442 2443 mutex_lock(&isert_np->mutex); 2444 if (list_empty(&isert_np->pending)) { 2445 mutex_unlock(&isert_np->mutex); 2446 goto accept_wait; 2447 } 2448 isert_conn = list_first_entry(&isert_np->pending, 2449 struct isert_conn, node); 2450 list_del_init(&isert_conn->node); 2451 mutex_unlock(&isert_np->mutex); 2452 2453 conn->context = isert_conn; 2454 isert_conn->conn = conn; 2455 isert_conn->state = ISER_CONN_BOUND; 2456 2457 isert_set_conn_info(np, conn, isert_conn); 2458 2459 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2460 2461 return 0; 2462 } 2463 2464 static void 2465 isert_free_np(struct iscsi_np *np) 2466 { 2467 struct isert_np *isert_np = np->np_context; 2468 struct isert_conn *isert_conn, *n; 2469 LIST_HEAD(drop_conn_list); 2470 2471 if (isert_np->cm_id) 2472 rdma_destroy_id(isert_np->cm_id); 2473 2474 /* 2475 * FIXME: At this point we don't have a good way to insure 2476 * that at this point we don't have hanging connections that 2477 * completed RDMA establishment but didn't start iscsi login 2478 * process. So work-around this by cleaning up what ever piled 2479 * up in accepted and pending lists. 2480 */ 2481 mutex_lock(&isert_np->mutex); 2482 if (!list_empty(&isert_np->pending)) { 2483 isert_info("Still have isert pending connections\n"); 2484 list_for_each_entry_safe(isert_conn, n, 2485 &isert_np->pending, 2486 node) { 2487 isert_info("cleaning isert_conn %p state (%d)\n", 2488 isert_conn, isert_conn->state); 2489 list_move_tail(&isert_conn->node, &drop_conn_list); 2490 } 2491 } 2492 2493 if (!list_empty(&isert_np->accepted)) { 2494 isert_info("Still have isert accepted connections\n"); 2495 list_for_each_entry_safe(isert_conn, n, 2496 &isert_np->accepted, 2497 node) { 2498 isert_info("cleaning isert_conn %p state (%d)\n", 2499 isert_conn, isert_conn->state); 2500 list_move_tail(&isert_conn->node, &drop_conn_list); 2501 } 2502 } 2503 mutex_unlock(&isert_np->mutex); 2504 2505 list_for_each_entry_safe(isert_conn, n, &drop_conn_list, node) { 2506 list_del_init(&isert_conn->node); 2507 isert_connect_release(isert_conn); 2508 } 2509 2510 np->np_context = NULL; 2511 kfree(isert_np); 2512 } 2513 2514 static void isert_release_work(struct work_struct *work) 2515 { 2516 struct isert_conn *isert_conn = container_of(work, 2517 struct isert_conn, 2518 release_work); 2519 2520 isert_info("Starting release conn %p\n", isert_conn); 2521 2522 mutex_lock(&isert_conn->mutex); 2523 isert_conn->state = ISER_CONN_DOWN; 2524 mutex_unlock(&isert_conn->mutex); 2525 2526 isert_info("Destroying conn %p\n", isert_conn); 2527 isert_put_conn(isert_conn); 2528 } 2529 2530 static void 2531 isert_wait4logout(struct isert_conn *isert_conn) 2532 { 2533 struct iscsit_conn *conn = isert_conn->conn; 2534 2535 isert_info("conn %p\n", isert_conn); 2536 2537 if (isert_conn->logout_posted) { 2538 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2539 wait_for_completion_timeout(&conn->conn_logout_comp, 2540 SECONDS_FOR_LOGOUT_COMP * HZ); 2541 } 2542 } 2543 2544 static void 2545 isert_wait4cmds(struct iscsit_conn *conn) 2546 { 2547 isert_info("iscsit_conn %p\n", conn); 2548 2549 if (conn->sess) { 2550 target_stop_cmd_counter(conn->cmd_cnt); 2551 target_wait_for_cmds(conn->cmd_cnt); 2552 } 2553 } 2554 2555 /** 2556 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2557 * unsolicitate dataout 2558 * @conn: iscsi connection 2559 * 2560 * We might still have commands that are waiting for unsolicited 2561 * dataouts messages. We must put the extra reference on those 2562 * before blocking on the target_wait_for_session_cmds 2563 */ 2564 static void 2565 isert_put_unsol_pending_cmds(struct iscsit_conn *conn) 2566 { 2567 struct iscsit_cmd *cmd, *tmp; 2568 static LIST_HEAD(drop_cmd_list); 2569 2570 spin_lock_bh(&conn->cmd_lock); 2571 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2572 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2573 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2574 (cmd->write_data_done < cmd->se_cmd.data_length)) 2575 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2576 } 2577 spin_unlock_bh(&conn->cmd_lock); 2578 2579 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2580 list_del_init(&cmd->i_conn_node); 2581 if (cmd->i_state != ISTATE_REMOVE) { 2582 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2583 2584 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2585 isert_put_cmd(isert_cmd, true); 2586 } 2587 } 2588 } 2589 2590 static void isert_wait_conn(struct iscsit_conn *conn) 2591 { 2592 struct isert_conn *isert_conn = conn->context; 2593 2594 isert_info("Starting conn %p\n", isert_conn); 2595 2596 mutex_lock(&isert_conn->mutex); 2597 isert_conn_terminate(isert_conn); 2598 mutex_unlock(&isert_conn->mutex); 2599 2600 ib_drain_qp(isert_conn->qp); 2601 isert_put_unsol_pending_cmds(conn); 2602 isert_wait4cmds(conn); 2603 isert_wait4logout(isert_conn); 2604 2605 queue_work(isert_release_wq, &isert_conn->release_work); 2606 } 2607 2608 static void isert_free_conn(struct iscsit_conn *conn) 2609 { 2610 struct isert_conn *isert_conn = conn->context; 2611 2612 ib_drain_qp(isert_conn->qp); 2613 isert_put_conn(isert_conn); 2614 } 2615 2616 static void isert_get_rx_pdu(struct iscsit_conn *conn) 2617 { 2618 struct isert_conn *isert_conn = conn->context; 2619 struct completion comp; 2620 2621 /* The login timeout timer can fail the login after isert_put_login_tx(). */ 2622 if (!isert_conn->login_rsp_pending) 2623 return; 2624 2625 isert_conn->login_rsp_pending = false; 2626 2627 /* The session is registered by now; see isert_put_login_tx(). */ 2628 if (isert_alloc_rx_descriptors(isert_conn)) 2629 return; 2630 2631 if (isert_post_recvm(isert_conn, ISERT_QP_MAX_RECV_DTOS)) 2632 return; 2633 2634 /* Now we are in FULL_FEATURE phase */ 2635 mutex_lock(&isert_conn->mutex); 2636 isert_conn->state = ISER_CONN_FULL_FEATURE; 2637 mutex_unlock(&isert_conn->mutex); 2638 2639 if (isert_login_post_send(isert_conn, &isert_conn->login_tx_desc)) 2640 return; 2641 2642 init_completion(&comp); 2643 2644 wait_for_completion_interruptible(&comp); 2645 } 2646 2647 static struct iscsit_transport iser_target_transport = { 2648 .name = "IB/iSER", 2649 .transport_type = ISCSI_INFINIBAND, 2650 .rdma_shutdown = true, 2651 .priv_size = sizeof(struct isert_cmd), 2652 .owner = THIS_MODULE, 2653 .iscsit_setup_np = isert_setup_np, 2654 .iscsit_accept_np = isert_accept_np, 2655 .iscsit_free_np = isert_free_np, 2656 .iscsit_wait_conn = isert_wait_conn, 2657 .iscsit_free_conn = isert_free_conn, 2658 .iscsit_get_login_rx = isert_get_login_rx, 2659 .iscsit_put_login_tx = isert_put_login_tx, 2660 .iscsit_immediate_queue = isert_immediate_queue, 2661 .iscsit_response_queue = isert_response_queue, 2662 .iscsit_get_dataout = isert_get_dataout, 2663 .iscsit_queue_data_in = isert_put_datain, 2664 .iscsit_queue_status = isert_put_response, 2665 .iscsit_aborted_task = isert_aborted_task, 2666 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2667 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2668 }; 2669 2670 static int __init isert_init(void) 2671 { 2672 isert_login_wq = alloc_workqueue("isert_login_wq", WQ_PERCPU, 0); 2673 if (!isert_login_wq) { 2674 isert_err("Unable to allocate isert_login_wq\n"); 2675 return -ENOMEM; 2676 } 2677 2678 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2679 WQ_UNBOUND | WQ_HIGHPRI, 0); 2680 if (!isert_comp_wq) { 2681 isert_err("Unable to allocate isert_comp_wq\n"); 2682 goto destroy_login_wq; 2683 } 2684 2685 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2686 WQ_UNBOUND_MAX_ACTIVE); 2687 if (!isert_release_wq) { 2688 isert_err("Unable to allocate isert_release_wq\n"); 2689 goto destroy_comp_wq; 2690 } 2691 2692 iscsit_register_transport(&iser_target_transport); 2693 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2694 2695 return 0; 2696 2697 destroy_comp_wq: 2698 destroy_workqueue(isert_comp_wq); 2699 destroy_login_wq: 2700 destroy_workqueue(isert_login_wq); 2701 2702 return -ENOMEM; 2703 } 2704 2705 static void __exit isert_exit(void) 2706 { 2707 flush_workqueue(isert_login_wq); 2708 destroy_workqueue(isert_release_wq); 2709 destroy_workqueue(isert_comp_wq); 2710 iscsit_unregister_transport(&iser_target_transport); 2711 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2712 destroy_workqueue(isert_login_wq); 2713 } 2714 2715 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2716 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2717 MODULE_LICENSE("GPL"); 2718 2719 module_init(isert_init); 2720 module_exit(isert_exit); 2721