1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains iSCSI extentions for RDMA (iSER) Verbs 4 * 5 * (c) Copyright 2013 Datera, Inc. 6 * 7 * Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ****************************************************************************/ 10 11 #include <linux/string.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/socket.h> 15 #include <linux/in.h> 16 #include <linux/in6.h> 17 #include <rdma/ib_verbs.h> 18 #include <rdma/ib_cm.h> 19 #include <rdma/rdma_cm.h> 20 #include <target/target_core_base.h> 21 #include <target/target_core_fabric.h> 22 #include <target/iscsi/iscsi_transport.h> 23 #include <linux/semaphore.h> 24 #include <linux/wait_bit.h> 25 26 #include "ib_isert.h" 27 28 static int isert_debug_level; 29 module_param_named(debug_level, isert_debug_level, int, 0644); 30 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 31 32 static int isert_sg_tablesize_set(const char *val, 33 const struct kernel_param *kp); 34 static const struct kernel_param_ops sg_tablesize_ops = { 35 .set = isert_sg_tablesize_set, 36 .get = param_get_int, 37 }; 38 39 static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE; 40 module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644); 41 MODULE_PARM_DESC(sg_tablesize, 42 "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)"); 43 44 static DEFINE_MUTEX(device_list_mutex); 45 static LIST_HEAD(device_list); 46 static struct workqueue_struct *isert_login_wq; 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static int 51 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd); 52 static int 53 isert_login_post_recv(struct isert_conn *isert_conn); 54 static int 55 isert_rdma_accept(struct isert_conn *isert_conn); 56 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 57 58 static void isert_release_work(struct work_struct *work); 59 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 61 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 62 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 63 static void isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, 64 struct ib_device *ib_dev); 65 66 static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) 67 { 68 int n = 0, ret; 69 70 ret = kstrtoint(val, 10, &n); 71 if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE || 72 n > ISCSI_ISER_MAX_SG_TABLESIZE) 73 return -EINVAL; 74 75 return param_set_int(val, kp); 76 } 77 78 static inline bool 79 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 80 { 81 return (conn->pi_support && 82 cmd->prot_op != TARGET_PROT_NORMAL); 83 } 84 85 static void 86 isert_qp_event_callback(struct ib_event *e, void *context) 87 { 88 struct isert_conn *isert_conn = context; 89 90 isert_err("%s (%d): conn %p\n", 91 ib_event_msg(e->event), e->event, isert_conn); 92 93 switch (e->event) { 94 case IB_EVENT_COMM_EST: 95 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 96 break; 97 default: 98 break; 99 } 100 } 101 102 static struct ib_qp * 103 isert_create_qp(struct isert_conn *isert_conn, 104 struct rdma_cm_id *cma_id) 105 { 106 u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; 107 struct isert_device *device = isert_conn->device; 108 struct ib_device *ib_dev = device->ib_device; 109 struct ib_qp_init_attr attr; 110 int ret, factor; 111 112 isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); 113 if (IS_ERR(isert_conn->cq)) { 114 isert_err("Unable to allocate cq\n"); 115 ret = PTR_ERR(isert_conn->cq); 116 return ERR_PTR(ret); 117 } 118 isert_conn->cq_size = cq_size; 119 120 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 121 attr.event_handler = isert_qp_event_callback; 122 attr.qp_context = isert_conn; 123 attr.send_cq = isert_conn->cq; 124 attr.recv_cq = isert_conn->cq; 125 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 126 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 127 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 128 isert_sg_tablesize); 129 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 130 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 131 attr.cap.max_recv_sge = 1; 132 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 133 attr.qp_type = IB_QPT_RC; 134 if (device->pi_capable) 135 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 136 137 ret = rdma_create_qp(cma_id, device->pd, &attr); 138 if (ret) { 139 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 140 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 141 142 return ERR_PTR(ret); 143 } 144 145 return cma_id->qp; 146 } 147 148 static int 149 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 150 { 151 struct isert_device *device = isert_conn->device; 152 struct ib_device *ib_dev = device->ib_device; 153 struct iser_rx_desc *rx_desc; 154 struct ib_sge *rx_sg; 155 u64 dma_addr; 156 int i, j; 157 158 isert_conn->rx_descs = kzalloc_objs(struct iser_rx_desc, 159 ISERT_QP_MAX_RECV_DTOS); 160 if (!isert_conn->rx_descs) 161 return -ENOMEM; 162 163 rx_desc = isert_conn->rx_descs; 164 165 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 166 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf, 167 ISER_RX_SIZE, DMA_FROM_DEVICE); 168 if (ib_dma_mapping_error(ib_dev, dma_addr)) 169 goto dma_map_fail; 170 171 rx_desc->dma_addr = dma_addr; 172 173 rx_sg = &rx_desc->rx_sg; 174 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc); 175 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 176 rx_sg->lkey = device->pd->local_dma_lkey; 177 rx_desc->rx_cqe.done = isert_recv_done; 178 } 179 180 return 0; 181 182 dma_map_fail: 183 rx_desc = isert_conn->rx_descs; 184 for (j = 0; j < i; j++, rx_desc++) { 185 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 186 ISER_RX_SIZE, DMA_FROM_DEVICE); 187 } 188 kfree(isert_conn->rx_descs); 189 isert_conn->rx_descs = NULL; 190 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 191 return -ENOMEM; 192 } 193 194 static void 195 isert_free_rx_descriptors(struct isert_conn *isert_conn) 196 { 197 struct ib_device *ib_dev = isert_conn->device->ib_device; 198 struct iser_rx_desc *rx_desc; 199 int i; 200 201 if (!isert_conn->rx_descs) 202 return; 203 204 rx_desc = isert_conn->rx_descs; 205 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 206 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 207 ISER_RX_SIZE, DMA_FROM_DEVICE); 208 } 209 210 kfree(isert_conn->rx_descs); 211 isert_conn->rx_descs = NULL; 212 } 213 214 static int 215 isert_create_device_ib_res(struct isert_device *device) 216 { 217 struct ib_device *ib_dev = device->ib_device; 218 int ret; 219 220 isert_dbg("devattr->max_send_sge: %u devattr->max_recv_sge %u\n", 221 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 222 isert_dbg("devattr->max_sge_rd: %u\n", ib_dev->attrs.max_sge_rd); 223 224 device->pd = ib_alloc_pd(ib_dev, 0); 225 if (IS_ERR(device->pd)) { 226 ret = PTR_ERR(device->pd); 227 isert_err("failed to allocate pd, device %p, ret=%d\n", 228 device, ret); 229 return ret; 230 } 231 232 /* Check signature cap */ 233 if (ib_dev->attrs.kernel_cap_flags & IBK_INTEGRITY_HANDOVER) 234 device->pi_capable = true; 235 else 236 device->pi_capable = false; 237 238 return 0; 239 } 240 241 static void 242 isert_free_device_ib_res(struct isert_device *device) 243 { 244 isert_info("device %p\n", device); 245 246 ib_dealloc_pd(device->pd); 247 } 248 249 static void 250 isert_device_put(struct isert_device *device) 251 { 252 mutex_lock(&device_list_mutex); 253 device->refcount--; 254 isert_info("device %p refcount %d\n", device, device->refcount); 255 if (!device->refcount) { 256 isert_free_device_ib_res(device); 257 list_del(&device->dev_node); 258 kfree(device); 259 } 260 mutex_unlock(&device_list_mutex); 261 } 262 263 static struct isert_device * 264 isert_device_get(struct rdma_cm_id *cma_id) 265 { 266 struct isert_device *device; 267 int ret; 268 269 mutex_lock(&device_list_mutex); 270 list_for_each_entry(device, &device_list, dev_node) { 271 if (device->ib_device->node_guid == cma_id->device->node_guid) { 272 device->refcount++; 273 isert_info("Found iser device %p refcount %d\n", 274 device, device->refcount); 275 mutex_unlock(&device_list_mutex); 276 return device; 277 } 278 } 279 280 device = kzalloc_obj(struct isert_device); 281 if (!device) { 282 mutex_unlock(&device_list_mutex); 283 return ERR_PTR(-ENOMEM); 284 } 285 286 INIT_LIST_HEAD(&device->dev_node); 287 288 device->ib_device = cma_id->device; 289 ret = isert_create_device_ib_res(device); 290 if (ret) { 291 kfree(device); 292 mutex_unlock(&device_list_mutex); 293 return ERR_PTR(ret); 294 } 295 296 device->refcount++; 297 list_add_tail(&device->dev_node, &device_list); 298 isert_info("Created a new iser device %p refcount %d\n", 299 device, device->refcount); 300 mutex_unlock(&device_list_mutex); 301 302 return device; 303 } 304 305 static void 306 isert_init_conn(struct isert_conn *isert_conn) 307 { 308 isert_conn->state = ISER_CONN_INIT; 309 INIT_LIST_HEAD(&isert_conn->node); 310 init_completion(&isert_conn->login_comp); 311 init_completion(&isert_conn->login_req_comp); 312 init_waitqueue_head(&isert_conn->rem_wait); 313 kref_init(&isert_conn->kref); 314 atomic_set(&isert_conn->ctrl_comp_cnt, 0); 315 mutex_init(&isert_conn->mutex); 316 INIT_WORK(&isert_conn->release_work, isert_release_work); 317 } 318 319 static void 320 isert_free_login_buf(struct isert_conn *isert_conn) 321 { 322 struct ib_device *ib_dev = isert_conn->device->ib_device; 323 324 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 325 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 326 kfree(isert_conn->login_rsp_buf); 327 328 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 329 ISER_RX_SIZE, DMA_FROM_DEVICE); 330 kfree(isert_conn->login_desc); 331 } 332 333 static int 334 isert_alloc_login_buf(struct isert_conn *isert_conn, 335 struct ib_device *ib_dev) 336 { 337 int ret; 338 339 isert_conn->login_desc = kzalloc_obj(*isert_conn->login_desc); 340 if (!isert_conn->login_desc) 341 return -ENOMEM; 342 343 isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev, 344 isert_conn->login_desc->buf, 345 ISER_RX_SIZE, DMA_FROM_DEVICE); 346 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr); 347 if (ret) { 348 isert_err("login_desc dma mapping error: %d\n", ret); 349 isert_conn->login_desc->dma_addr = 0; 350 goto out_free_login_desc; 351 } 352 353 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 354 if (!isert_conn->login_rsp_buf) { 355 ret = -ENOMEM; 356 goto out_unmap_login_desc; 357 } 358 359 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 360 isert_conn->login_rsp_buf, 361 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 362 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 363 if (ret) { 364 isert_err("login_rsp_dma mapping error: %d\n", ret); 365 isert_conn->login_rsp_dma = 0; 366 goto out_free_login_rsp_buf; 367 } 368 369 return 0; 370 371 out_free_login_rsp_buf: 372 kfree(isert_conn->login_rsp_buf); 373 out_unmap_login_desc: 374 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 375 ISER_RX_SIZE, DMA_FROM_DEVICE); 376 out_free_login_desc: 377 kfree(isert_conn->login_desc); 378 return ret; 379 } 380 381 static void 382 isert_set_nego_params(struct isert_conn *isert_conn, 383 struct rdma_conn_param *param) 384 { 385 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 386 387 /* Set max inflight RDMA READ requests */ 388 isert_conn->initiator_depth = min(param->initiator_depth, attr->max_qp_init_rd_atom); 389 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 390 391 if (param->private_data) { 392 u8 flags = *(u8 *)param->private_data; 393 394 /* 395 * use remote invalidation if the both initiator 396 * and the HCA support it 397 */ 398 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 399 (attr->device_cap_flags & 400 IB_DEVICE_MEM_MGT_EXTENSIONS); 401 if (isert_conn->snd_w_inv) 402 isert_info("Using remote invalidation\n"); 403 } 404 } 405 406 static void 407 isert_destroy_qp(struct isert_conn *isert_conn) 408 { 409 ib_destroy_qp(isert_conn->qp); 410 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 411 } 412 413 static int 414 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 415 { 416 struct isert_np *isert_np = cma_id->context; 417 struct iscsi_np *np = isert_np->np; 418 struct isert_conn *isert_conn; 419 struct isert_device *device; 420 int ret = 0; 421 422 spin_lock_bh(&np->np_thread_lock); 423 if (!np->enabled) { 424 spin_unlock_bh(&np->np_thread_lock); 425 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 426 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 427 } 428 spin_unlock_bh(&np->np_thread_lock); 429 430 isert_dbg("cma_id: %p, portal: %p\n", 431 cma_id, cma_id->context); 432 433 isert_conn = kzalloc_obj(struct isert_conn); 434 if (!isert_conn) 435 return -ENOMEM; 436 437 isert_init_conn(isert_conn); 438 isert_conn->cm_id = cma_id; 439 440 device = isert_device_get(cma_id); 441 if (IS_ERR(device)) { 442 ret = PTR_ERR(device); 443 goto out; 444 } 445 isert_conn->device = device; 446 447 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 448 if (ret) 449 goto out_conn_dev; 450 451 isert_set_nego_params(isert_conn, &event->param.conn); 452 453 isert_conn->qp = isert_create_qp(isert_conn, cma_id); 454 if (IS_ERR(isert_conn->qp)) { 455 ret = PTR_ERR(isert_conn->qp); 456 goto out_rsp_dma_map; 457 } 458 459 ret = isert_login_post_recv(isert_conn); 460 if (ret) 461 goto out_destroy_qp; 462 463 ret = isert_rdma_accept(isert_conn); 464 if (ret) 465 goto out_destroy_qp; 466 467 mutex_lock(&isert_np->mutex); 468 list_add_tail(&isert_conn->node, &isert_np->accepted); 469 mutex_unlock(&isert_np->mutex); 470 471 return 0; 472 473 out_destroy_qp: 474 isert_destroy_qp(isert_conn); 475 out_rsp_dma_map: 476 isert_free_login_buf(isert_conn); 477 out_conn_dev: 478 isert_device_put(device); 479 out: 480 kfree(isert_conn); 481 rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 482 return ret; 483 } 484 485 static void 486 isert_connect_release(struct isert_conn *isert_conn) 487 { 488 struct isert_device *device = isert_conn->device; 489 490 isert_dbg("conn %p\n", isert_conn); 491 492 BUG_ON(!device); 493 494 isert_free_rx_descriptors(isert_conn); 495 if (isert_conn->cm_id && 496 !isert_conn->dev_removed) 497 rdma_destroy_id(isert_conn->cm_id); 498 499 if (isert_conn->qp) 500 isert_destroy_qp(isert_conn); 501 502 isert_unmap_tx_desc(&isert_conn->login_tx_desc, device->ib_device); 503 504 if (isert_conn->login_desc) 505 isert_free_login_buf(isert_conn); 506 507 isert_device_put(device); 508 509 if (isert_conn->dev_removed) 510 wake_up_interruptible(&isert_conn->rem_wait); 511 else 512 kfree(isert_conn); 513 } 514 515 static void 516 isert_connected_handler(struct rdma_cm_id *cma_id) 517 { 518 struct isert_conn *isert_conn = cma_id->qp->qp_context; 519 struct isert_np *isert_np = cma_id->context; 520 521 isert_info("conn %p\n", isert_conn); 522 523 mutex_lock(&isert_conn->mutex); 524 isert_conn->state = ISER_CONN_UP; 525 kref_get(&isert_conn->kref); 526 mutex_unlock(&isert_conn->mutex); 527 528 mutex_lock(&isert_np->mutex); 529 list_move_tail(&isert_conn->node, &isert_np->pending); 530 mutex_unlock(&isert_np->mutex); 531 532 isert_info("np %p: Allow accept_np to continue\n", isert_np); 533 up(&isert_np->sem); 534 } 535 536 static void 537 isert_release_kref(struct kref *kref) 538 { 539 struct isert_conn *isert_conn = container_of(kref, 540 struct isert_conn, kref); 541 542 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 543 current->pid); 544 545 isert_connect_release(isert_conn); 546 } 547 548 static void 549 isert_put_conn(struct isert_conn *isert_conn) 550 { 551 kref_put(&isert_conn->kref, isert_release_kref); 552 } 553 554 static void 555 isert_handle_unbound_conn(struct isert_conn *isert_conn) 556 { 557 struct isert_np *isert_np = isert_conn->cm_id->context; 558 559 mutex_lock(&isert_np->mutex); 560 if (!list_empty(&isert_conn->node)) { 561 /* 562 * This means iscsi doesn't know this connection 563 * so schedule a cleanup ourselves 564 */ 565 list_del_init(&isert_conn->node); 566 isert_put_conn(isert_conn); 567 queue_work(isert_release_wq, &isert_conn->release_work); 568 } 569 mutex_unlock(&isert_np->mutex); 570 } 571 572 /** 573 * isert_conn_terminate() - Initiate connection termination 574 * @isert_conn: isert connection struct 575 * 576 * Notes: 577 * In case the connection state is BOUND, move state 578 * to TEMINATING and start teardown sequence (rdma_disconnect). 579 * In case the connection state is UP, complete flush as well. 580 * 581 * This routine must be called with mutex held. Thus it is 582 * safe to call multiple times. 583 */ 584 static void 585 isert_conn_terminate(struct isert_conn *isert_conn) 586 { 587 int err; 588 589 if (isert_conn->state >= ISER_CONN_TERMINATING) 590 return; 591 592 isert_info("Terminating conn %p state %d\n", 593 isert_conn, isert_conn->state); 594 isert_conn->state = ISER_CONN_TERMINATING; 595 err = rdma_disconnect(isert_conn->cm_id); 596 if (err) 597 isert_warn("Failed rdma_disconnect isert_conn %p\n", 598 isert_conn); 599 } 600 601 static int 602 isert_np_cma_handler(struct isert_np *isert_np, 603 enum rdma_cm_event_type event) 604 { 605 isert_dbg("%s (%d): isert np %p\n", 606 rdma_event_msg(event), event, isert_np); 607 608 switch (event) { 609 case RDMA_CM_EVENT_DEVICE_REMOVAL: 610 isert_np->cm_id = NULL; 611 break; 612 case RDMA_CM_EVENT_ADDR_CHANGE: 613 isert_np->cm_id = isert_setup_id(isert_np); 614 if (IS_ERR(isert_np->cm_id)) { 615 isert_err("isert np %p setup id failed: %ld\n", 616 isert_np, PTR_ERR(isert_np->cm_id)); 617 isert_np->cm_id = NULL; 618 } 619 break; 620 default: 621 isert_err("isert np %p Unexpected event %d\n", 622 isert_np, event); 623 } 624 625 return -1; 626 } 627 628 static int 629 isert_disconnected_handler(struct rdma_cm_id *cma_id, 630 enum rdma_cm_event_type event) 631 { 632 struct isert_conn *isert_conn = cma_id->qp->qp_context; 633 634 mutex_lock(&isert_conn->mutex); 635 switch (isert_conn->state) { 636 case ISER_CONN_TERMINATING: 637 break; 638 case ISER_CONN_UP: 639 isert_conn_terminate(isert_conn); 640 ib_drain_qp(isert_conn->qp); 641 isert_handle_unbound_conn(isert_conn); 642 break; 643 case ISER_CONN_BOUND: 644 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 645 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 646 break; 647 default: 648 isert_warn("conn %p terminating in state %d\n", 649 isert_conn, isert_conn->state); 650 } 651 mutex_unlock(&isert_conn->mutex); 652 653 return 0; 654 } 655 656 static int 657 isert_connect_error(struct rdma_cm_id *cma_id) 658 { 659 struct isert_conn *isert_conn = cma_id->qp->qp_context; 660 struct isert_np *isert_np = cma_id->context; 661 662 ib_drain_qp(isert_conn->qp); 663 664 mutex_lock(&isert_np->mutex); 665 list_del_init(&isert_conn->node); 666 mutex_unlock(&isert_np->mutex); 667 isert_conn->cm_id = NULL; 668 isert_put_conn(isert_conn); 669 670 return -1; 671 } 672 673 static int 674 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 675 { 676 struct isert_np *isert_np = cma_id->context; 677 struct isert_conn *isert_conn; 678 int ret = 0; 679 680 isert_info("%s (%d): status %d id %p np %p\n", 681 rdma_event_msg(event->event), event->event, 682 event->status, cma_id, cma_id->context); 683 684 if (isert_np->cm_id == cma_id) 685 return isert_np_cma_handler(cma_id->context, event->event); 686 687 switch (event->event) { 688 case RDMA_CM_EVENT_CONNECT_REQUEST: 689 ret = isert_connect_request(cma_id, event); 690 if (ret) 691 isert_err("failed handle connect request %d\n", ret); 692 break; 693 case RDMA_CM_EVENT_ESTABLISHED: 694 isert_connected_handler(cma_id); 695 break; 696 case RDMA_CM_EVENT_ADDR_CHANGE: 697 case RDMA_CM_EVENT_DISCONNECTED: 698 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 699 ret = isert_disconnected_handler(cma_id, event->event); 700 break; 701 case RDMA_CM_EVENT_DEVICE_REMOVAL: 702 isert_conn = cma_id->qp->qp_context; 703 isert_conn->dev_removed = true; 704 isert_disconnected_handler(cma_id, event->event); 705 wait_event_interruptible(isert_conn->rem_wait, 706 isert_conn->state == ISER_CONN_DOWN); 707 kfree(isert_conn); 708 /* 709 * return non-zero from the callback to destroy 710 * the rdma cm id 711 */ 712 return 1; 713 case RDMA_CM_EVENT_REJECTED: 714 isert_info("Connection rejected: %s\n", 715 rdma_reject_msg(cma_id, event->status)); 716 fallthrough; 717 case RDMA_CM_EVENT_UNREACHABLE: 718 case RDMA_CM_EVENT_CONNECT_ERROR: 719 ret = isert_connect_error(cma_id); 720 break; 721 default: 722 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 723 break; 724 } 725 726 return ret; 727 } 728 729 static int 730 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 731 { 732 struct ib_recv_wr *rx_wr; 733 int i, ret; 734 struct iser_rx_desc *rx_desc; 735 736 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 737 rx_desc = &isert_conn->rx_descs[i]; 738 739 rx_wr->wr_cqe = &rx_desc->rx_cqe; 740 rx_wr->sg_list = &rx_desc->rx_sg; 741 rx_wr->num_sge = 1; 742 rx_wr->next = rx_wr + 1; 743 rx_desc->in_use = false; 744 } 745 rx_wr--; 746 rx_wr->next = NULL; /* mark end of work requests list */ 747 748 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 749 if (ret) 750 isert_err("ib_post_recv() failed with ret: %d\n", ret); 751 752 return ret; 753 } 754 755 static int 756 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 757 { 758 struct ib_recv_wr rx_wr; 759 int ret; 760 761 if (!rx_desc->in_use) { 762 /* 763 * if the descriptor is not in-use we already reposted it 764 * for recv, so just silently return 765 */ 766 return 0; 767 } 768 769 rx_desc->in_use = false; 770 rx_wr.wr_cqe = &rx_desc->rx_cqe; 771 rx_wr.sg_list = &rx_desc->rx_sg; 772 rx_wr.num_sge = 1; 773 rx_wr.next = NULL; 774 775 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 776 if (ret) 777 isert_err("ib_post_recv() failed with ret: %d\n", ret); 778 779 return ret; 780 } 781 782 static int 783 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 784 { 785 struct ib_device *ib_dev = isert_conn->cm_id->device; 786 struct ib_send_wr send_wr; 787 int ret; 788 789 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 790 ISER_HEADERS_LEN, DMA_TO_DEVICE); 791 792 tx_desc->tx_cqe.done = isert_login_send_done; 793 794 send_wr.next = NULL; 795 send_wr.wr_cqe = &tx_desc->tx_cqe; 796 send_wr.sg_list = tx_desc->tx_sg; 797 send_wr.num_sge = tx_desc->num_sge; 798 send_wr.opcode = IB_WR_SEND; 799 send_wr.send_flags = IB_SEND_SIGNALED; 800 801 ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 802 if (ret) 803 isert_err("ib_post_send() failed, ret: %d\n", ret); 804 805 return ret; 806 } 807 808 static void 809 __isert_create_send_desc(struct isert_device *device, 810 struct iser_tx_desc *tx_desc) 811 { 812 813 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 814 tx_desc->iser_header.flags = ISCSI_CTRL; 815 816 tx_desc->num_sge = 1; 817 818 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 819 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 820 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 821 } 822 } 823 824 static void 825 isert_create_send_desc(struct isert_conn *isert_conn, 826 struct isert_cmd *isert_cmd, 827 struct iser_tx_desc *tx_desc) 828 { 829 struct isert_device *device = isert_conn->device; 830 struct ib_device *ib_dev = device->ib_device; 831 832 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 833 ISER_HEADERS_LEN, DMA_TO_DEVICE); 834 835 __isert_create_send_desc(device, tx_desc); 836 } 837 838 static int 839 isert_init_tx_hdrs(struct isert_conn *isert_conn, 840 struct iser_tx_desc *tx_desc) 841 { 842 struct isert_device *device = isert_conn->device; 843 struct ib_device *ib_dev = device->ib_device; 844 u64 dma_addr; 845 846 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 847 ISER_HEADERS_LEN, DMA_TO_DEVICE); 848 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 849 isert_err("ib_dma_mapping_error() failed\n"); 850 return -ENOMEM; 851 } 852 853 tx_desc->dma_addr = dma_addr; 854 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 855 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 856 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 857 858 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 859 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 860 tx_desc->tx_sg[0].lkey); 861 862 return 0; 863 } 864 865 static void 866 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 867 struct ib_send_wr *send_wr) 868 { 869 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 870 871 tx_desc->tx_cqe.done = isert_send_done; 872 send_wr->wr_cqe = &tx_desc->tx_cqe; 873 874 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 875 send_wr->opcode = IB_WR_SEND_WITH_INV; 876 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 877 } else { 878 send_wr->opcode = IB_WR_SEND; 879 } 880 881 send_wr->sg_list = &tx_desc->tx_sg[0]; 882 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 883 send_wr->send_flags = IB_SEND_SIGNALED; 884 } 885 886 static int 887 isert_login_post_recv(struct isert_conn *isert_conn) 888 { 889 struct ib_recv_wr rx_wr; 890 struct ib_sge sge; 891 int ret; 892 893 memset(&sge, 0, sizeof(struct ib_sge)); 894 sge.addr = isert_conn->login_desc->dma_addr + 895 isert_get_hdr_offset(isert_conn->login_desc); 896 sge.length = ISER_RX_PAYLOAD_SIZE; 897 sge.lkey = isert_conn->device->pd->local_dma_lkey; 898 899 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 900 sge.addr, sge.length, sge.lkey); 901 902 isert_conn->login_desc->rx_cqe.done = isert_login_recv_done; 903 904 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 905 rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe; 906 rx_wr.sg_list = &sge; 907 rx_wr.num_sge = 1; 908 909 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 910 if (ret) 911 isert_err("ib_post_recv() failed: %d\n", ret); 912 913 return ret; 914 } 915 916 static int 917 isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, 918 u32 length) 919 { 920 struct isert_conn *isert_conn = conn->context; 921 struct isert_device *device = isert_conn->device; 922 struct ib_device *ib_dev = device->ib_device; 923 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 924 int ret; 925 926 __isert_create_send_desc(device, tx_desc); 927 928 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 929 sizeof(struct iscsi_hdr)); 930 931 isert_init_tx_hdrs(isert_conn, tx_desc); 932 933 if (length > 0) { 934 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 935 936 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 937 length, DMA_TO_DEVICE); 938 939 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 940 941 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 942 length, DMA_TO_DEVICE); 943 944 tx_dsg->addr = isert_conn->login_rsp_dma; 945 tx_dsg->length = length; 946 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 947 tx_desc->num_sge = 2; 948 } 949 if (!login->login_failed) { 950 if (login->login_complete) { 951 /* Posted and sent from isert_get_rx_pdu(). */ 952 isert_conn->login_rsp_pending = true; 953 return 0; 954 } 955 956 ret = isert_login_post_recv(isert_conn); 957 if (ret) 958 return ret; 959 } 960 961 ret = isert_login_post_send(isert_conn, tx_desc); 962 if (ret) 963 return ret; 964 965 return 0; 966 } 967 968 static int 969 isert_check_login_req(struct isert_conn *isert_conn) 970 { 971 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(isert_conn->login_desc); 972 u32 dlength = ntoh24(hdr->dlength); 973 974 if (unlikely(dlength > (u32)isert_conn->login_req_len)) { 975 isert_dbg("login PDU declares %u data bytes but only %d were received\n", 976 dlength, isert_conn->login_req_len); 977 return -EINVAL; 978 } 979 980 return 0; 981 } 982 983 static void 984 isert_rx_login_req(struct isert_conn *isert_conn) 985 { 986 struct iser_rx_desc *rx_desc = isert_conn->login_desc; 987 int rx_buflen = isert_conn->login_req_len; 988 struct iscsit_conn *conn = isert_conn->conn; 989 struct iscsi_login *login = conn->conn_login; 990 int size; 991 992 isert_info("conn %p\n", isert_conn); 993 994 WARN_ON_ONCE(!login); 995 996 if (login->first_request) { 997 struct iscsi_login_req *login_req = 998 (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc); 999 /* 1000 * Setup the initial iscsi_login values from the leading 1001 * login request PDU. 1002 */ 1003 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1004 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1005 login_req->flags); 1006 login->version_min = login_req->min_version; 1007 login->version_max = login_req->max_version; 1008 memcpy(login->isid, login_req->isid, 6); 1009 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1010 login->init_task_tag = login_req->itt; 1011 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1012 login->cid = be16_to_cpu(login_req->cid); 1013 login->tsih = be16_to_cpu(login_req->tsih); 1014 } 1015 1016 memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN); 1017 1018 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1019 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1020 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1021 MAX_KEY_VALUE_PAIRS); 1022 memcpy(login->req_buf, isert_get_data(rx_desc), size); 1023 1024 if (login->first_request) { 1025 complete(&isert_conn->login_comp); 1026 return; 1027 } 1028 queue_delayed_work(isert_login_wq, &conn->login_work, 0); 1029 } 1030 1031 static struct iscsit_cmd 1032 *isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc) 1033 { 1034 struct isert_conn *isert_conn = conn->context; 1035 struct isert_cmd *isert_cmd; 1036 struct iscsit_cmd *cmd; 1037 1038 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1039 if (!cmd) { 1040 isert_err("Unable to allocate iscsit_cmd + isert_cmd\n"); 1041 return NULL; 1042 } 1043 isert_cmd = iscsit_priv_cmd(cmd); 1044 isert_cmd->conn = isert_conn; 1045 isert_cmd->iscsit_cmd = cmd; 1046 isert_cmd->rx_desc = rx_desc; 1047 1048 return cmd; 1049 } 1050 1051 static int 1052 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1053 struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd, 1054 struct iser_rx_desc *rx_desc, unsigned char *buf) 1055 { 1056 struct iscsit_conn *conn = isert_conn->conn; 1057 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1058 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1059 bool dump_payload = false; 1060 unsigned int data_len; 1061 1062 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1063 if (rc < 0) 1064 return rc; 1065 1066 imm_data = cmd->immediate_data; 1067 imm_data_len = cmd->first_burst_len; 1068 unsol_data = cmd->unsolicited_data; 1069 data_len = cmd->se_cmd.data_length; 1070 1071 if (imm_data && imm_data_len == data_len) 1072 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1073 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1074 if (rc < 0) { 1075 return 0; 1076 } else if (rc > 0) { 1077 dump_payload = true; 1078 goto sequence_cmd; 1079 } 1080 1081 if (!imm_data) 1082 return 0; 1083 1084 if (imm_data_len != data_len) { 1085 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1086 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1087 isert_get_data(rx_desc), imm_data_len); 1088 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1089 sg_nents, imm_data_len); 1090 } else { 1091 sg_init_table(&isert_cmd->sg, 1); 1092 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1093 cmd->se_cmd.t_data_nents = 1; 1094 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc), 1095 imm_data_len); 1096 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1097 imm_data_len); 1098 } 1099 1100 cmd->write_data_done += imm_data_len; 1101 1102 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1103 spin_lock_bh(&cmd->istate_lock); 1104 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1105 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1106 spin_unlock_bh(&cmd->istate_lock); 1107 } 1108 1109 sequence_cmd: 1110 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1111 1112 if (!rc && !dump_payload && unsol_data) 1113 iscsit_set_unsolicited_dataout(cmd); 1114 else if (dump_payload && imm_data) 1115 target_put_sess_cmd(&cmd->se_cmd); 1116 1117 return 0; 1118 } 1119 1120 static int 1121 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1122 struct iser_rx_desc *rx_desc, unsigned char *buf) 1123 { 1124 struct scatterlist *sg_start; 1125 struct iscsit_conn *conn = isert_conn->conn; 1126 struct iscsit_cmd *cmd = NULL; 1127 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1128 u32 unsol_data_len = ntoh24(hdr->dlength); 1129 int rc, sg_nents, sg_off, page_off; 1130 1131 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1132 if (rc < 0) 1133 return rc; 1134 else if (!cmd) 1135 return 0; 1136 /* 1137 * FIXME: Unexpected unsolicited_data out 1138 */ 1139 if (!cmd->unsolicited_data) { 1140 isert_err("Received unexpected solicited data payload\n"); 1141 dump_stack(); 1142 return -1; 1143 } 1144 1145 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1146 "write_data_done: %u, data_length: %u\n", 1147 unsol_data_len, cmd->write_data_done, 1148 cmd->se_cmd.data_length); 1149 1150 sg_off = cmd->write_data_done / PAGE_SIZE; 1151 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1152 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1153 page_off = cmd->write_data_done % PAGE_SIZE; 1154 /* 1155 * FIXME: Non page-aligned unsolicited_data out 1156 */ 1157 if (page_off) { 1158 isert_err("unexpected non-page aligned data payload\n"); 1159 dump_stack(); 1160 return -1; 1161 } 1162 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1163 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1164 sg_nents, isert_get_data(rx_desc), unsol_data_len); 1165 1166 sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc), 1167 unsol_data_len); 1168 1169 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1170 if (rc < 0) 1171 return rc; 1172 1173 /* 1174 * multiple data-outs on the same command can arrive - 1175 * so post the buffer before hand 1176 */ 1177 return isert_post_recv(isert_conn, rx_desc); 1178 } 1179 1180 static int 1181 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1182 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1183 unsigned char *buf) 1184 { 1185 struct iscsit_conn *conn = isert_conn->conn; 1186 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1187 int rc; 1188 1189 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1190 if (rc < 0) 1191 return rc; 1192 /* 1193 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1194 */ 1195 1196 return iscsit_process_nop_out(conn, cmd, hdr); 1197 } 1198 1199 static int 1200 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1201 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1202 struct iscsi_text *hdr) 1203 { 1204 struct iscsit_conn *conn = isert_conn->conn; 1205 u32 payload_length = ntoh24(hdr->dlength); 1206 int rc; 1207 unsigned char *text_in = NULL; 1208 1209 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1210 if (rc < 0) 1211 return rc; 1212 1213 if (payload_length) { 1214 text_in = kzalloc(payload_length, GFP_KERNEL); 1215 if (!text_in) 1216 return -ENOMEM; 1217 } 1218 cmd->text_in_ptr = text_in; 1219 1220 memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length); 1221 1222 return iscsit_process_text_cmd(conn, cmd, hdr); 1223 } 1224 1225 static int 1226 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1227 uint32_t read_stag, uint64_t read_va, 1228 uint32_t write_stag, uint64_t write_va) 1229 { 1230 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1231 struct iscsit_conn *conn = isert_conn->conn; 1232 struct iscsit_cmd *cmd; 1233 struct isert_cmd *isert_cmd; 1234 int ret = -EINVAL; 1235 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1236 1237 if (conn->sess->sess_ops->SessionType && 1238 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1239 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1240 " ignoring\n", opcode); 1241 return 0; 1242 } 1243 1244 switch (opcode) { 1245 case ISCSI_OP_SCSI_CMD: 1246 cmd = isert_allocate_cmd(conn, rx_desc); 1247 if (!cmd) 1248 break; 1249 1250 isert_cmd = iscsit_priv_cmd(cmd); 1251 isert_cmd->read_stag = read_stag; 1252 isert_cmd->read_va = read_va; 1253 isert_cmd->write_stag = write_stag; 1254 isert_cmd->write_va = write_va; 1255 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1256 1257 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1258 rx_desc, (unsigned char *)hdr); 1259 break; 1260 case ISCSI_OP_NOOP_OUT: 1261 cmd = isert_allocate_cmd(conn, rx_desc); 1262 if (!cmd) 1263 break; 1264 1265 isert_cmd = iscsit_priv_cmd(cmd); 1266 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1267 rx_desc, (unsigned char *)hdr); 1268 break; 1269 case ISCSI_OP_SCSI_DATA_OUT: 1270 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1271 (unsigned char *)hdr); 1272 break; 1273 case ISCSI_OP_SCSI_TMFUNC: 1274 cmd = isert_allocate_cmd(conn, rx_desc); 1275 if (!cmd) 1276 break; 1277 1278 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1279 (unsigned char *)hdr); 1280 break; 1281 case ISCSI_OP_LOGOUT: 1282 cmd = isert_allocate_cmd(conn, rx_desc); 1283 if (!cmd) 1284 break; 1285 1286 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1287 break; 1288 case ISCSI_OP_TEXT: 1289 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1290 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1291 else 1292 cmd = isert_allocate_cmd(conn, rx_desc); 1293 1294 if (!cmd) 1295 break; 1296 1297 isert_cmd = iscsit_priv_cmd(cmd); 1298 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1299 rx_desc, (struct iscsi_text *)hdr); 1300 break; 1301 default: 1302 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1303 dump_stack(); 1304 break; 1305 } 1306 1307 return ret; 1308 } 1309 1310 static void 1311 isert_print_wc(struct ib_wc *wc, const char *type) 1312 { 1313 if (wc->status != IB_WC_WR_FLUSH_ERR) 1314 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1315 ib_wc_status_msg(wc->status), wc->status, 1316 wc->vendor_err); 1317 else 1318 isert_dbg("%s failure: %s (%d)\n", type, 1319 ib_wc_status_msg(wc->status), wc->status); 1320 } 1321 1322 static void 1323 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1324 { 1325 struct isert_conn *isert_conn = wc->qp->qp_context; 1326 struct ib_device *ib_dev = isert_conn->cm_id->device; 1327 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1328 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1329 struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc); 1330 uint64_t read_va = 0, write_va = 0; 1331 uint32_t read_stag = 0, write_stag = 0; 1332 1333 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1334 isert_print_wc(wc, "recv"); 1335 if (wc->status != IB_WC_WR_FLUSH_ERR) 1336 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1337 return; 1338 } 1339 1340 rx_desc->in_use = true; 1341 1342 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1343 ISER_RX_SIZE, DMA_FROM_DEVICE); 1344 1345 /* 1346 * The data segment length declared in the BHS is attacker controlled 1347 * and is used further down to read that many bytes out of the fixed 1348 * size receive descriptor, so it has to be checked against the number 1349 * of bytes that were actually received. Comparing without subtracting 1350 * also rejects PDUs shorter than the iSER and iSCSI headers, which 1351 * would otherwise be parsed out of stale descriptor contents. 1352 */ 1353 if (unlikely(wc->byte_len < ISER_HEADERS_LEN + ntoh24(hdr->dlength))) { 1354 isert_err("PDU declares %u data bytes but only %u bytes were received\n", 1355 ntoh24(hdr->dlength), wc->byte_len); 1356 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1357 return; 1358 } 1359 1360 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1361 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1362 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1363 1364 switch (iser_ctrl->flags & 0xF0) { 1365 case ISCSI_CTRL: 1366 if (iser_ctrl->flags & ISER_RSV) { 1367 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1368 read_va = be64_to_cpu(iser_ctrl->read_va); 1369 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1370 read_stag, (unsigned long long)read_va); 1371 } 1372 if (iser_ctrl->flags & ISER_WSV) { 1373 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1374 write_va = be64_to_cpu(iser_ctrl->write_va); 1375 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1376 write_stag, (unsigned long long)write_va); 1377 } 1378 1379 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1380 break; 1381 case ISER_HELLO: 1382 isert_err("iSER Hello message\n"); 1383 break; 1384 default: 1385 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1386 break; 1387 } 1388 1389 isert_rx_opcode(isert_conn, rx_desc, 1390 read_stag, read_va, write_stag, write_va); 1391 1392 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1393 ISER_RX_SIZE, DMA_FROM_DEVICE); 1394 } 1395 1396 static void 1397 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1398 { 1399 struct isert_conn *isert_conn = wc->qp->qp_context; 1400 struct ib_device *ib_dev = isert_conn->device->ib_device; 1401 1402 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1403 isert_print_wc(wc, "login recv"); 1404 return; 1405 } 1406 1407 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr, 1408 ISER_RX_SIZE, DMA_FROM_DEVICE); 1409 1410 if (unlikely(wc->byte_len < ISER_HEADERS_LEN)) { 1411 isert_dbg("login request length %u is too short\n", 1412 wc->byte_len); 1413 return; 1414 } 1415 1416 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1417 1418 if (isert_conn->conn) { 1419 struct iscsi_login *login = isert_conn->conn->conn_login; 1420 1421 if (login && !login->first_request) { 1422 if (isert_check_login_req(isert_conn)) 1423 return; 1424 1425 isert_rx_login_req(isert_conn); 1426 } 1427 } 1428 1429 mutex_lock(&isert_conn->mutex); 1430 complete(&isert_conn->login_req_comp); 1431 mutex_unlock(&isert_conn->mutex); 1432 1433 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr, 1434 ISER_RX_SIZE, DMA_FROM_DEVICE); 1435 } 1436 1437 static void 1438 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1439 { 1440 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 1441 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1442 1443 if (!cmd->rw.nr_ops) 1444 return; 1445 1446 if (isert_prot_cmd(conn, se_cmd)) { 1447 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1448 conn->cm_id->port_num, se_cmd->t_data_sg, 1449 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1450 se_cmd->t_prot_nents, dir); 1451 } else { 1452 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1453 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1454 } 1455 1456 cmd->rw.nr_ops = 0; 1457 } 1458 1459 static void 1460 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1461 { 1462 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1463 struct isert_conn *isert_conn = isert_cmd->conn; 1464 struct iscsit_conn *conn = isert_conn->conn; 1465 struct iscsi_text_rsp *hdr; 1466 1467 isert_dbg("Cmd %p\n", isert_cmd); 1468 1469 switch (cmd->iscsi_opcode) { 1470 case ISCSI_OP_SCSI_CMD: 1471 spin_lock_bh(&conn->cmd_lock); 1472 if (!list_empty(&cmd->i_conn_node)) 1473 list_del_init(&cmd->i_conn_node); 1474 spin_unlock_bh(&conn->cmd_lock); 1475 1476 if (cmd->data_direction == DMA_TO_DEVICE) { 1477 iscsit_stop_dataout_timer(cmd); 1478 /* 1479 * Check for special case during comp_err where 1480 * WRITE_PENDING has been handed off from core, 1481 * but requires an extra target_put_sess_cmd() 1482 * before transport_generic_free_cmd() below. 1483 */ 1484 if (comp_err && 1485 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1486 struct se_cmd *se_cmd = &cmd->se_cmd; 1487 1488 target_put_sess_cmd(se_cmd); 1489 } 1490 } 1491 1492 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1493 transport_generic_free_cmd(&cmd->se_cmd, 0); 1494 break; 1495 case ISCSI_OP_SCSI_TMFUNC: 1496 spin_lock_bh(&conn->cmd_lock); 1497 if (!list_empty(&cmd->i_conn_node)) 1498 list_del_init(&cmd->i_conn_node); 1499 spin_unlock_bh(&conn->cmd_lock); 1500 1501 transport_generic_free_cmd(&cmd->se_cmd, 0); 1502 break; 1503 case ISCSI_OP_REJECT: 1504 case ISCSI_OP_NOOP_OUT: 1505 case ISCSI_OP_TEXT: 1506 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1507 /* If the continue bit is on, keep the command alive */ 1508 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1509 break; 1510 1511 spin_lock_bh(&conn->cmd_lock); 1512 if (!list_empty(&cmd->i_conn_node)) 1513 list_del_init(&cmd->i_conn_node); 1514 spin_unlock_bh(&conn->cmd_lock); 1515 1516 /* 1517 * Handle special case for REJECT when iscsi_add_reject*() has 1518 * overwritten the original iscsi_opcode assignment, and the 1519 * associated cmd->se_cmd needs to be released. 1520 */ 1521 if (cmd->se_cmd.se_tfo != NULL) { 1522 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1523 cmd->iscsi_opcode); 1524 transport_generic_free_cmd(&cmd->se_cmd, 0); 1525 break; 1526 } 1527 fallthrough; 1528 default: 1529 iscsit_release_cmd(cmd); 1530 break; 1531 } 1532 } 1533 1534 static void 1535 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1536 { 1537 if (tx_desc->dma_addr != 0) { 1538 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1539 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1540 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1541 tx_desc->dma_addr = 0; 1542 } 1543 } 1544 1545 static void 1546 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1547 struct ib_device *ib_dev, bool comp_err) 1548 { 1549 if (isert_cmd->pdu_buf_dma != 0) { 1550 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1551 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1552 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1553 isert_cmd->pdu_buf_dma = 0; 1554 } 1555 1556 isert_unmap_tx_desc(tx_desc, ib_dev); 1557 isert_put_cmd(isert_cmd, comp_err); 1558 } 1559 1560 static int 1561 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1562 { 1563 struct ib_mr_status mr_status; 1564 int ret; 1565 1566 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1567 if (ret) { 1568 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1569 goto fail_mr_status; 1570 } 1571 1572 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1573 u64 sec_offset_err; 1574 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1575 1576 switch (mr_status.sig_err.err_type) { 1577 case IB_SIG_BAD_GUARD: 1578 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1579 break; 1580 case IB_SIG_BAD_REFTAG: 1581 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1582 break; 1583 case IB_SIG_BAD_APPTAG: 1584 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1585 break; 1586 } 1587 sec_offset_err = mr_status.sig_err.sig_err_offset; 1588 do_div(sec_offset_err, block_size); 1589 se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba; 1590 1591 isert_err("PI error found type %d at sector 0x%llx " 1592 "expected 0x%x vs actual 0x%x\n", 1593 mr_status.sig_err.err_type, 1594 (unsigned long long)se_cmd->sense_info, 1595 mr_status.sig_err.expected, 1596 mr_status.sig_err.actual); 1597 ret = 1; 1598 } 1599 1600 fail_mr_status: 1601 return ret; 1602 } 1603 1604 static void 1605 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1606 { 1607 struct isert_conn *isert_conn = wc->qp->qp_context; 1608 struct isert_device *device = isert_conn->device; 1609 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1610 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1611 struct se_cmd *cmd = &isert_cmd->iscsit_cmd->se_cmd; 1612 int ret = 0; 1613 1614 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1615 isert_print_wc(wc, "rdma write"); 1616 if (wc->status != IB_WC_WR_FLUSH_ERR) 1617 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1618 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1619 return; 1620 } 1621 1622 isert_dbg("Cmd %p\n", isert_cmd); 1623 1624 ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 1625 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1626 1627 if (ret) { 1628 /* 1629 * transport_generic_request_failure() expects to have 1630 * plus two references to handle queue-full, so re-add 1631 * one here as target-core will have already dropped 1632 * it after the first isert_put_datain() callback. 1633 */ 1634 kref_get(&cmd->cmd_kref); 1635 transport_generic_request_failure(cmd, cmd->pi_err); 1636 } else { 1637 /* 1638 * XXX: isert_put_response() failure is not retried. 1639 */ 1640 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsit_cmd); 1641 if (ret) 1642 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1643 } 1644 } 1645 1646 static void 1647 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1648 { 1649 struct isert_conn *isert_conn = wc->qp->qp_context; 1650 struct isert_device *device = isert_conn->device; 1651 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1652 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1653 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1654 struct se_cmd *se_cmd = &cmd->se_cmd; 1655 int ret = 0; 1656 1657 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1658 isert_print_wc(wc, "rdma read"); 1659 if (wc->status != IB_WC_WR_FLUSH_ERR) 1660 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1661 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1662 return; 1663 } 1664 1665 isert_dbg("Cmd %p\n", isert_cmd); 1666 1667 iscsit_stop_dataout_timer(cmd); 1668 1669 if (isert_prot_cmd(isert_conn, se_cmd)) 1670 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 1671 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1672 cmd->write_data_done = 0; 1673 1674 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1675 spin_lock_bh(&cmd->istate_lock); 1676 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1677 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1678 spin_unlock_bh(&cmd->istate_lock); 1679 1680 /* 1681 * transport_generic_request_failure() will drop the extra 1682 * se_cmd->cmd_kref reference after T10-PI error, and handle 1683 * any non-zero ->queue_status() callback error retries. 1684 */ 1685 if (ret) 1686 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1687 else 1688 target_execute_cmd(se_cmd); 1689 } 1690 1691 static void 1692 isert_do_control_comp(struct work_struct *work) 1693 { 1694 struct isert_cmd *isert_cmd = container_of(work, 1695 struct isert_cmd, comp_work); 1696 struct isert_conn *isert_conn = isert_cmd->conn; 1697 struct ib_device *ib_dev = isert_conn->cm_id->device; 1698 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1699 /* The switch below may free isert_cmd. */ 1700 bool counted = isert_cmd->ctrl_counted; 1701 1702 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1703 1704 switch (cmd->i_state) { 1705 case ISTATE_SEND_TASKMGTRSP: 1706 iscsit_tmr_post_handler(cmd, cmd->conn); 1707 fallthrough; 1708 case ISTATE_SEND_REJECT: 1709 case ISTATE_SEND_TEXTRSP: 1710 cmd->i_state = ISTATE_SENT_STATUS; 1711 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1712 ib_dev, false); 1713 break; 1714 case ISTATE_SEND_LOGOUTRSP: 1715 iscsit_logout_post_handler(cmd, cmd->conn); 1716 break; 1717 default: 1718 isert_err("Unknown i_state %d\n", cmd->i_state); 1719 dump_stack(); 1720 break; 1721 } 1722 1723 /* 1724 * The count is what keeps isert_conn alive, so drop it last. The wait 1725 * queue lives in the global hash table, not in isert_conn, so this is 1726 * safe even if the waiter has already freed the connection. 1727 */ 1728 if (counted && atomic_dec_and_test(&isert_conn->ctrl_comp_cnt)) 1729 wake_up_var(&isert_conn->ctrl_comp_cnt); 1730 } 1731 1732 static void 1733 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1734 { 1735 struct isert_conn *isert_conn = wc->qp->qp_context; 1736 struct ib_device *ib_dev = isert_conn->cm_id->device; 1737 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1738 1739 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1740 isert_print_wc(wc, "login send"); 1741 if (wc->status != IB_WC_WR_FLUSH_ERR) 1742 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1743 } 1744 1745 isert_unmap_tx_desc(tx_desc, ib_dev); 1746 } 1747 1748 static void 1749 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1750 { 1751 struct isert_conn *isert_conn = wc->qp->qp_context; 1752 struct ib_device *ib_dev = isert_conn->cm_id->device; 1753 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1754 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1755 1756 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1757 isert_print_wc(wc, "send"); 1758 if (wc->status != IB_WC_WR_FLUSH_ERR) 1759 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1760 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1761 return; 1762 } 1763 1764 isert_dbg("Cmd %p\n", isert_cmd); 1765 1766 switch (isert_cmd->iscsit_cmd->i_state) { 1767 case ISTATE_SEND_TASKMGTRSP: 1768 case ISTATE_SEND_LOGOUTRSP: 1769 case ISTATE_SEND_REJECT: 1770 case ISTATE_SEND_TEXTRSP: 1771 isert_unmap_tx_desc(tx_desc, ib_dev); 1772 1773 /* Paired with the wait in isert_wait_conn(). */ 1774 isert_cmd->ctrl_counted = 1775 isert_cmd->iscsit_cmd->i_state != ISTATE_SEND_LOGOUTRSP; 1776 if (isert_cmd->ctrl_counted) 1777 atomic_inc(&isert_conn->ctrl_comp_cnt); 1778 1779 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1780 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1781 return; 1782 default: 1783 isert_cmd->iscsit_cmd->i_state = ISTATE_SENT_STATUS; 1784 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1785 break; 1786 } 1787 } 1788 1789 static int 1790 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1791 { 1792 int ret; 1793 1794 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1795 if (ret) 1796 return ret; 1797 1798 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1799 if (ret) { 1800 isert_err("ib_post_send failed with %d\n", ret); 1801 return ret; 1802 } 1803 return ret; 1804 } 1805 1806 static int 1807 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1808 { 1809 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1810 struct isert_conn *isert_conn = conn->context; 1811 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1812 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1813 &isert_cmd->tx_desc.iscsi_header; 1814 1815 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1816 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1817 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1818 /* 1819 * Attach SENSE DATA payload to iSCSI Response PDU 1820 */ 1821 if (cmd->se_cmd.sense_buffer && 1822 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1823 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1824 struct isert_device *device = isert_conn->device; 1825 struct ib_device *ib_dev = device->ib_device; 1826 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1827 u32 padding, pdu_len; 1828 1829 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1830 cmd->sense_buffer); 1831 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1832 1833 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1834 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1835 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1836 1837 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1838 (void *)cmd->sense_buffer, pdu_len, 1839 DMA_TO_DEVICE); 1840 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1841 return -ENOMEM; 1842 1843 isert_cmd->pdu_buf_len = pdu_len; 1844 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1845 tx_dsg->length = pdu_len; 1846 tx_dsg->lkey = device->pd->local_dma_lkey; 1847 isert_cmd->tx_desc.num_sge = 2; 1848 } 1849 1850 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1851 1852 isert_dbg("Posting SCSI Response\n"); 1853 1854 return isert_post_response(isert_conn, isert_cmd); 1855 } 1856 1857 static void 1858 isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1859 { 1860 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1861 struct isert_conn *isert_conn = conn->context; 1862 1863 spin_lock_bh(&conn->cmd_lock); 1864 if (!list_empty(&cmd->i_conn_node)) 1865 list_del_init(&cmd->i_conn_node); 1866 spin_unlock_bh(&conn->cmd_lock); 1867 1868 if (cmd->data_direction == DMA_TO_DEVICE) 1869 iscsit_stop_dataout_timer(cmd); 1870 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1871 } 1872 1873 static enum target_prot_op 1874 isert_get_sup_prot_ops(struct iscsit_conn *conn) 1875 { 1876 struct isert_conn *isert_conn = conn->context; 1877 struct isert_device *device = isert_conn->device; 1878 1879 if (conn->tpg->tpg_attrib.t10_pi) { 1880 if (device->pi_capable) { 1881 isert_info("conn %p PI offload enabled\n", isert_conn); 1882 isert_conn->pi_support = true; 1883 return TARGET_PROT_ALL; 1884 } 1885 } 1886 1887 isert_info("conn %p PI offload disabled\n", isert_conn); 1888 isert_conn->pi_support = false; 1889 1890 return TARGET_PROT_NORMAL; 1891 } 1892 1893 static int 1894 isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn, 1895 bool nopout_response) 1896 { 1897 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1898 struct isert_conn *isert_conn = conn->context; 1899 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1900 1901 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1902 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1903 &isert_cmd->tx_desc.iscsi_header, 1904 nopout_response); 1905 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1906 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1907 1908 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1909 1910 return isert_post_response(isert_conn, isert_cmd); 1911 } 1912 1913 static int 1914 isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1915 { 1916 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1917 struct isert_conn *isert_conn = conn->context; 1918 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1919 1920 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1921 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1922 &isert_cmd->tx_desc.iscsi_header); 1923 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1924 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1925 1926 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1927 1928 return isert_post_response(isert_conn, isert_cmd); 1929 } 1930 1931 static int 1932 isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1933 { 1934 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1935 struct isert_conn *isert_conn = conn->context; 1936 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1937 1938 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1939 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1940 &isert_cmd->tx_desc.iscsi_header); 1941 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1942 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1943 1944 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1945 1946 return isert_post_response(isert_conn, isert_cmd); 1947 } 1948 1949 static int 1950 isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1951 { 1952 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1953 struct isert_conn *isert_conn = conn->context; 1954 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1955 struct isert_device *device = isert_conn->device; 1956 struct ib_device *ib_dev = device->ib_device; 1957 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1958 struct iscsi_reject *hdr = 1959 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1960 1961 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1962 iscsit_build_reject(cmd, conn, hdr); 1963 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1964 1965 hton24(hdr->dlength, ISCSI_HDR_LEN); 1966 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1967 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1968 DMA_TO_DEVICE); 1969 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1970 return -ENOMEM; 1971 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1972 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1973 tx_dsg->length = ISCSI_HDR_LEN; 1974 tx_dsg->lkey = device->pd->local_dma_lkey; 1975 isert_cmd->tx_desc.num_sge = 2; 1976 1977 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1978 1979 isert_dbg("conn %p Posting Reject\n", isert_conn); 1980 1981 return isert_post_response(isert_conn, isert_cmd); 1982 } 1983 1984 static int 1985 isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1986 { 1987 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1988 struct isert_conn *isert_conn = conn->context; 1989 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1990 struct iscsi_text_rsp *hdr = 1991 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1992 u32 txt_rsp_len; 1993 int rc; 1994 1995 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1996 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1997 if (rc < 0) 1998 return rc; 1999 2000 txt_rsp_len = rc; 2001 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2002 2003 if (txt_rsp_len) { 2004 struct isert_device *device = isert_conn->device; 2005 struct ib_device *ib_dev = device->ib_device; 2006 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2007 void *txt_rsp_buf = cmd->buf_ptr; 2008 2009 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2010 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2011 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2012 return -ENOMEM; 2013 2014 isert_cmd->pdu_buf_len = txt_rsp_len; 2015 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2016 tx_dsg->length = txt_rsp_len; 2017 tx_dsg->lkey = device->pd->local_dma_lkey; 2018 isert_cmd->tx_desc.num_sge = 2; 2019 } 2020 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2021 2022 isert_dbg("conn %p Text Response\n", isert_conn); 2023 2024 return isert_post_response(isert_conn, isert_cmd); 2025 } 2026 2027 static inline void 2028 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 2029 { 2030 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2031 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2032 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2033 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2034 /* 2035 * At the moment we hard code those, but if in the future 2036 * the target core would like to use it, we will take it 2037 * from se_cmd. 2038 */ 2039 domain->sig.dif.apptag_check_mask = 0xffff; 2040 domain->sig.dif.app_escape = true; 2041 domain->sig.dif.ref_escape = true; 2042 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2043 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2044 domain->sig.dif.ref_remap = true; 2045 } 2046 2047 static int 2048 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2049 { 2050 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2051 2052 switch (se_cmd->prot_op) { 2053 case TARGET_PROT_DIN_INSERT: 2054 case TARGET_PROT_DOUT_STRIP: 2055 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2056 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2057 break; 2058 case TARGET_PROT_DOUT_INSERT: 2059 case TARGET_PROT_DIN_STRIP: 2060 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2061 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2062 break; 2063 case TARGET_PROT_DIN_PASS: 2064 case TARGET_PROT_DOUT_PASS: 2065 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2066 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2067 break; 2068 default: 2069 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2070 return -EINVAL; 2071 } 2072 2073 if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 2074 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 2075 if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 2076 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 2077 if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 2078 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 2079 2080 return 0; 2081 } 2082 2083 static int 2084 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2085 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2086 { 2087 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 2088 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2089 u8 port_num = conn->cm_id->port_num; 2090 u64 addr; 2091 u32 rkey, offset; 2092 int ret; 2093 2094 if (cmd->ctx_init_done) 2095 goto rdma_ctx_post; 2096 2097 if (dir == DMA_FROM_DEVICE) { 2098 addr = cmd->write_va; 2099 rkey = cmd->write_stag; 2100 offset = cmd->iscsit_cmd->write_data_done; 2101 } else { 2102 addr = cmd->read_va; 2103 rkey = cmd->read_stag; 2104 offset = 0; 2105 } 2106 2107 if (isert_prot_cmd(conn, se_cmd)) { 2108 struct ib_sig_attrs sig_attrs; 2109 2110 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2111 if (ret) 2112 return ret; 2113 2114 WARN_ON_ONCE(offset); 2115 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2116 se_cmd->t_data_sg, se_cmd->t_data_nents, 2117 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2118 &sig_attrs, addr, rkey, dir); 2119 } else { 2120 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2121 se_cmd->t_data_sg, se_cmd->t_data_nents, 2122 offset, addr, rkey, dir); 2123 } 2124 2125 if (ret < 0) { 2126 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2127 return ret; 2128 } 2129 2130 cmd->ctx_init_done = true; 2131 2132 rdma_ctx_post: 2133 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2134 if (ret < 0) 2135 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2136 return ret; 2137 } 2138 2139 static int 2140 isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 2141 { 2142 struct se_cmd *se_cmd = &cmd->se_cmd; 2143 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2144 struct isert_conn *isert_conn = conn->context; 2145 struct ib_cqe *cqe = NULL; 2146 struct ib_send_wr *chain_wr = NULL; 2147 int rc; 2148 2149 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2150 isert_cmd, se_cmd->data_length); 2151 2152 if (isert_prot_cmd(isert_conn, se_cmd)) { 2153 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2154 cqe = &isert_cmd->tx_desc.tx_cqe; 2155 } else { 2156 /* 2157 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2158 */ 2159 isert_create_send_desc(isert_conn, isert_cmd, 2160 &isert_cmd->tx_desc); 2161 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2162 &isert_cmd->tx_desc.iscsi_header); 2163 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2164 isert_init_send_wr(isert_conn, isert_cmd, 2165 &isert_cmd->tx_desc.send_wr); 2166 2167 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2168 if (rc) 2169 return rc; 2170 2171 chain_wr = &isert_cmd->tx_desc.send_wr; 2172 } 2173 2174 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2175 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2176 isert_cmd, rc); 2177 return rc; 2178 } 2179 2180 static int 2181 isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery) 2182 { 2183 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2184 int ret; 2185 2186 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2187 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2188 2189 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2190 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2191 &isert_cmd->tx_desc.tx_cqe, NULL); 2192 2193 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2194 isert_cmd, ret); 2195 return ret; 2196 } 2197 2198 static int 2199 isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2200 { 2201 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2202 int ret = 0; 2203 2204 switch (state) { 2205 case ISTATE_REMOVE: 2206 spin_lock_bh(&conn->cmd_lock); 2207 list_del_init(&cmd->i_conn_node); 2208 spin_unlock_bh(&conn->cmd_lock); 2209 isert_put_cmd(isert_cmd, true); 2210 break; 2211 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2212 ret = isert_put_nopin(cmd, conn, false); 2213 break; 2214 default: 2215 isert_err("Unknown immediate state: 0x%02x\n", state); 2216 ret = -EINVAL; 2217 break; 2218 } 2219 2220 return ret; 2221 } 2222 2223 static int 2224 isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2225 { 2226 struct isert_conn *isert_conn = conn->context; 2227 int ret; 2228 2229 switch (state) { 2230 case ISTATE_SEND_LOGOUTRSP: 2231 ret = isert_put_logout_rsp(cmd, conn); 2232 if (!ret) 2233 isert_conn->logout_posted = true; 2234 break; 2235 case ISTATE_SEND_NOPIN: 2236 ret = isert_put_nopin(cmd, conn, true); 2237 break; 2238 case ISTATE_SEND_TASKMGTRSP: 2239 ret = isert_put_tm_rsp(cmd, conn); 2240 break; 2241 case ISTATE_SEND_REJECT: 2242 ret = isert_put_reject(cmd, conn); 2243 break; 2244 case ISTATE_SEND_TEXTRSP: 2245 ret = isert_put_text_rsp(cmd, conn); 2246 break; 2247 case ISTATE_SEND_STATUS: 2248 /* 2249 * Special case for sending non GOOD SCSI status from TX thread 2250 * context during pre se_cmd excecution failure. 2251 */ 2252 ret = isert_put_response(conn, cmd); 2253 break; 2254 default: 2255 isert_err("Unknown response state: 0x%02x\n", state); 2256 ret = -EINVAL; 2257 break; 2258 } 2259 2260 return ret; 2261 } 2262 2263 struct rdma_cm_id * 2264 isert_setup_id(struct isert_np *isert_np) 2265 { 2266 struct iscsi_np *np = isert_np->np; 2267 struct rdma_cm_id *id; 2268 struct sockaddr *sa; 2269 int ret; 2270 2271 sa = (struct sockaddr *)&np->np_sockaddr; 2272 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2273 2274 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2275 RDMA_PS_TCP, IB_QPT_RC); 2276 if (IS_ERR(id)) { 2277 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2278 ret = PTR_ERR(id); 2279 goto out; 2280 } 2281 isert_dbg("id %p context %p\n", id, id->context); 2282 2283 /* 2284 * Allow both IPv4 and IPv6 sockets to bind a single port 2285 * at the same time. 2286 */ 2287 ret = rdma_set_afonly(id, 1); 2288 if (ret) { 2289 isert_err("rdma_set_afonly() failed: %d\n", ret); 2290 goto out_id; 2291 } 2292 2293 ret = rdma_bind_addr(id, sa); 2294 if (ret) { 2295 isert_err("rdma_bind_addr() failed: %d\n", ret); 2296 goto out_id; 2297 } 2298 2299 ret = rdma_listen(id, 0); 2300 if (ret) { 2301 isert_err("rdma_listen() failed: %d\n", ret); 2302 goto out_id; 2303 } 2304 2305 return id; 2306 out_id: 2307 rdma_destroy_id(id); 2308 out: 2309 return ERR_PTR(ret); 2310 } 2311 2312 static int 2313 isert_setup_np(struct iscsi_np *np, 2314 struct sockaddr_storage *ksockaddr) 2315 { 2316 struct isert_np *isert_np; 2317 struct rdma_cm_id *isert_lid; 2318 int ret; 2319 2320 isert_np = kzalloc_obj(struct isert_np); 2321 if (!isert_np) 2322 return -ENOMEM; 2323 2324 sema_init(&isert_np->sem, 0); 2325 mutex_init(&isert_np->mutex); 2326 INIT_LIST_HEAD(&isert_np->accepted); 2327 INIT_LIST_HEAD(&isert_np->pending); 2328 isert_np->np = np; 2329 2330 /* 2331 * Setup the np->np_sockaddr from the passed sockaddr setup 2332 * in iscsi_target_configfs.c code.. 2333 */ 2334 memcpy(&np->np_sockaddr, ksockaddr, 2335 sizeof(struct sockaddr_storage)); 2336 2337 isert_lid = isert_setup_id(isert_np); 2338 if (IS_ERR(isert_lid)) { 2339 ret = PTR_ERR(isert_lid); 2340 goto out; 2341 } 2342 2343 isert_np->cm_id = isert_lid; 2344 np->np_context = isert_np; 2345 2346 return 0; 2347 2348 out: 2349 kfree(isert_np); 2350 2351 return ret; 2352 } 2353 2354 static int 2355 isert_rdma_accept(struct isert_conn *isert_conn) 2356 { 2357 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2358 struct rdma_conn_param cp; 2359 int ret; 2360 struct iser_cm_hdr rsp_hdr; 2361 2362 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2363 cp.initiator_depth = isert_conn->initiator_depth; 2364 cp.retry_count = 7; 2365 cp.rnr_retry_count = 7; 2366 2367 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2368 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2369 if (!isert_conn->snd_w_inv) 2370 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2371 cp.private_data = (void *)&rsp_hdr; 2372 cp.private_data_len = sizeof(rsp_hdr); 2373 2374 ret = rdma_accept(cm_id, &cp); 2375 if (ret) { 2376 isert_err("rdma_accept() failed with: %d\n", ret); 2377 return ret; 2378 } 2379 2380 return 0; 2381 } 2382 2383 static int 2384 isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) 2385 { 2386 struct isert_conn *isert_conn = conn->context; 2387 int ret; 2388 2389 isert_info("before login_req comp conn: %p\n", isert_conn); 2390 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2391 if (ret) { 2392 isert_err("isert_conn %p interrupted before got login req\n", 2393 isert_conn); 2394 return ret; 2395 } 2396 reinit_completion(&isert_conn->login_req_comp); 2397 2398 /* 2399 * For login requests after the first PDU, isert_rx_login_req() will 2400 * kick queue_delayed_work(isert_login_wq, &conn->login_work) as 2401 * the packet is received, which turns this callback from 2402 * iscsi_target_do_login_rx() into a NOP. 2403 */ 2404 if (!login->first_request) 2405 return 0; 2406 2407 ret = isert_check_login_req(isert_conn); 2408 if (ret) 2409 return ret; 2410 2411 isert_rx_login_req(isert_conn); 2412 2413 isert_info("before login_comp conn: %p\n", conn); 2414 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2415 if (ret) 2416 return ret; 2417 2418 isert_info("processing login->req: %p\n", login->req); 2419 2420 return 0; 2421 } 2422 2423 static void 2424 isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn, 2425 struct isert_conn *isert_conn) 2426 { 2427 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2428 struct rdma_route *cm_route = &cm_id->route; 2429 2430 conn->login_family = np->np_sockaddr.ss_family; 2431 2432 conn->login_sockaddr = cm_route->addr.dst_addr; 2433 conn->local_sockaddr = cm_route->addr.src_addr; 2434 } 2435 2436 static int 2437 isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn) 2438 { 2439 struct isert_np *isert_np = np->np_context; 2440 struct isert_conn *isert_conn; 2441 int ret; 2442 2443 accept_wait: 2444 ret = down_interruptible(&isert_np->sem); 2445 if (ret) 2446 return -ENODEV; 2447 2448 spin_lock_bh(&np->np_thread_lock); 2449 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2450 spin_unlock_bh(&np->np_thread_lock); 2451 isert_dbg("np_thread_state %d\n", 2452 np->np_thread_state); 2453 /* 2454 * No point in stalling here when np_thread 2455 * is in state RESET/SHUTDOWN/EXIT - bail 2456 */ 2457 return -ENODEV; 2458 } 2459 spin_unlock_bh(&np->np_thread_lock); 2460 2461 mutex_lock(&isert_np->mutex); 2462 if (list_empty(&isert_np->pending)) { 2463 mutex_unlock(&isert_np->mutex); 2464 goto accept_wait; 2465 } 2466 isert_conn = list_first_entry(&isert_np->pending, 2467 struct isert_conn, node); 2468 list_del_init(&isert_conn->node); 2469 mutex_unlock(&isert_np->mutex); 2470 2471 conn->context = isert_conn; 2472 isert_conn->conn = conn; 2473 isert_conn->state = ISER_CONN_BOUND; 2474 2475 isert_set_conn_info(np, conn, isert_conn); 2476 2477 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2478 2479 return 0; 2480 } 2481 2482 static void 2483 isert_free_np(struct iscsi_np *np) 2484 { 2485 struct isert_np *isert_np = np->np_context; 2486 struct isert_conn *isert_conn, *n; 2487 LIST_HEAD(drop_conn_list); 2488 2489 if (isert_np->cm_id) 2490 rdma_destroy_id(isert_np->cm_id); 2491 2492 /* 2493 * FIXME: At this point we don't have a good way to insure 2494 * that at this point we don't have hanging connections that 2495 * completed RDMA establishment but didn't start iscsi login 2496 * process. So work-around this by cleaning up what ever piled 2497 * up in accepted and pending lists. 2498 */ 2499 mutex_lock(&isert_np->mutex); 2500 if (!list_empty(&isert_np->pending)) { 2501 isert_info("Still have isert pending connections\n"); 2502 list_for_each_entry_safe(isert_conn, n, 2503 &isert_np->pending, 2504 node) { 2505 isert_info("cleaning isert_conn %p state (%d)\n", 2506 isert_conn, isert_conn->state); 2507 list_move_tail(&isert_conn->node, &drop_conn_list); 2508 } 2509 } 2510 2511 if (!list_empty(&isert_np->accepted)) { 2512 isert_info("Still have isert accepted connections\n"); 2513 list_for_each_entry_safe(isert_conn, n, 2514 &isert_np->accepted, 2515 node) { 2516 isert_info("cleaning isert_conn %p state (%d)\n", 2517 isert_conn, isert_conn->state); 2518 list_move_tail(&isert_conn->node, &drop_conn_list); 2519 } 2520 } 2521 mutex_unlock(&isert_np->mutex); 2522 2523 list_for_each_entry_safe(isert_conn, n, &drop_conn_list, node) { 2524 list_del_init(&isert_conn->node); 2525 isert_connect_release(isert_conn); 2526 } 2527 2528 np->np_context = NULL; 2529 kfree(isert_np); 2530 } 2531 2532 static void isert_release_work(struct work_struct *work) 2533 { 2534 struct isert_conn *isert_conn = container_of(work, 2535 struct isert_conn, 2536 release_work); 2537 2538 isert_info("Starting release conn %p\n", isert_conn); 2539 2540 mutex_lock(&isert_conn->mutex); 2541 isert_conn->state = ISER_CONN_DOWN; 2542 mutex_unlock(&isert_conn->mutex); 2543 2544 isert_info("Destroying conn %p\n", isert_conn); 2545 isert_put_conn(isert_conn); 2546 } 2547 2548 static void 2549 isert_wait4logout(struct isert_conn *isert_conn) 2550 { 2551 struct iscsit_conn *conn = isert_conn->conn; 2552 2553 isert_info("conn %p\n", isert_conn); 2554 2555 if (isert_conn->logout_posted) { 2556 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2557 wait_for_completion_timeout(&conn->conn_logout_comp, 2558 SECONDS_FOR_LOGOUT_COMP * HZ); 2559 } 2560 } 2561 2562 static void 2563 isert_wait4cmds(struct iscsit_conn *conn) 2564 { 2565 isert_info("iscsit_conn %p\n", conn); 2566 2567 if (conn->sess) { 2568 target_stop_cmd_counter(conn->cmd_cnt); 2569 target_wait_for_cmds(conn->cmd_cnt); 2570 } 2571 } 2572 2573 /** 2574 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2575 * unsolicitate dataout 2576 * @conn: iscsi connection 2577 * 2578 * We might still have commands that are waiting for unsolicited 2579 * dataouts messages. We must put the extra reference on those 2580 * before blocking on the target_wait_for_session_cmds 2581 */ 2582 static void 2583 isert_put_unsol_pending_cmds(struct iscsit_conn *conn) 2584 { 2585 struct iscsit_cmd *cmd, *tmp; 2586 LIST_HEAD(drop_cmd_list); 2587 2588 spin_lock_bh(&conn->cmd_lock); 2589 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2590 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2591 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2592 (cmd->write_data_done < cmd->se_cmd.data_length)) 2593 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2594 } 2595 spin_unlock_bh(&conn->cmd_lock); 2596 2597 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2598 list_del_init(&cmd->i_conn_node); 2599 if (cmd->i_state != ISTATE_REMOVE) { 2600 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2601 2602 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2603 isert_put_cmd(isert_cmd, true); 2604 } 2605 } 2606 } 2607 2608 static void isert_wait_conn(struct iscsit_conn *conn) 2609 { 2610 struct isert_conn *isert_conn = conn->context; 2611 2612 isert_info("Starting conn %p\n", isert_conn); 2613 2614 mutex_lock(&isert_conn->mutex); 2615 isert_conn_terminate(isert_conn); 2616 mutex_unlock(&isert_conn->mutex); 2617 2618 ib_drain_qp(isert_conn->qp); 2619 isert_put_unsol_pending_cmds(conn); 2620 isert_wait4cmds(conn); 2621 isert_wait4logout(isert_conn); 2622 2623 /* Paired with the count taken in isert_send_done(). */ 2624 wait_var_event(&isert_conn->ctrl_comp_cnt, 2625 !atomic_read(&isert_conn->ctrl_comp_cnt)); 2626 2627 queue_work(isert_release_wq, &isert_conn->release_work); 2628 } 2629 2630 static void isert_free_conn(struct iscsit_conn *conn) 2631 { 2632 struct isert_conn *isert_conn = conn->context; 2633 2634 ib_drain_qp(isert_conn->qp); 2635 isert_put_conn(isert_conn); 2636 } 2637 2638 static void isert_get_rx_pdu(struct iscsit_conn *conn) 2639 { 2640 struct isert_conn *isert_conn = conn->context; 2641 struct completion comp; 2642 2643 /* The login timeout timer can fail the login after isert_put_login_tx(). */ 2644 if (!isert_conn->login_rsp_pending) 2645 return; 2646 2647 isert_conn->login_rsp_pending = false; 2648 2649 /* The session is registered by now; see isert_put_login_tx(). */ 2650 if (isert_alloc_rx_descriptors(isert_conn)) 2651 return; 2652 2653 if (isert_post_recvm(isert_conn, ISERT_QP_MAX_RECV_DTOS)) 2654 return; 2655 2656 /* Now we are in FULL_FEATURE phase */ 2657 mutex_lock(&isert_conn->mutex); 2658 isert_conn->state = ISER_CONN_FULL_FEATURE; 2659 mutex_unlock(&isert_conn->mutex); 2660 2661 if (isert_login_post_send(isert_conn, &isert_conn->login_tx_desc)) 2662 return; 2663 2664 init_completion(&comp); 2665 2666 wait_for_completion_interruptible(&comp); 2667 } 2668 2669 static struct iscsit_transport iser_target_transport = { 2670 .name = "IB/iSER", 2671 .transport_type = ISCSI_INFINIBAND, 2672 .rdma_shutdown = true, 2673 .priv_size = sizeof(struct isert_cmd), 2674 .owner = THIS_MODULE, 2675 .iscsit_setup_np = isert_setup_np, 2676 .iscsit_accept_np = isert_accept_np, 2677 .iscsit_free_np = isert_free_np, 2678 .iscsit_wait_conn = isert_wait_conn, 2679 .iscsit_free_conn = isert_free_conn, 2680 .iscsit_get_login_rx = isert_get_login_rx, 2681 .iscsit_put_login_tx = isert_put_login_tx, 2682 .iscsit_immediate_queue = isert_immediate_queue, 2683 .iscsit_response_queue = isert_response_queue, 2684 .iscsit_get_dataout = isert_get_dataout, 2685 .iscsit_queue_data_in = isert_put_datain, 2686 .iscsit_queue_status = isert_put_response, 2687 .iscsit_aborted_task = isert_aborted_task, 2688 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2689 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2690 }; 2691 2692 static int __init isert_init(void) 2693 { 2694 isert_login_wq = alloc_workqueue("isert_login_wq", WQ_PERCPU, 0); 2695 if (!isert_login_wq) { 2696 isert_err("Unable to allocate isert_login_wq\n"); 2697 return -ENOMEM; 2698 } 2699 2700 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2701 WQ_UNBOUND | WQ_HIGHPRI, 0); 2702 if (!isert_comp_wq) { 2703 isert_err("Unable to allocate isert_comp_wq\n"); 2704 goto destroy_login_wq; 2705 } 2706 2707 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2708 WQ_UNBOUND_MAX_ACTIVE); 2709 if (!isert_release_wq) { 2710 isert_err("Unable to allocate isert_release_wq\n"); 2711 goto destroy_comp_wq; 2712 } 2713 2714 iscsit_register_transport(&iser_target_transport); 2715 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2716 2717 return 0; 2718 2719 destroy_comp_wq: 2720 destroy_workqueue(isert_comp_wq); 2721 destroy_login_wq: 2722 destroy_workqueue(isert_login_wq); 2723 2724 return -ENOMEM; 2725 } 2726 2727 static void __exit isert_exit(void) 2728 { 2729 flush_workqueue(isert_login_wq); 2730 destroy_workqueue(isert_release_wq); 2731 destroy_workqueue(isert_comp_wq); 2732 iscsit_unregister_transport(&iser_target_transport); 2733 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2734 destroy_workqueue(isert_login_wq); 2735 } 2736 2737 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2738 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2739 MODULE_LICENSE("GPL"); 2740 2741 module_init(isert_init); 2742 module_exit(isert_exit); 2743