1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 * $Id: iser_verbs.c 7051 2006-05-10 12:29:11Z ogerlitz $ 34 */ 35 #include <asm/io.h> 36 #include <linux/kernel.h> 37 #include <linux/module.h> 38 #include <linux/smp_lock.h> 39 #include <linux/delay.h> 40 #include <linux/version.h> 41 42 #include "iscsi_iser.h" 43 44 #define ISCSI_ISER_MAX_CONN 8 45 #define ISER_MAX_CQ_LEN ((ISER_QP_MAX_RECV_DTOS + \ 46 ISER_QP_MAX_REQ_DTOS) * \ 47 ISCSI_ISER_MAX_CONN) 48 49 static void iser_cq_tasklet_fn(unsigned long data); 50 static void iser_cq_callback(struct ib_cq *cq, void *cq_context); 51 static void iser_comp_error_worker(struct work_struct *work); 52 53 static void iser_cq_event_callback(struct ib_event *cause, void *context) 54 { 55 iser_err("got cq event %d \n", cause->event); 56 } 57 58 static void iser_qp_event_callback(struct ib_event *cause, void *context) 59 { 60 iser_err("got qp event %d\n",cause->event); 61 } 62 63 /** 64 * iser_create_device_ib_res - creates Protection Domain (PD), Completion 65 * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with 66 * the adapator. 67 * 68 * returns 0 on success, -1 on failure 69 */ 70 static int iser_create_device_ib_res(struct iser_device *device) 71 { 72 device->pd = ib_alloc_pd(device->ib_device); 73 if (IS_ERR(device->pd)) 74 goto pd_err; 75 76 device->cq = ib_create_cq(device->ib_device, 77 iser_cq_callback, 78 iser_cq_event_callback, 79 (void *)device, 80 ISER_MAX_CQ_LEN); 81 if (IS_ERR(device->cq)) 82 goto cq_err; 83 84 if (ib_req_notify_cq(device->cq, IB_CQ_NEXT_COMP)) 85 goto cq_arm_err; 86 87 tasklet_init(&device->cq_tasklet, 88 iser_cq_tasklet_fn, 89 (unsigned long)device); 90 91 device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE | 92 IB_ACCESS_REMOTE_WRITE | 93 IB_ACCESS_REMOTE_READ); 94 if (IS_ERR(device->mr)) 95 goto dma_mr_err; 96 97 return 0; 98 99 dma_mr_err: 100 tasklet_kill(&device->cq_tasklet); 101 cq_arm_err: 102 ib_destroy_cq(device->cq); 103 cq_err: 104 ib_dealloc_pd(device->pd); 105 pd_err: 106 iser_err("failed to allocate an IB resource\n"); 107 return -1; 108 } 109 110 /** 111 * iser_free_device_ib_res - destory/dealloc/dereg the DMA MR, 112 * CQ and PD created with the device associated with the adapator. 113 */ 114 static void iser_free_device_ib_res(struct iser_device *device) 115 { 116 BUG_ON(device->mr == NULL); 117 118 tasklet_kill(&device->cq_tasklet); 119 120 (void)ib_dereg_mr(device->mr); 121 (void)ib_destroy_cq(device->cq); 122 (void)ib_dealloc_pd(device->pd); 123 124 device->mr = NULL; 125 device->cq = NULL; 126 device->pd = NULL; 127 } 128 129 /** 130 * iser_create_ib_conn_res - Creates FMR pool and Queue-Pair (QP) 131 * 132 * returns 0 on success, -1 on failure 133 */ 134 static int iser_create_ib_conn_res(struct iser_conn *ib_conn) 135 { 136 struct iser_device *device; 137 struct ib_qp_init_attr init_attr; 138 int ret; 139 struct ib_fmr_pool_param params; 140 141 BUG_ON(ib_conn->device == NULL); 142 143 device = ib_conn->device; 144 145 ib_conn->page_vec = kmalloc(sizeof(struct iser_page_vec) + 146 (sizeof(u64) * (ISCSI_ISER_SG_TABLESIZE +1)), 147 GFP_KERNEL); 148 if (!ib_conn->page_vec) { 149 ret = -ENOMEM; 150 goto alloc_err; 151 } 152 ib_conn->page_vec->pages = (u64 *) (ib_conn->page_vec + 1); 153 154 params.page_shift = SHIFT_4K; 155 /* when the first/last SG element are not start/end * 156 * page aligned, the map whould be of N+1 pages */ 157 params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE + 1; 158 /* make the pool size twice the max number of SCSI commands * 159 * the ML is expected to queue, watermark for unmap at 50% */ 160 params.pool_size = ISCSI_XMIT_CMDS_MAX * 2; 161 params.dirty_watermark = ISCSI_XMIT_CMDS_MAX; 162 params.cache = 0; 163 params.flush_function = NULL; 164 params.access = (IB_ACCESS_LOCAL_WRITE | 165 IB_ACCESS_REMOTE_WRITE | 166 IB_ACCESS_REMOTE_READ); 167 168 ib_conn->fmr_pool = ib_create_fmr_pool(device->pd, ¶ms); 169 if (IS_ERR(ib_conn->fmr_pool)) { 170 ret = PTR_ERR(ib_conn->fmr_pool); 171 goto fmr_pool_err; 172 } 173 174 memset(&init_attr, 0, sizeof init_attr); 175 176 init_attr.event_handler = iser_qp_event_callback; 177 init_attr.qp_context = (void *)ib_conn; 178 init_attr.send_cq = device->cq; 179 init_attr.recv_cq = device->cq; 180 init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS; 181 init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; 182 init_attr.cap.max_send_sge = MAX_REGD_BUF_VECTOR_LEN; 183 init_attr.cap.max_recv_sge = 2; 184 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 185 init_attr.qp_type = IB_QPT_RC; 186 187 ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); 188 if (ret) 189 goto qp_err; 190 191 ib_conn->qp = ib_conn->cma_id->qp; 192 iser_err("setting conn %p cma_id %p: fmr_pool %p qp %p\n", 193 ib_conn, ib_conn->cma_id, 194 ib_conn->fmr_pool, ib_conn->cma_id->qp); 195 return ret; 196 197 qp_err: 198 (void)ib_destroy_fmr_pool(ib_conn->fmr_pool); 199 fmr_pool_err: 200 kfree(ib_conn->page_vec); 201 alloc_err: 202 iser_err("unable to alloc mem or create resource, err %d\n", ret); 203 return ret; 204 } 205 206 /** 207 * releases the FMR pool, QP and CMA ID objects, returns 0 on success, 208 * -1 on failure 209 */ 210 static int iser_free_ib_conn_res(struct iser_conn *ib_conn) 211 { 212 BUG_ON(ib_conn == NULL); 213 214 iser_err("freeing conn %p cma_id %p fmr pool %p qp %p\n", 215 ib_conn, ib_conn->cma_id, 216 ib_conn->fmr_pool, ib_conn->qp); 217 218 /* qp is created only once both addr & route are resolved */ 219 if (ib_conn->fmr_pool != NULL) 220 ib_destroy_fmr_pool(ib_conn->fmr_pool); 221 222 if (ib_conn->qp != NULL) 223 rdma_destroy_qp(ib_conn->cma_id); 224 225 if (ib_conn->cma_id != NULL) 226 rdma_destroy_id(ib_conn->cma_id); 227 228 ib_conn->fmr_pool = NULL; 229 ib_conn->qp = NULL; 230 ib_conn->cma_id = NULL; 231 kfree(ib_conn->page_vec); 232 233 return 0; 234 } 235 236 /** 237 * based on the resolved device node GUID see if there already allocated 238 * device for this device. If there's no such, create one. 239 */ 240 static 241 struct iser_device *iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) 242 { 243 struct list_head *p_list; 244 struct iser_device *device = NULL; 245 246 mutex_lock(&ig.device_list_mutex); 247 248 p_list = ig.device_list.next; 249 while (p_list != &ig.device_list) { 250 device = list_entry(p_list, struct iser_device, ig_list); 251 /* find if there's a match using the node GUID */ 252 if (device->ib_device->node_guid == cma_id->device->node_guid) 253 break; 254 } 255 256 if (device == NULL) { 257 device = kzalloc(sizeof *device, GFP_KERNEL); 258 if (device == NULL) 259 goto out; 260 /* assign this device to the device */ 261 device->ib_device = cma_id->device; 262 /* init the device and link it into ig device list */ 263 if (iser_create_device_ib_res(device)) { 264 kfree(device); 265 device = NULL; 266 goto out; 267 } 268 list_add(&device->ig_list, &ig.device_list); 269 } 270 out: 271 BUG_ON(device == NULL); 272 device->refcount++; 273 mutex_unlock(&ig.device_list_mutex); 274 return device; 275 } 276 277 /* if there's no demand for this device, release it */ 278 static void iser_device_try_release(struct iser_device *device) 279 { 280 mutex_lock(&ig.device_list_mutex); 281 device->refcount--; 282 iser_err("device %p refcount %d\n",device,device->refcount); 283 if (!device->refcount) { 284 iser_free_device_ib_res(device); 285 list_del(&device->ig_list); 286 kfree(device); 287 } 288 mutex_unlock(&ig.device_list_mutex); 289 } 290 291 int iser_conn_state_comp(struct iser_conn *ib_conn, 292 enum iser_ib_conn_state comp) 293 { 294 int ret; 295 296 spin_lock_bh(&ib_conn->lock); 297 ret = (ib_conn->state == comp); 298 spin_unlock_bh(&ib_conn->lock); 299 return ret; 300 } 301 302 static int iser_conn_state_comp_exch(struct iser_conn *ib_conn, 303 enum iser_ib_conn_state comp, 304 enum iser_ib_conn_state exch) 305 { 306 int ret; 307 308 spin_lock_bh(&ib_conn->lock); 309 if ((ret = (ib_conn->state == comp))) 310 ib_conn->state = exch; 311 spin_unlock_bh(&ib_conn->lock); 312 return ret; 313 } 314 315 /** 316 * triggers start of the disconnect procedures and wait for them to be done 317 */ 318 void iser_conn_terminate(struct iser_conn *ib_conn) 319 { 320 int err = 0; 321 322 /* change the ib conn state only if the conn is UP, however always call 323 * rdma_disconnect since this is the only way to cause the CMA to change 324 * the QP state to ERROR 325 */ 326 327 iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, ISER_CONN_TERMINATING); 328 err = rdma_disconnect(ib_conn->cma_id); 329 if (err) 330 iser_err("Failed to disconnect, conn: 0x%p err %d\n", 331 ib_conn,err); 332 333 wait_event_interruptible(ib_conn->wait, 334 ib_conn->state == ISER_CONN_DOWN); 335 336 iser_conn_release(ib_conn); 337 } 338 339 static void iser_connect_error(struct rdma_cm_id *cma_id) 340 { 341 struct iser_conn *ib_conn; 342 ib_conn = (struct iser_conn *)cma_id->context; 343 344 ib_conn->state = ISER_CONN_DOWN; 345 wake_up_interruptible(&ib_conn->wait); 346 } 347 348 static void iser_addr_handler(struct rdma_cm_id *cma_id) 349 { 350 struct iser_device *device; 351 struct iser_conn *ib_conn; 352 int ret; 353 354 device = iser_device_find_by_ib_device(cma_id); 355 ib_conn = (struct iser_conn *)cma_id->context; 356 ib_conn->device = device; 357 358 ret = rdma_resolve_route(cma_id, 1000); 359 if (ret) { 360 iser_err("resolve route failed: %d\n", ret); 361 iser_connect_error(cma_id); 362 } 363 return; 364 } 365 366 static void iser_route_handler(struct rdma_cm_id *cma_id) 367 { 368 struct rdma_conn_param conn_param; 369 int ret; 370 371 ret = iser_create_ib_conn_res((struct iser_conn *)cma_id->context); 372 if (ret) 373 goto failure; 374 375 iser_dbg("path.mtu is %d setting it to %d\n", 376 cma_id->route.path_rec->mtu, IB_MTU_1024); 377 378 /* we must set the MTU to 1024 as this is what the target is assuming */ 379 if (cma_id->route.path_rec->mtu > IB_MTU_1024) 380 cma_id->route.path_rec->mtu = IB_MTU_1024; 381 382 memset(&conn_param, 0, sizeof conn_param); 383 conn_param.responder_resources = 4; 384 conn_param.initiator_depth = 1; 385 conn_param.retry_count = 7; 386 conn_param.rnr_retry_count = 6; 387 388 ret = rdma_connect(cma_id, &conn_param); 389 if (ret) { 390 iser_err("failure connecting: %d\n", ret); 391 goto failure; 392 } 393 394 return; 395 failure: 396 iser_connect_error(cma_id); 397 } 398 399 static void iser_connected_handler(struct rdma_cm_id *cma_id) 400 { 401 struct iser_conn *ib_conn; 402 403 ib_conn = (struct iser_conn *)cma_id->context; 404 ib_conn->state = ISER_CONN_UP; 405 wake_up_interruptible(&ib_conn->wait); 406 } 407 408 static void iser_disconnected_handler(struct rdma_cm_id *cma_id) 409 { 410 struct iser_conn *ib_conn; 411 412 ib_conn = (struct iser_conn *)cma_id->context; 413 ib_conn->disc_evt_flag = 1; 414 415 /* getting here when the state is UP means that the conn is being * 416 * terminated asynchronously from the iSCSI layer's perspective. */ 417 if (iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, 418 ISER_CONN_TERMINATING)) 419 iscsi_conn_failure(ib_conn->iser_conn->iscsi_conn, 420 ISCSI_ERR_CONN_FAILED); 421 422 /* Complete the termination process if no posts are pending */ 423 if ((atomic_read(&ib_conn->post_recv_buf_count) == 0) && 424 (atomic_read(&ib_conn->post_send_buf_count) == 0)) { 425 ib_conn->state = ISER_CONN_DOWN; 426 wake_up_interruptible(&ib_conn->wait); 427 } 428 } 429 430 static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 431 { 432 int ret = 0; 433 434 iser_err("event %d conn %p id %p\n",event->event,cma_id->context,cma_id); 435 436 switch (event->event) { 437 case RDMA_CM_EVENT_ADDR_RESOLVED: 438 iser_addr_handler(cma_id); 439 break; 440 case RDMA_CM_EVENT_ROUTE_RESOLVED: 441 iser_route_handler(cma_id); 442 break; 443 case RDMA_CM_EVENT_ESTABLISHED: 444 iser_connected_handler(cma_id); 445 break; 446 case RDMA_CM_EVENT_ADDR_ERROR: 447 case RDMA_CM_EVENT_ROUTE_ERROR: 448 case RDMA_CM_EVENT_CONNECT_ERROR: 449 case RDMA_CM_EVENT_UNREACHABLE: 450 case RDMA_CM_EVENT_REJECTED: 451 iser_err("event: %d, error: %d\n", event->event, event->status); 452 iser_connect_error(cma_id); 453 break; 454 case RDMA_CM_EVENT_DISCONNECTED: 455 iser_disconnected_handler(cma_id); 456 break; 457 case RDMA_CM_EVENT_DEVICE_REMOVAL: 458 BUG(); 459 break; 460 case RDMA_CM_EVENT_CONNECT_RESPONSE: 461 BUG(); 462 break; 463 case RDMA_CM_EVENT_CONNECT_REQUEST: 464 default: 465 break; 466 } 467 return ret; 468 } 469 470 int iser_conn_init(struct iser_conn **ibconn) 471 { 472 struct iser_conn *ib_conn; 473 474 ib_conn = kzalloc(sizeof *ib_conn, GFP_KERNEL); 475 if (!ib_conn) { 476 iser_err("can't alloc memory for struct iser_conn\n"); 477 return -ENOMEM; 478 } 479 ib_conn->state = ISER_CONN_INIT; 480 init_waitqueue_head(&ib_conn->wait); 481 atomic_set(&ib_conn->post_recv_buf_count, 0); 482 atomic_set(&ib_conn->post_send_buf_count, 0); 483 INIT_WORK(&ib_conn->comperror_work, iser_comp_error_worker); 484 INIT_LIST_HEAD(&ib_conn->conn_list); 485 spin_lock_init(&ib_conn->lock); 486 487 *ibconn = ib_conn; 488 return 0; 489 } 490 491 /** 492 * starts the process of connecting to the target 493 * sleeps untill the connection is established or rejected 494 */ 495 int iser_connect(struct iser_conn *ib_conn, 496 struct sockaddr_in *src_addr, 497 struct sockaddr_in *dst_addr, 498 int non_blocking) 499 { 500 struct sockaddr *src, *dst; 501 int err = 0; 502 503 sprintf(ib_conn->name,"%d.%d.%d.%d:%d", 504 NIPQUAD(dst_addr->sin_addr.s_addr), dst_addr->sin_port); 505 506 /* the device is known only --after-- address resolution */ 507 ib_conn->device = NULL; 508 509 iser_err("connecting to: %d.%d.%d.%d, port 0x%x\n", 510 NIPQUAD(dst_addr->sin_addr), dst_addr->sin_port); 511 512 ib_conn->state = ISER_CONN_PENDING; 513 514 ib_conn->cma_id = rdma_create_id(iser_cma_handler, 515 (void *)ib_conn, 516 RDMA_PS_TCP); 517 if (IS_ERR(ib_conn->cma_id)) { 518 err = PTR_ERR(ib_conn->cma_id); 519 iser_err("rdma_create_id failed: %d\n", err); 520 goto id_failure; 521 } 522 523 src = (struct sockaddr *)src_addr; 524 dst = (struct sockaddr *)dst_addr; 525 err = rdma_resolve_addr(ib_conn->cma_id, src, dst, 1000); 526 if (err) { 527 iser_err("rdma_resolve_addr failed: %d\n", err); 528 goto addr_failure; 529 } 530 531 if (!non_blocking) { 532 wait_event_interruptible(ib_conn->wait, 533 (ib_conn->state != ISER_CONN_PENDING)); 534 535 if (ib_conn->state != ISER_CONN_UP) { 536 err = -EIO; 537 goto connect_failure; 538 } 539 } 540 541 mutex_lock(&ig.connlist_mutex); 542 list_add(&ib_conn->conn_list, &ig.connlist); 543 mutex_unlock(&ig.connlist_mutex); 544 return 0; 545 546 id_failure: 547 ib_conn->cma_id = NULL; 548 addr_failure: 549 ib_conn->state = ISER_CONN_DOWN; 550 connect_failure: 551 iser_conn_release(ib_conn); 552 return err; 553 } 554 555 /** 556 * Frees all conn objects and deallocs conn descriptor 557 */ 558 void iser_conn_release(struct iser_conn *ib_conn) 559 { 560 struct iser_device *device = ib_conn->device; 561 562 BUG_ON(ib_conn->state != ISER_CONN_DOWN); 563 564 mutex_lock(&ig.connlist_mutex); 565 list_del(&ib_conn->conn_list); 566 mutex_unlock(&ig.connlist_mutex); 567 568 iser_free_ib_conn_res(ib_conn); 569 ib_conn->device = NULL; 570 /* on EVENT_ADDR_ERROR there's no device yet for this conn */ 571 if (device != NULL) 572 iser_device_try_release(device); 573 if (ib_conn->iser_conn) 574 ib_conn->iser_conn->ib_conn = NULL; 575 kfree(ib_conn); 576 } 577 578 579 /** 580 * iser_reg_page_vec - Register physical memory 581 * 582 * returns: 0 on success, errno code on failure 583 */ 584 int iser_reg_page_vec(struct iser_conn *ib_conn, 585 struct iser_page_vec *page_vec, 586 struct iser_mem_reg *mem_reg) 587 { 588 struct ib_pool_fmr *mem; 589 u64 io_addr; 590 u64 *page_list; 591 int status; 592 593 page_list = page_vec->pages; 594 io_addr = page_list[0]; 595 596 mem = ib_fmr_pool_map_phys(ib_conn->fmr_pool, 597 page_list, 598 page_vec->length, 599 io_addr); 600 601 if (IS_ERR(mem)) { 602 status = (int)PTR_ERR(mem); 603 iser_err("ib_fmr_pool_map_phys failed: %d\n", status); 604 return status; 605 } 606 607 mem_reg->lkey = mem->fmr->lkey; 608 mem_reg->rkey = mem->fmr->rkey; 609 mem_reg->len = page_vec->length * SIZE_4K; 610 mem_reg->va = io_addr; 611 mem_reg->is_fmr = 1; 612 mem_reg->mem_h = (void *)mem; 613 614 mem_reg->va += page_vec->offset; 615 mem_reg->len = page_vec->data_size; 616 617 iser_dbg("PHYSICAL Mem.register, [PHYS p_array: 0x%p, sz: %d, " 618 "entry[0]: (0x%08lx,%ld)] -> " 619 "[lkey: 0x%08X mem_h: 0x%p va: 0x%08lX sz: %ld]\n", 620 page_vec, page_vec->length, 621 (unsigned long)page_vec->pages[0], 622 (unsigned long)page_vec->data_size, 623 (unsigned int)mem_reg->lkey, mem_reg->mem_h, 624 (unsigned long)mem_reg->va, (unsigned long)mem_reg->len); 625 return 0; 626 } 627 628 /** 629 * Unregister (previosuly registered) memory. 630 */ 631 void iser_unreg_mem(struct iser_mem_reg *reg) 632 { 633 int ret; 634 635 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n",reg->mem_h); 636 637 ret = ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h); 638 if (ret) 639 iser_err("ib_fmr_pool_unmap failed %d\n", ret); 640 641 reg->mem_h = NULL; 642 } 643 644 /** 645 * iser_dto_to_iov - builds IOV from a dto descriptor 646 */ 647 static void iser_dto_to_iov(struct iser_dto *dto, struct ib_sge *iov, int iov_len) 648 { 649 int i; 650 struct ib_sge *sge; 651 struct iser_regd_buf *regd_buf; 652 653 if (dto->regd_vector_len > iov_len) { 654 iser_err("iov size %d too small for posting dto of len %d\n", 655 iov_len, dto->regd_vector_len); 656 BUG(); 657 } 658 659 for (i = 0; i < dto->regd_vector_len; i++) { 660 sge = &iov[i]; 661 regd_buf = dto->regd[i]; 662 663 sge->addr = regd_buf->reg.va; 664 sge->length = regd_buf->reg.len; 665 sge->lkey = regd_buf->reg.lkey; 666 667 if (dto->used_sz[i] > 0) /* Adjust size */ 668 sge->length = dto->used_sz[i]; 669 670 /* offset and length should not exceed the regd buf length */ 671 if (sge->length + dto->offset[i] > regd_buf->reg.len) { 672 iser_err("Used len:%ld + offset:%d, exceed reg.buf.len:" 673 "%ld in dto:0x%p [%d], va:0x%08lX\n", 674 (unsigned long)sge->length, dto->offset[i], 675 (unsigned long)regd_buf->reg.len, dto, i, 676 (unsigned long)sge->addr); 677 BUG(); 678 } 679 680 sge->addr += dto->offset[i]; /* Adjust offset */ 681 } 682 } 683 684 /** 685 * iser_post_recv - Posts a receive buffer. 686 * 687 * returns 0 on success, -1 on failure 688 */ 689 int iser_post_recv(struct iser_desc *rx_desc) 690 { 691 int ib_ret, ret_val = 0; 692 struct ib_recv_wr recv_wr, *recv_wr_failed; 693 struct ib_sge iov[2]; 694 struct iser_conn *ib_conn; 695 struct iser_dto *recv_dto = &rx_desc->dto; 696 697 /* Retrieve conn */ 698 ib_conn = recv_dto->ib_conn; 699 700 iser_dto_to_iov(recv_dto, iov, 2); 701 702 recv_wr.next = NULL; 703 recv_wr.sg_list = iov; 704 recv_wr.num_sge = recv_dto->regd_vector_len; 705 recv_wr.wr_id = (unsigned long)rx_desc; 706 707 atomic_inc(&ib_conn->post_recv_buf_count); 708 ib_ret = ib_post_recv(ib_conn->qp, &recv_wr, &recv_wr_failed); 709 if (ib_ret) { 710 iser_err("ib_post_recv failed ret=%d\n", ib_ret); 711 atomic_dec(&ib_conn->post_recv_buf_count); 712 ret_val = -1; 713 } 714 715 return ret_val; 716 } 717 718 /** 719 * iser_start_send - Initiate a Send DTO operation 720 * 721 * returns 0 on success, -1 on failure 722 */ 723 int iser_post_send(struct iser_desc *tx_desc) 724 { 725 int ib_ret, ret_val = 0; 726 struct ib_send_wr send_wr, *send_wr_failed; 727 struct ib_sge iov[MAX_REGD_BUF_VECTOR_LEN]; 728 struct iser_conn *ib_conn; 729 struct iser_dto *dto = &tx_desc->dto; 730 731 ib_conn = dto->ib_conn; 732 733 iser_dto_to_iov(dto, iov, MAX_REGD_BUF_VECTOR_LEN); 734 735 send_wr.next = NULL; 736 send_wr.wr_id = (unsigned long)tx_desc; 737 send_wr.sg_list = iov; 738 send_wr.num_sge = dto->regd_vector_len; 739 send_wr.opcode = IB_WR_SEND; 740 send_wr.send_flags = dto->notify_enable ? IB_SEND_SIGNALED : 0; 741 742 atomic_inc(&ib_conn->post_send_buf_count); 743 744 ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed); 745 if (ib_ret) { 746 iser_err("Failed to start SEND DTO, dto: 0x%p, IOV len: %d\n", 747 dto, dto->regd_vector_len); 748 iser_err("ib_post_send failed, ret:%d\n", ib_ret); 749 atomic_dec(&ib_conn->post_send_buf_count); 750 ret_val = -1; 751 } 752 753 return ret_val; 754 } 755 756 static void iser_comp_error_worker(struct work_struct *work) 757 { 758 struct iser_conn *ib_conn = 759 container_of(work, struct iser_conn, comperror_work); 760 761 /* getting here when the state is UP means that the conn is being * 762 * terminated asynchronously from the iSCSI layer's perspective. */ 763 if (iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, 764 ISER_CONN_TERMINATING)) 765 iscsi_conn_failure(ib_conn->iser_conn->iscsi_conn, 766 ISCSI_ERR_CONN_FAILED); 767 768 /* complete the termination process if disconnect event was delivered * 769 * note there are no more non completed posts to the QP */ 770 if (ib_conn->disc_evt_flag) { 771 ib_conn->state = ISER_CONN_DOWN; 772 wake_up_interruptible(&ib_conn->wait); 773 } 774 } 775 776 static void iser_handle_comp_error(struct iser_desc *desc) 777 { 778 struct iser_dto *dto = &desc->dto; 779 struct iser_conn *ib_conn = dto->ib_conn; 780 781 iser_dto_buffs_release(dto); 782 783 if (desc->type == ISCSI_RX) { 784 kfree(desc->data); 785 kmem_cache_free(ig.desc_cache, desc); 786 atomic_dec(&ib_conn->post_recv_buf_count); 787 } else { /* type is TX control/command/dataout */ 788 if (desc->type == ISCSI_TX_DATAOUT) 789 kmem_cache_free(ig.desc_cache, desc); 790 atomic_dec(&ib_conn->post_send_buf_count); 791 } 792 793 if (atomic_read(&ib_conn->post_recv_buf_count) == 0 && 794 atomic_read(&ib_conn->post_send_buf_count) == 0) 795 schedule_work(&ib_conn->comperror_work); 796 } 797 798 static void iser_cq_tasklet_fn(unsigned long data) 799 { 800 struct iser_device *device = (struct iser_device *)data; 801 struct ib_cq *cq = device->cq; 802 struct ib_wc wc; 803 struct iser_desc *desc; 804 unsigned long xfer_len; 805 806 while (ib_poll_cq(cq, 1, &wc) == 1) { 807 desc = (struct iser_desc *) (unsigned long) wc.wr_id; 808 BUG_ON(desc == NULL); 809 810 if (wc.status == IB_WC_SUCCESS) { 811 if (desc->type == ISCSI_RX) { 812 xfer_len = (unsigned long)wc.byte_len; 813 iser_rcv_completion(desc, xfer_len); 814 } else /* type == ISCSI_TX_CONTROL/SCSI_CMD/DOUT */ 815 iser_snd_completion(desc); 816 } else { 817 iser_err("comp w. error op %d status %d\n",desc->type,wc.status); 818 iser_handle_comp_error(desc); 819 } 820 } 821 /* #warning "it is assumed here that arming CQ only once its empty" * 822 * " would not cause interrupts to be missed" */ 823 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 824 } 825 826 static void iser_cq_callback(struct ib_cq *cq, void *cq_context) 827 { 828 struct iser_device *device = (struct iser_device *)cq_context; 829 830 tasklet_schedule(&device->cq_tasklet); 831 } 832