1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * RDMA channel interface for Solaris SCSI RDMA Protocol Target (SRP) 28 * transport port provider module for the COMSTAR framework. 29 */ 30 31 #include <sys/cpuvar.h> 32 #include <sys/types.h> 33 #include <sys/conf.h> 34 #include <sys/stat.h> 35 #include <sys/file.h> 36 #include <sys/ddi.h> 37 #include <sys/sunddi.h> 38 #include <sys/modctl.h> 39 #include <sys/sysmacros.h> 40 #include <sys/sdt.h> 41 #include <sys/taskq.h> 42 #include <sys/scsi/scsi.h> 43 #include <sys/ib/ibtl/ibti.h> 44 45 #include <sys/stmf.h> 46 #include <sys/stmf_ioctl.h> 47 #include <sys/portif.h> 48 49 #include "srp.h" 50 #include "srpt_impl.h" 51 #include "srpt_ioc.h" 52 #include "srpt_stp.h" 53 #include "srpt_ch.h" 54 55 extern srpt_ctxt_t *srpt_ctxt; 56 extern uint16_t srpt_send_msg_depth; 57 58 /* 59 * Prototypes. 60 */ 61 static void srpt_ch_scq_hdlr(ibt_cq_hdl_t cq_dhl, void *arg); 62 static void srpt_ch_rcq_hdlr(ibt_cq_hdl_t cq_dhl, void *arg); 63 static void srpt_ch_process_iu(srpt_channel_t *ch, srpt_iu_t *iu); 64 65 /* 66 * srpt_ch_alloc() 67 */ 68 srpt_channel_t * 69 srpt_ch_alloc(srpt_target_port_t *tgt, uint8_t port) 70 { 71 ibt_status_t status; 72 srpt_channel_t *ch; 73 ibt_cq_attr_t cq_attr; 74 ibt_rc_chan_alloc_args_t ch_args; 75 uint32_t cq_real_size; 76 srpt_ioc_t *ioc; 77 78 ASSERT(tgt != NULL); 79 ioc = tgt->tp_ioc; 80 ASSERT(ioc != NULL); 81 82 ch = kmem_zalloc(sizeof (*ch), KM_SLEEP); 83 rw_init(&ch->ch_rwlock, NULL, RW_DRIVER, NULL); 84 mutex_init(&ch->ch_reflock, NULL, MUTEX_DRIVER, NULL); 85 cv_init(&ch->ch_cv_complete, NULL, CV_DRIVER, NULL); 86 ch->ch_refcnt = 1; 87 ch->ch_cv_waiters = 0; 88 89 ch->ch_state = SRPT_CHANNEL_CONNECTING; 90 ch->ch_tgt = tgt; 91 ch->ch_req_lim_delta = 0; 92 ch->ch_ti_iu_len = 0; 93 94 cq_attr.cq_size = srpt_send_msg_depth * 2; 95 cq_attr.cq_sched = 0; 96 cq_attr.cq_flags = IBT_CQ_NO_FLAGS; 97 98 status = ibt_alloc_cq(ioc->ioc_ibt_hdl, &cq_attr, &ch->ch_scq_hdl, 99 &cq_real_size); 100 if (status != IBT_SUCCESS) { 101 SRPT_DPRINTF_L1("ch_alloc, send CQ alloc error (%d)", 102 status); 103 goto scq_alloc_err; 104 } 105 106 cq_attr.cq_size = srpt_send_msg_depth + 1; 107 cq_attr.cq_sched = 0; 108 cq_attr.cq_flags = IBT_CQ_NO_FLAGS; 109 110 status = ibt_alloc_cq(ioc->ioc_ibt_hdl, &cq_attr, &ch->ch_rcq_hdl, 111 &cq_real_size); 112 if (status != IBT_SUCCESS) { 113 SRPT_DPRINTF_L2("ch_alloc, receive CQ alloc error (%d)", 114 status); 115 goto rcq_alloc_err; 116 } 117 118 ibt_set_cq_handler(ch->ch_scq_hdl, srpt_ch_scq_hdlr, ch); 119 ibt_set_cq_handler(ch->ch_rcq_hdl, srpt_ch_rcq_hdlr, ch); 120 (void) ibt_enable_cq_notify(ch->ch_scq_hdl, IBT_NEXT_COMPLETION); 121 (void) ibt_enable_cq_notify(ch->ch_rcq_hdl, IBT_NEXT_COMPLETION); 122 123 ch_args.rc_flags = IBT_WR_SIGNALED; 124 125 /* Maker certain initiator can not read/write our memory */ 126 ch_args.rc_control = 0; 127 128 ch_args.rc_hca_port_num = port; 129 130 /* 131 * Any SRP IU can result in a number of STMF data buffer transfers 132 * and those transfers themselves could span multiple initiator 133 * buffers. Therefore, the number of send WQE's actually required 134 * can vary. Here we assume that on average an I/O will require 135 * no more than SRPT_MAX_OUT_IO_PER_CMD send WQE's. In practice 136 * this will prevent send work queue overrun, but we will also 137 * inform STMF to throttle I/O should the work queue become full. 138 * 139 * If the HCA tells us the max outstanding WRs for a channel is 140 * lower than our default, use the HCA value. 141 */ 142 ch_args.rc_sizes.cs_sq = min(ioc->ioc_attr.hca_max_chan_sz, 143 (srpt_send_msg_depth * SRPT_MAX_OUT_IO_PER_CMD)); 144 ch_args.rc_sizes.cs_rq = 0; 145 ch_args.rc_sizes.cs_sq_sgl = 2; 146 ch_args.rc_sizes.cs_rq_sgl = 0; 147 148 ch_args.rc_scq = ch->ch_scq_hdl; 149 ch_args.rc_rcq = ch->ch_rcq_hdl; 150 ch_args.rc_pd = ioc->ioc_pd_hdl; 151 ch_args.rc_clone_chan = NULL; 152 ch_args.rc_srq = ioc->ioc_srq_hdl; 153 154 status = ibt_alloc_rc_channel(ioc->ioc_ibt_hdl, IBT_ACHAN_USES_SRQ, 155 &ch_args, &ch->ch_chan_hdl, &ch->ch_sizes); 156 if (status != IBT_SUCCESS) { 157 SRPT_DPRINTF_L2("ch_alloc, IBT channel alloc error (%d)", 158 status); 159 goto qp_alloc_err; 160 } 161 162 /* 163 * Create pool of send WQE entries to map send wqe work IDs 164 * to various types (specifically in error cases where OP 165 * is not known). 166 */ 167 ch->ch_num_swqe = ch->ch_sizes.cs_sq; 168 SRPT_DPRINTF_L2("ch_alloc, number of SWQEs = %u", ch->ch_num_swqe); 169 ch->ch_swqe = kmem_zalloc(sizeof (srpt_swqe_t) * ch->ch_num_swqe, 170 KM_SLEEP); 171 if (ch->ch_swqe == NULL) { 172 SRPT_DPRINTF_L2("ch_alloc, SWQE alloc error"); 173 (void) ibt_free_channel(ch->ch_chan_hdl); 174 goto qp_alloc_err; 175 } 176 mutex_init(&ch->ch_swqe_lock, NULL, MUTEX_DRIVER, NULL); 177 ch->ch_head = 1; 178 for (ch->ch_tail = 1; ch->ch_tail < ch->ch_num_swqe -1; ch->ch_tail++) { 179 ch->ch_swqe[ch->ch_tail].sw_next = ch->ch_tail + 1; 180 } 181 ch->ch_swqe[ch->ch_tail].sw_next = 0; 182 183 ibt_set_chan_private(ch->ch_chan_hdl, ch); 184 return (ch); 185 186 qp_alloc_err: 187 (void) ibt_free_cq(ch->ch_rcq_hdl); 188 189 rcq_alloc_err: 190 (void) ibt_free_cq(ch->ch_scq_hdl); 191 192 scq_alloc_err: 193 cv_destroy(&ch->ch_cv_complete); 194 mutex_destroy(&ch->ch_reflock); 195 rw_destroy(&ch->ch_rwlock); 196 kmem_free(ch, sizeof (*ch)); 197 198 return (NULL); 199 } 200 201 /* 202 * srpt_ch_add_ref() 203 */ 204 void 205 srpt_ch_add_ref(srpt_channel_t *ch) 206 { 207 mutex_enter(&ch->ch_reflock); 208 ch->ch_refcnt++; 209 SRPT_DPRINTF_L4("ch_add_ref, ch (%p), refcnt (%d)", 210 (void *)ch, ch->ch_refcnt); 211 ASSERT(ch->ch_refcnt != 0); 212 mutex_exit(&ch->ch_reflock); 213 } 214 215 /* 216 * srpt_ch_release_ref() 217 * 218 * A non-zero value for wait causes thread to block until all references 219 * to channel are released. 220 */ 221 void 222 srpt_ch_release_ref(srpt_channel_t *ch, uint_t wait) 223 { 224 mutex_enter(&ch->ch_reflock); 225 226 SRPT_DPRINTF_L4("ch_release_ref, ch (%p), refcnt (%d), wait (%d)", 227 (void *)ch, ch->ch_refcnt, wait); 228 229 ASSERT(ch->ch_refcnt != 0); 230 231 ch->ch_refcnt--; 232 233 if (ch->ch_refcnt != 0) { 234 if (wait) { 235 ch->ch_cv_waiters++; 236 while (ch->ch_refcnt != 0) { 237 cv_wait(&ch->ch_cv_complete, &ch->ch_reflock); 238 } 239 ch->ch_cv_waiters--; 240 } else { 241 mutex_exit(&ch->ch_reflock); 242 return; 243 } 244 } 245 246 /* 247 * Last thread out frees the IB resources, locks/conditions and memory 248 */ 249 if (ch->ch_cv_waiters > 0) { 250 /* we're not last, wake someone else up */ 251 cv_signal(&ch->ch_cv_complete); 252 mutex_exit(&ch->ch_reflock); 253 return; 254 } 255 256 SRPT_DPRINTF_L3("ch_release_ref - release resources"); 257 if (ch->ch_chan_hdl) { 258 SRPT_DPRINTF_L3("ch_release_ref - free channel"); 259 (void) ibt_free_channel(ch->ch_chan_hdl); 260 } 261 262 if (ch->ch_scq_hdl) { 263 (void) ibt_free_cq(ch->ch_scq_hdl); 264 } 265 266 if (ch->ch_rcq_hdl) { 267 (void) ibt_free_cq(ch->ch_rcq_hdl); 268 } 269 270 /* 271 * There should be no IU's associated with this 272 * channel on the SCSI session. 273 */ 274 if (ch->ch_session != NULL) { 275 ASSERT(list_is_empty(&ch->ch_session->ss_task_list)); 276 277 /* 278 * Currently only have one channel per session, we will 279 * need to release a reference when support is added 280 * for multi-channel target login. 281 */ 282 srpt_stp_free_session(ch->ch_session); 283 ch->ch_session = NULL; 284 } 285 286 kmem_free(ch->ch_swqe, sizeof (srpt_swqe_t) * ch->ch_num_swqe); 287 mutex_destroy(&ch->ch_swqe_lock); 288 mutex_exit(&ch->ch_reflock); 289 mutex_destroy(&ch->ch_reflock); 290 rw_destroy(&ch->ch_rwlock); 291 kmem_free(ch, sizeof (srpt_channel_t)); 292 } 293 294 /* 295 * srpt_ch_disconnect() 296 */ 297 void 298 srpt_ch_disconnect(srpt_channel_t *ch) 299 { 300 ibt_status_t status; 301 302 SRPT_DPRINTF_L3("ch_disconnect, invoked for ch (%p)", 303 (void *)ch); 304 305 rw_enter(&ch->ch_rwlock, RW_WRITER); 306 307 /* 308 * If we are already in the process of disconnecting then 309 * nothing need be done, CM will call-back into us when done. 310 */ 311 if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { 312 SRPT_DPRINTF_L2("ch_disconnect, called when" 313 " disconnect in progress"); 314 rw_exit(&ch->ch_rwlock); 315 return; 316 } 317 ch->ch_state = SRPT_CHANNEL_DISCONNECTING; 318 rw_exit(&ch->ch_rwlock); 319 320 /* 321 * Initiate the sending of the CM DREQ message, the private data 322 * should be the SRP Target logout IU. We don't really care about 323 * the remote CM DREP message returned. We issue this in an 324 * asynchronous manner and will cleanup when called back by CM. 325 */ 326 status = ibt_close_rc_channel(ch->ch_chan_hdl, IBT_NONBLOCKING, 327 NULL, 0, NULL, NULL, 0); 328 329 if (status != IBT_SUCCESS) { 330 SRPT_DPRINTF_L2("ch_disconnect, close RC channel" 331 " err(%d)", status); 332 } 333 } 334 335 /* 336 * srpt_ch_cleanup() 337 */ 338 void 339 srpt_ch_cleanup(srpt_channel_t *ch) 340 { 341 srpt_iu_t *iu; 342 srpt_iu_t *next; 343 ibt_wc_t wc; 344 srpt_target_port_t *tgt; 345 srpt_channel_t *tgt_ch; 346 scsi_task_t *iutask; 347 348 SRPT_DPRINTF_L3("ch_cleanup, invoked for ch(%p), state(%d)", 349 (void *)ch, ch->ch_state); 350 351 /* add a ref for the channel until we're done */ 352 srpt_ch_add_ref(ch); 353 354 tgt = ch->ch_tgt; 355 ASSERT(tgt != NULL); 356 357 /* 358 * Make certain the channel is in the target ports list of 359 * known channels and remove it (releasing the target 360 * ports reference to the channel). 361 */ 362 mutex_enter(&tgt->tp_ch_list_lock); 363 tgt_ch = list_head(&tgt->tp_ch_list); 364 while (tgt_ch != NULL) { 365 if (tgt_ch == ch) { 366 list_remove(&tgt->tp_ch_list, tgt_ch); 367 srpt_ch_release_ref(tgt_ch, 0); 368 break; 369 } 370 tgt_ch = list_next(&tgt->tp_ch_list, tgt_ch); 371 } 372 mutex_exit(&tgt->tp_ch_list_lock); 373 374 if (tgt_ch == NULL) { 375 SRPT_DPRINTF_L2("ch_cleanup, target channel no" 376 "longer known to target"); 377 srpt_ch_release_ref(ch, 0); 378 return; 379 } 380 381 rw_enter(&ch->ch_rwlock, RW_WRITER); 382 ch->ch_state = SRPT_CHANNEL_DISCONNECTING; 383 rw_exit(&ch->ch_rwlock); 384 385 /* 386 * Don't accept any further incoming requests, and clean 387 * up the receive queue. The send queue is left alone 388 * so tasks can finish and clean up (whether normally 389 * or via abort). 390 */ 391 if (ch->ch_rcq_hdl) { 392 ibt_set_cq_handler(ch->ch_rcq_hdl, NULL, NULL); 393 394 while (ibt_poll_cq(ch->ch_rcq_hdl, &wc, 1, NULL) == 395 IBT_SUCCESS) { 396 iu = (srpt_iu_t *)(uintptr_t)wc.wc_id; 397 SRPT_DPRINTF_L4("ch_cleanup, recovering" 398 " outstanding RX iu(%p)", (void *)iu); 399 mutex_enter(&iu->iu_lock); 400 srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); 401 /* 402 * Channel reference has not yet been added for this 403 * IU, so do not decrement. 404 */ 405 mutex_exit(&iu->iu_lock); 406 } 407 } 408 409 /* 410 * Go through the list of outstanding IU for the channel's SCSI 411 * session and for each either abort or complete an abort. 412 */ 413 rw_enter(&ch->ch_rwlock, RW_READER); 414 if (ch->ch_session != NULL) { 415 rw_enter(&ch->ch_session->ss_rwlock, RW_READER); 416 iu = list_head(&ch->ch_session->ss_task_list); 417 while (iu != NULL) { 418 next = list_next(&ch->ch_session->ss_task_list, iu); 419 420 mutex_enter(&iu->iu_lock); 421 if (ch == iu->iu_ch) { 422 if (iu->iu_stmf_task == NULL) { 423 cmn_err(CE_NOTE, 424 "ch_cleanup, NULL stmf task"); 425 ASSERT(0); 426 } 427 iutask = iu->iu_stmf_task; 428 } else { 429 iutask = NULL; 430 } 431 mutex_exit(&iu->iu_lock); 432 433 if (iutask != NULL) { 434 SRPT_DPRINTF_L4("ch_cleanup, aborting " 435 "task(%p)", (void *)iutask); 436 stmf_abort(STMF_QUEUE_TASK_ABORT, iutask, 437 STMF_ABORTED, NULL); 438 } 439 iu = next; 440 } 441 rw_exit(&ch->ch_session->ss_rwlock); 442 } 443 rw_exit(&ch->ch_rwlock); 444 445 srpt_ch_release_ref(ch, 0); 446 } 447 448 /* 449 * srpt_ch_rsp_comp() 450 * 451 * Process a completion for an IB SEND message. A SEND completion 452 * is for a SRP response packet sent back to the initiator. It 453 * will not have a STMF SCSI task associated with it if it was 454 * sent for a rejected IU, or was a task management abort response. 455 */ 456 static void 457 srpt_ch_rsp_comp(srpt_channel_t *ch, srpt_iu_t *iu, 458 ibt_wc_status_t wc_status) 459 { 460 stmf_status_t st = STMF_SUCCESS; 461 462 ASSERT(iu->iu_ch == ch); 463 464 /* 465 * Process the completion regardless whether it's a failure or 466 * success. At this point, we've processed as far as we can and 467 * just need to complete the associated task. 468 */ 469 470 if (wc_status != IBT_SUCCESS) { 471 SRPT_DPRINTF_L2("ch_rsp_comp, WC status err(%d)", 472 wc_status); 473 474 st = STMF_FAILURE; 475 476 if (wc_status != IBT_WC_WR_FLUSHED_ERR) { 477 srpt_ch_disconnect(ch); 478 } 479 } 480 481 /* 482 * If the IU response completion is not associated with 483 * with a SCSI task, release the IU to return the resource 484 * and the reference to the channel it holds. 485 */ 486 mutex_enter(&iu->iu_lock); 487 atomic_dec_32(&iu->iu_sq_posted_cnt); 488 489 if (iu->iu_stmf_task == NULL) { 490 srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); 491 mutex_exit(&iu->iu_lock); 492 srpt_ch_release_ref(ch, 0); 493 return; 494 } 495 496 /* 497 * We should not get a SEND completion where the task has already 498 * completed aborting and STMF has been informed. 499 */ 500 ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0); 501 502 /* 503 * Let STMF know we are done. 504 */ 505 mutex_exit(&iu->iu_lock); 506 507 stmf_send_status_done(iu->iu_stmf_task, st, STMF_IOF_LPORT_DONE); 508 } 509 510 /* 511 * srpt_ch_data_comp() 512 * 513 * Process an IB completion for a RDMA operation. This completion 514 * should be associated with the last RDMA operation for any 515 * data buffer transfer. 516 */ 517 static void 518 srpt_ch_data_comp(srpt_channel_t *ch, stmf_data_buf_t *stmf_dbuf, 519 ibt_wc_status_t wc_status) 520 { 521 srpt_ds_dbuf_t *dbuf; 522 srpt_iu_t *iu; 523 stmf_status_t status; 524 525 ASSERT(stmf_dbuf != NULL); 526 527 dbuf = (srpt_ds_dbuf_t *)stmf_dbuf->db_port_private; 528 529 ASSERT(dbuf != NULL); 530 531 iu = dbuf->db_iu; 532 533 ASSERT(iu != NULL); 534 ASSERT(iu->iu_ch == ch); 535 536 /* 537 * If work completion indicates non-flush failure, then 538 * start a channel disconnect (asynchronous) and release 539 * the reference to the IU. The task will be cleaned 540 * up with STMF during channel shutdown processing. 541 */ 542 if (wc_status != IBT_SUCCESS) { 543 SRPT_DPRINTF_L2("ch_data_comp, WC status err(%d)", 544 wc_status); 545 if (wc_status != IBT_WC_WR_FLUSHED_ERR) { 546 srpt_ch_disconnect(ch); 547 } 548 atomic_dec_32(&iu->iu_sq_posted_cnt); 549 return; 550 } 551 552 /* 553 * If STMF has requested this task be aborted, then if this is the 554 * last I/O operation outstanding, notify STMF the task has been 555 * aborted and ignore the completion. 556 */ 557 mutex_enter(&iu->iu_lock); 558 atomic_dec_32(&iu->iu_sq_posted_cnt); 559 560 if ((iu->iu_flags & SRPT_IU_STMF_ABORTING) != 0) { 561 scsi_task_t *abort_task = iu->iu_stmf_task; 562 563 mutex_exit(&iu->iu_lock); 564 stmf_abort(STMF_REQUEUE_TASK_ABORT_LPORT, abort_task, 565 STMF_ABORTED, NULL); 566 return; 567 } 568 569 /* 570 * We should not get an RDMA completion where the task has already 571 * completed aborting and STMF has been informed. 572 */ 573 ASSERT((iu->iu_flags & SRPT_IU_ABORTED) == 0); 574 575 /* 576 * Good completion for last RDMA op associated with a data buffer 577 * I/O, if specified initiate status otherwise let STMF know we are 578 * done. 579 */ 580 stmf_dbuf->db_xfer_status = STMF_SUCCESS; 581 mutex_exit(&iu->iu_lock); 582 583 DTRACE_SRP_8(xfer__done, srpt_channel_t, ch, 584 ibt_wr_ds_t, &(dbuf->db_sge), srpt_iu_t, iu, 585 ibt_send_wr_t, 0, uint32_t, stmf_dbuf->db_data_size, 586 uint32_t, 0, uint32_t, 0, 587 uint32_t, (stmf_dbuf->db_flags & DB_DIRECTION_TO_RPORT) ? 1 : 0); 588 589 if ((stmf_dbuf->db_flags & DB_SEND_STATUS_GOOD) != 0) { 590 status = srpt_stp_send_status(dbuf->db_iu->iu_stmf_task, 0); 591 if (status == STMF_SUCCESS) { 592 return; 593 } 594 stmf_dbuf->db_xfer_status = STMF_FAILURE; 595 } 596 stmf_data_xfer_done(dbuf->db_iu->iu_stmf_task, stmf_dbuf, 0); 597 } 598 599 /* 600 * srpt_ch_scq_hdlr() 601 */ 602 static void 603 srpt_ch_scq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg) 604 { 605 ibt_status_t status; 606 srpt_channel_t *ch = arg; 607 ibt_wc_t wc[SRPT_SEND_WC_POLL_SIZE]; 608 ibt_wc_t *wcp; 609 int i; 610 uint32_t cq_rearmed = 0; 611 uint32_t entries; 612 srpt_swqe_t *swqe; 613 614 ASSERT(ch != NULL); 615 616 /* Reference channel for the duration of this call */ 617 srpt_ch_add_ref(ch); 618 619 for (;;) { 620 status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_SEND_WC_POLL_SIZE, 621 &entries); 622 623 if (status != IBT_SUCCESS) { 624 if (status != IBT_CQ_EMPTY) { 625 /* 626 * This error should not happen. It indicates 627 * something abnormal has gone wrong and means 628 * either a hardware or programming logic error. 629 */ 630 SRPT_DPRINTF_L2( 631 "ch_scq_hdlr, unexpected CQ err(%d)", 632 status); 633 srpt_ch_disconnect(ch); 634 } 635 636 /* 637 * If we have not rearmed the CQ do so now and poll to 638 * eliminate race; otherwise we are done. 639 */ 640 if (cq_rearmed == 0) { 641 (void) ibt_enable_cq_notify(ch->ch_scq_hdl, 642 IBT_NEXT_COMPLETION); 643 cq_rearmed = 1; 644 continue; 645 } else { 646 break; 647 } 648 } 649 650 for (wcp = wc, i = 0; i < entries; i++, wcp++) { 651 652 /* 653 * A zero work ID indicates this CQE is associated 654 * with an intermediate post of a RDMA data transfer 655 * operation. Since intermediate data requests are 656 * unsignaled, we should only get these if there was 657 * an error. No action is required. 658 */ 659 if (wcp->wc_id == 0) { 660 continue; 661 } 662 swqe = ch->ch_swqe + wcp->wc_id; 663 664 switch (swqe->sw_type) { 665 case SRPT_SWQE_TYPE_RESP: 666 srpt_ch_rsp_comp(ch, (srpt_iu_t *) 667 swqe->sw_addr, wcp->wc_status); 668 break; 669 670 case SRPT_SWQE_TYPE_DATA: 671 srpt_ch_data_comp(ch, (stmf_data_buf_t *) 672 swqe->sw_addr, wcp->wc_status); 673 break; 674 675 default: 676 SRPT_DPRINTF_L2("ch_scq_hdlr, bad type(%d)", 677 swqe->sw_type); 678 ASSERT(0); 679 } 680 681 srpt_ch_free_swqe_wrid(ch, wcp->wc_id); 682 } 683 } 684 685 srpt_ch_release_ref(ch, 0); 686 } 687 688 /* 689 * srpt_ch_rcq_hdlr() 690 */ 691 static void 692 srpt_ch_rcq_hdlr(ibt_cq_hdl_t cq_hdl, void *arg) 693 { 694 ibt_status_t status; 695 srpt_channel_t *ch = arg; 696 ibt_wc_t wc[SRPT_RECV_WC_POLL_SIZE]; 697 ibt_wc_t *wcp; 698 int i; 699 uint32_t entries; 700 srpt_iu_t *iu; 701 uint_t cq_rearmed = 0; 702 703 /* 704 * The channel object will exists while the CQ handler call-back 705 * is installed. 706 */ 707 ASSERT(ch != NULL); 708 srpt_ch_add_ref(ch); 709 710 /* 711 * If we know a channel disconnect has started do nothing 712 * and let channel cleanup code recover resources from the CQ. 713 * We are not concerned about races with the state transition 714 * since the code will do the correct thing either way. This 715 * is simply to circumvent rearming the CQ, and it will 716 * catch the state next time. 717 */ 718 rw_enter(&ch->ch_rwlock, RW_READER); 719 if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { 720 SRPT_DPRINTF_L2("ch_rcq_hdlr, channel disconnecting"); 721 rw_exit(&ch->ch_rwlock); 722 srpt_ch_release_ref(ch, 0); 723 return; 724 } 725 rw_exit(&ch->ch_rwlock); 726 727 for (;;) { 728 status = ibt_poll_cq(cq_hdl, &wc[0], SRPT_RECV_WC_POLL_SIZE, 729 &entries); 730 731 if (status != IBT_SUCCESS) { 732 if (status != IBT_CQ_EMPTY) { 733 /* 734 * This error should not happen. It indicates 735 * something abnormal has gone wrong and means 736 * either a hardware or programming logic error. 737 */ 738 SRPT_DPRINTF_L2( 739 "ch_rcq_hdlr, unexpected CQ err(%d)", 740 status); 741 srpt_ch_disconnect(ch); 742 break; 743 } 744 745 /* 746 * If we have not rearmed the CQ do so now and poll to 747 * eliminate race; otherwise we are done. 748 */ 749 if (cq_rearmed == 0) { 750 (void) ibt_enable_cq_notify(ch->ch_rcq_hdl, 751 IBT_NEXT_COMPLETION); 752 cq_rearmed = 1; 753 continue; 754 } else { 755 break; 756 } 757 } 758 759 for (wcp = wc, i = 0; i < entries; i++, wcp++) { 760 761 /* 762 * Check wc_status before proceeding. If the 763 * status indicates a channel problem, stop processing. 764 */ 765 if (wcp->wc_status != IBT_WC_SUCCESS) { 766 if (wcp->wc_status == IBT_WC_WR_FLUSHED_ERR) { 767 SRPT_DPRINTF_L2( 768 "ch_rcq, unexpected" 769 " wc_status err(%d)", 770 wcp->wc_status); 771 srpt_ch_disconnect(ch); 772 goto done; 773 } else { 774 /* skip IUs with errors */ 775 SRPT_DPRINTF_L2( 776 "ch_rcq, ERROR comp(%d)", 777 wcp->wc_status); 778 /* XXX - verify not leaking IUs */ 779 continue; 780 } 781 } 782 783 iu = (srpt_iu_t *)(uintptr_t)wcp->wc_id; 784 ASSERT(iu != NULL); 785 786 /* 787 * Process the IU. 788 */ 789 ASSERT(wcp->wc_type == IBT_WRC_RECV); 790 srpt_ch_process_iu(ch, iu); 791 } 792 } 793 794 done: 795 srpt_ch_release_ref(ch, 0); 796 } 797 798 /* 799 * srpt_ch_srp_cmd() 800 */ 801 static int 802 srpt_ch_srp_cmd(srpt_channel_t *ch, srpt_iu_t *iu) 803 { 804 srp_cmd_req_t *cmd = (srp_cmd_req_t *)iu->iu_buf; 805 srp_indirect_desc_t *i_desc; 806 uint_t i_di_cnt; 807 uint_t i_do_cnt; 808 uint8_t do_fmt; 809 uint8_t di_fmt; 810 uint32_t *cur_desc_off; 811 int i; 812 ibt_status_t status; 813 uint8_t addlen; 814 815 816 DTRACE_SRP_2(task__command, srpt_channel_t, ch, srp_cmd_req_t, cmd); 817 iu->iu_ch = ch; 818 iu->iu_tag = cmd->cr_tag; 819 820 /* 821 * The SRP specification and SAM require support for bi-directional 822 * data transfer, so we create a single buffer descriptor list that 823 * in the IU buffer that covers the data-in and data-out buffers. 824 * In practice we will just see unidirectional transfers with either 825 * data-in or data out descriptors. If we were to take that as fact, 826 * we could reduce overhead slightly. 827 */ 828 829 /* 830 * additional length is a 6-bit number in 4-byte words, so multiply by 4 831 * to get bytes. 832 */ 833 addlen = cmd->cr_add_cdb_len & 0x3f; /* mask off 6 bits */ 834 835 cur_desc_off = (uint32_t *)(void *)&cmd->cr_add_data; 836 cur_desc_off += addlen; /* 32-bit arithmetic */ 837 iu->iu_num_rdescs = 0; 838 iu->iu_rdescs = (srp_direct_desc_t *)(void *)cur_desc_off; 839 840 /* 841 * Examine buffer description for Data In (i.e. data flows 842 * to the initiator). 843 */ 844 i_do_cnt = i_di_cnt = 0; 845 di_fmt = cmd->cr_buf_fmt >> 4; 846 if (di_fmt == SRP_DATA_DESC_DIRECT) { 847 iu->iu_num_rdescs = 1; 848 cur_desc_off = (uint32_t *)(void *)&iu->iu_rdescs[1]; 849 } else if (di_fmt == SRP_DATA_DESC_INDIRECT) { 850 i_desc = (srp_indirect_desc_t *)iu->iu_rdescs; 851 i_di_cnt = b2h32(i_desc->id_table.dd_len) / 852 sizeof (srp_direct_desc_t); 853 854 /* 855 * Some initiators like OFED occasionally use the wrong counts, 856 * so check total to allow for this. NOTE: we do not support 857 * reading of the descriptor table from the initiator, so if 858 * not all descriptors are in the IU we drop the task. 859 */ 860 if (i_di_cnt > (cmd->cr_dicnt + cmd->cr_docnt)) { 861 SRPT_DPRINTF_L2("ch_srp_cmd, remote RDMA of" 862 " descriptors not supported"); 863 SRPT_DPRINTF_L2("ch_srp_cmd, sizeof entry (%d)," 864 " i_di_cnt(%d), cr_dicnt(%d)", 865 (uint_t)sizeof (srp_direct_desc_t), 866 i_di_cnt, cmd->cr_dicnt); 867 iu->iu_rdescs = NULL; 868 return (1); 869 } 870 bcopy(&i_desc->id_desc[0], iu->iu_rdescs, 871 sizeof (srp_direct_desc_t) * i_di_cnt); 872 iu->iu_num_rdescs += i_di_cnt; 873 cur_desc_off = (uint32_t *)(void *)&i_desc->id_desc[i_di_cnt]; 874 } 875 876 /* 877 * Examine buffer description for Data Out (i.e. data flows 878 * from the initiator). 879 */ 880 do_fmt = cmd->cr_buf_fmt & 0x0F; 881 if (do_fmt == SRP_DATA_DESC_DIRECT) { 882 if (di_fmt == SRP_DATA_DESC_DIRECT) { 883 bcopy(cur_desc_off, &iu->iu_rdescs[iu->iu_num_rdescs], 884 sizeof (srp_direct_desc_t)); 885 } 886 iu->iu_num_rdescs++; 887 } else if (do_fmt == SRP_DATA_DESC_INDIRECT) { 888 i_desc = (srp_indirect_desc_t *)cur_desc_off; 889 i_do_cnt = b2h32(i_desc->id_table.dd_len) / 890 sizeof (srp_direct_desc_t); 891 892 /* 893 * Some initiators like OFED occasionally use the wrong counts, 894 * so check total to allow for this. NOTE: we do not support 895 * reading of the descriptor table from the initiator, so if 896 * not all descriptors are in the IU we drop the task. 897 */ 898 if ((i_di_cnt + i_do_cnt) > (cmd->cr_dicnt + cmd->cr_docnt)) { 899 SRPT_DPRINTF_L2("ch_srp_cmd, remote RDMA of" 900 " descriptors not supported"); 901 SRPT_DPRINTF_L2("ch_srp_cmd, sizeof entry (%d)," 902 " i_do_cnt(%d), cr_docnt(%d)", 903 (uint_t)sizeof (srp_direct_desc_t), 904 i_do_cnt, cmd->cr_docnt); 905 iu->iu_rdescs = 0; 906 return (1); 907 } 908 bcopy(&i_desc->id_desc[0], &iu->iu_rdescs[iu->iu_num_rdescs], 909 sizeof (srp_direct_desc_t) * i_do_cnt); 910 iu->iu_num_rdescs += i_do_cnt; 911 } 912 913 iu->iu_tot_xfer_len = 0; 914 for (i = 0; i < iu->iu_num_rdescs; i++) { 915 iu->iu_rdescs[i].dd_vaddr = b2h64(iu->iu_rdescs[i].dd_vaddr); 916 iu->iu_rdescs[i].dd_hdl = b2h32(iu->iu_rdescs[i].dd_hdl); 917 iu->iu_rdescs[i].dd_len = b2h32(iu->iu_rdescs[i].dd_len); 918 iu->iu_tot_xfer_len += iu->iu_rdescs[i].dd_len; 919 } 920 921 #ifdef DEBUG 922 if (srpt_errlevel >= SRPT_LOG_L4) { 923 SRPT_DPRINTF_L4("ch_srp_cmd, iu->iu_tot_xfer_len (%d)", 924 iu->iu_tot_xfer_len); 925 for (i = 0; i < iu->iu_num_rdescs; i++) { 926 SRPT_DPRINTF_L4("ch_srp_cmd, rdescs[%d].dd_vaddr" 927 " (0x%08llx)", 928 i, (u_longlong_t)iu->iu_rdescs[i].dd_vaddr); 929 SRPT_DPRINTF_L4("ch_srp_cmd, rdescs[%d].dd_hdl" 930 " (0x%08x)", i, iu->iu_rdescs[i].dd_hdl); 931 SRPT_DPRINTF_L4("ch_srp_cmd, rdescs[%d].dd_len (%d)", 932 i, iu->iu_rdescs[i].dd_len); 933 } 934 SRPT_DPRINTF_L4("ch_srp_cmd, LUN (0x%08lx)", 935 (unsigned long int) *((uint64_t *)(void *) cmd->cr_lun)); 936 } 937 #endif 938 rw_enter(&ch->ch_rwlock, RW_READER); 939 940 if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { 941 /* 942 * The channel has begun disconnecting, so ignore the 943 * the command returning the IU resources. 944 */ 945 rw_exit(&ch->ch_rwlock); 946 return (1); 947 } 948 949 /* 950 * Once a SCSI task is allocated and assigned to the IU, it 951 * owns those IU resources, which will be held until STMF 952 * is notified the task is done (from a lport perspective). 953 */ 954 iu->iu_stmf_task = stmf_task_alloc(ch->ch_tgt->tp_lport, 955 ch->ch_session->ss_ss, cmd->cr_lun, 956 SRP_CDB_SIZE + (addlen * 4), 0); 957 if (iu->iu_stmf_task == NULL) { 958 /* 959 * Could not allocate, return status to the initiator 960 * indicating that we are temporarily unable to process 961 * commands. If unable to send, immediately return IU 962 * resource. 963 */ 964 SRPT_DPRINTF_L2("ch_srp_cmd, SCSI task allocation failure"); 965 rw_exit(&ch->ch_rwlock); 966 mutex_enter(&iu->iu_lock); 967 status = srpt_stp_send_response(iu, STATUS_BUSY, 0, 0, 0, 968 NULL, SRPT_NO_FENCE_SEND); 969 mutex_exit(&iu->iu_lock); 970 if (status != IBT_SUCCESS) { 971 SRPT_DPRINTF_L2("ch_srp_cmd, error(%d) posting error" 972 " response", status); 973 return (1); 974 } else { 975 return (0); 976 } 977 } 978 979 iu->iu_stmf_task->task_port_private = iu; 980 iu->iu_stmf_task->task_flags = 0; 981 982 if (di_fmt != 0) { 983 iu->iu_stmf_task->task_flags |= TF_WRITE_DATA; 984 } 985 if (do_fmt != 0) { 986 iu->iu_stmf_task->task_flags |= TF_READ_DATA; 987 } 988 989 switch (cmd->cr_task_attr) { 990 case SRP_TSK_ATTR_QTYPE_SIMPLE: 991 iu->iu_stmf_task->task_flags |= TF_ATTR_SIMPLE_QUEUE; 992 break; 993 994 case SRP_TSK_ATTR_QTYPE_HEAD_OF_Q: 995 iu->iu_stmf_task->task_flags |= TF_ATTR_HEAD_OF_QUEUE; 996 break; 997 998 case SRP_TSK_ATTR_QTYPE_ORDERED: 999 iu->iu_stmf_task->task_flags |= TF_ATTR_ORDERED_QUEUE; 1000 break; 1001 1002 case SRP_TSK_ATTR_QTYPE_ACA_Q_TAG: 1003 iu->iu_stmf_task->task_flags |= TF_ATTR_ACA; 1004 break; 1005 1006 default: 1007 SRPT_DPRINTF_L2("ch_srp_cmd, reserved task attr (%d)", 1008 cmd->cr_task_attr); 1009 iu->iu_stmf_task->task_flags |= TF_ATTR_ORDERED_QUEUE; 1010 break; 1011 } 1012 iu->iu_stmf_task->task_additional_flags = 0; 1013 iu->iu_stmf_task->task_priority = 0; 1014 iu->iu_stmf_task->task_mgmt_function = TM_NONE; 1015 iu->iu_stmf_task->task_max_nbufs = STMF_BUFS_MAX; 1016 iu->iu_stmf_task->task_expected_xfer_length = iu->iu_tot_xfer_len; 1017 iu->iu_stmf_task->task_csn_size = 0; 1018 1019 bcopy(cmd->cr_cdb, iu->iu_stmf_task->task_cdb, 1020 SRP_CDB_SIZE); 1021 if (addlen != 0) { 1022 bcopy(&cmd->cr_add_data, 1023 iu->iu_stmf_task->task_cdb + SRP_CDB_SIZE, 1024 addlen * 4); 1025 } 1026 1027 /* 1028 * Add the IU/task to the session and post to STMF. The task will 1029 * remain in the session's list until STMF is informed by SRP that 1030 * it is done with the task. 1031 */ 1032 DTRACE_SRP_3(scsi__command, srpt_channel_t, iu->iu_ch, 1033 scsi_task_t, iu->iu_stmf_task, srp_cmd_req_t, cmd); 1034 srpt_stp_add_task(ch->ch_session, iu); 1035 1036 SRPT_DPRINTF_L3("ch_srp_cmd, new task (%p) posted", 1037 (void *)iu->iu_stmf_task); 1038 stmf_post_task(iu->iu_stmf_task, NULL); 1039 rw_exit(&ch->ch_rwlock); 1040 1041 return (0); 1042 } 1043 1044 /* 1045 * srpt_ch_task_mgmt_abort() 1046 * 1047 * Returns 0 on success, indicating we've sent a management response. 1048 * Returns !0 to indicate failure; the IU should be reposted. 1049 */ 1050 static ibt_status_t 1051 srpt_ch_task_mgmt_abort(srpt_channel_t *ch, srpt_iu_t *iu, 1052 uint64_t tag_to_abort) 1053 { 1054 srpt_session_t *session = ch->ch_session; 1055 srpt_iu_t *ss_iu; 1056 ibt_status_t status; 1057 1058 /* 1059 * Locate the associated task (tag_to_abort) in the 1060 * session's active task list. 1061 */ 1062 rw_enter(&session->ss_rwlock, RW_READER); 1063 ss_iu = list_head(&session->ss_task_list); 1064 while (ss_iu != NULL) { 1065 mutex_enter(&ss_iu->iu_lock); 1066 if ((tag_to_abort == ss_iu->iu_tag)) { 1067 mutex_exit(&ss_iu->iu_lock); 1068 break; 1069 } 1070 mutex_exit(&ss_iu->iu_lock); 1071 ss_iu = list_next(&session->ss_task_list, ss_iu); 1072 } 1073 rw_exit(&session->ss_rwlock); 1074 1075 /* 1076 * Take appropriate action based on state of task 1077 * to be aborted: 1078 * 1) No longer exists - do nothing. 1079 * 2) Previously aborted or status queued - do nothing. 1080 * 3) Otherwise - initiate abort. 1081 */ 1082 if (ss_iu == NULL) { 1083 goto send_mgmt_resp; 1084 } 1085 1086 mutex_enter(&ss_iu->iu_lock); 1087 if ((ss_iu->iu_flags & (SRPT_IU_STMF_ABORTING | 1088 SRPT_IU_ABORTED | SRPT_IU_RESP_SENT)) != 0) { 1089 mutex_exit(&ss_iu->iu_lock); 1090 goto send_mgmt_resp; 1091 } 1092 1093 /* 1094 * Set aborting flag and notify STMF of abort request. No 1095 * additional I/O will be queued for this IU. 1096 */ 1097 SRPT_DPRINTF_L3("ch_task_mgmt_abort, task found"); 1098 ss_iu->iu_flags |= SRPT_IU_SRP_ABORTING; 1099 mutex_exit(&ss_iu->iu_lock); 1100 stmf_abort(STMF_QUEUE_TASK_ABORT, 1101 ss_iu->iu_stmf_task, STMF_ABORTED, NULL); 1102 1103 send_mgmt_resp: 1104 mutex_enter(&iu->iu_lock); 1105 status = srpt_stp_send_mgmt_response(iu, SRP_TM_SUCCESS, 1106 SRPT_FENCE_SEND); 1107 mutex_exit(&iu->iu_lock); 1108 1109 if (status != IBT_SUCCESS) { 1110 SRPT_DPRINTF_L2("ch_task_mgmt_abort, err(%d)" 1111 " posting abort response", status); 1112 } 1113 1114 return (status); 1115 } 1116 1117 /* 1118 * srpt_ch_srp_task_mgmt() 1119 */ 1120 static int 1121 srpt_ch_srp_task_mgmt(srpt_channel_t *ch, srpt_iu_t *iu) 1122 { 1123 srp_tsk_mgmt_t *tsk = (srp_tsk_mgmt_t *)iu->iu_buf; 1124 uint8_t tm_fn; 1125 ibt_status_t status; 1126 1127 SRPT_DPRINTF_L3("ch_srp_task_mgmt, SRP TASK MGMT func(%d)", 1128 tsk->tm_function); 1129 1130 /* 1131 * Both tag and lun fileds have the same corresponding offsets 1132 * in both srp_tsk_mgmt_t and srp_cmd_req_t structures. The 1133 * casting will allow us to use the same dtrace translator. 1134 */ 1135 DTRACE_SRP_2(task__command, srpt_channel_t, ch, 1136 srp_cmd_req_t, (srp_cmd_req_t *)tsk); 1137 1138 iu->iu_ch = ch; 1139 iu->iu_tag = tsk->tm_tag; 1140 1141 /* 1142 * Task management aborts are processed directly by the SRP driver; 1143 * all other task management requests are handed off to STMF. 1144 */ 1145 switch (tsk->tm_function) { 1146 case SRP_TSK_MGMT_ABORT_TASK: 1147 /* 1148 * Initiate SCSI transport protocol specific task abort 1149 * logic. 1150 */ 1151 status = srpt_ch_task_mgmt_abort(ch, iu, tsk->tm_task_tag); 1152 if (status != IBT_SUCCESS) { 1153 /* repost this IU */ 1154 return (1); 1155 } else { 1156 return (0); 1157 } 1158 1159 case SRP_TSK_MGMT_ABORT_TASK_SET: 1160 tm_fn = TM_ABORT_TASK_SET; 1161 break; 1162 1163 case SRP_TSK_MGMT_CLEAR_TASK_SET: 1164 tm_fn = TM_CLEAR_TASK_SET; 1165 break; 1166 1167 case SRP_TSK_MGMT_LUN_RESET: 1168 tm_fn = TM_LUN_RESET; 1169 break; 1170 1171 case SRP_TSK_MGMT_CLEAR_ACA: 1172 tm_fn = TM_CLEAR_ACA; 1173 break; 1174 1175 default: 1176 /* 1177 * SRP does not support the requested task management 1178 * function; return a not supported status in the response. 1179 */ 1180 SRPT_DPRINTF_L2("ch_srp_task_mgmt, SRP task mgmt fn(%d)" 1181 " not supported", tsk->tm_function); 1182 mutex_enter(&iu->iu_lock); 1183 status = srpt_stp_send_mgmt_response(iu, 1184 SRP_TM_NOT_SUPPORTED, SRPT_NO_FENCE_SEND); 1185 mutex_exit(&iu->iu_lock); 1186 if (status != IBT_SUCCESS) { 1187 SRPT_DPRINTF_L2("ch_srp_task_mgmt, err(%d) posting" 1188 " response", status); 1189 return (1); 1190 } 1191 return (0); 1192 } 1193 1194 rw_enter(&ch->ch_rwlock, RW_READER); 1195 if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { 1196 /* 1197 * The channel has begun disconnecting, so ignore the 1198 * the command returning the IU resources. 1199 */ 1200 rw_exit(&ch->ch_rwlock); 1201 return (1); 1202 } 1203 1204 /* 1205 * Once a SCSI mgmt task is allocated and assigned to the IU, it 1206 * owns those IU resources, which will be held until we inform 1207 * STMF that we are done with the task (from an lports perspective). 1208 */ 1209 iu->iu_stmf_task = stmf_task_alloc(ch->ch_tgt->tp_lport, 1210 ch->ch_session->ss_ss, tsk->tm_lun, 0, STMF_TASK_EXT_NONE); 1211 if (iu->iu_stmf_task == NULL) { 1212 /* 1213 * Could not allocate, return status to the initiator 1214 * indicating that we are temporarily unable to process 1215 * commands. If unable to send, immediately return IU 1216 * resource. 1217 */ 1218 SRPT_DPRINTF_L2("ch_srp_task_mgmt, SCSI task allocation" 1219 " failure"); 1220 rw_exit(&ch->ch_rwlock); 1221 mutex_enter(&iu->iu_lock); 1222 status = srpt_stp_send_response(iu, STATUS_BUSY, 0, 0, 0, 1223 NULL, SRPT_NO_FENCE_SEND); 1224 mutex_exit(&iu->iu_lock); 1225 if (status != IBT_SUCCESS) { 1226 SRPT_DPRINTF_L2("ch_srp_task_mgmt, err(%d) posting" 1227 "busy response", status); 1228 /* repost the IU */ 1229 return (1); 1230 } 1231 return (0); 1232 } 1233 1234 iu->iu_stmf_task->task_port_private = iu; 1235 iu->iu_stmf_task->task_flags = 0; 1236 iu->iu_stmf_task->task_additional_flags = 1237 TASK_AF_NO_EXPECTED_XFER_LENGTH; 1238 iu->iu_stmf_task->task_priority = 0; 1239 iu->iu_stmf_task->task_mgmt_function = tm_fn; 1240 iu->iu_stmf_task->task_max_nbufs = STMF_BUFS_MAX; 1241 iu->iu_stmf_task->task_expected_xfer_length = 0; 1242 iu->iu_stmf_task->task_csn_size = 0; 1243 1244 /* 1245 * Add the IU/task to the session and post to STMF. The task will 1246 * remain in the session's list until STMF is informed by SRP that 1247 * it is done with the task. 1248 */ 1249 srpt_stp_add_task(ch->ch_session, iu); 1250 1251 SRPT_DPRINTF_L3("ch_srp_task_mgmt, new mgmt task(%p) posted", 1252 (void *)iu->iu_stmf_task); 1253 stmf_post_task(iu->iu_stmf_task, NULL); 1254 rw_exit(&ch->ch_rwlock); 1255 1256 return (0); 1257 } 1258 1259 /* 1260 * srpt_ch_process_iu() 1261 */ 1262 static void 1263 srpt_ch_process_iu(srpt_channel_t *ch, srpt_iu_t *iu) 1264 { 1265 srpt_iu_data_t *iud; 1266 int status = 1; 1267 1268 /* 1269 * IU adds reference to channel which will represent a 1270 * a reference by STMF. If for whatever reason the IU 1271 * is not handed off to STMF, then this reference will be 1272 * released. Otherwise, the reference will be released when 1273 * SRP informs STMF that the associated SCSI task is done. 1274 */ 1275 srpt_ch_add_ref(ch); 1276 1277 /* 1278 * Validate login RC channel state. Normally active, if 1279 * not active then we need to handle a possible race between the 1280 * receipt of a implied RTU and CM calling back to notify of the 1281 * state transition. 1282 */ 1283 rw_enter(&ch->ch_rwlock, RW_READER); 1284 if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { 1285 rw_exit(&ch->ch_rwlock); 1286 goto repost_iu; 1287 } 1288 rw_exit(&ch->ch_rwlock); 1289 1290 iud = iu->iu_buf; 1291 1292 switch (iud->rx_iu.srp_op) { 1293 case SRP_IU_CMD: 1294 status = srpt_ch_srp_cmd(ch, iu); 1295 break; 1296 1297 case SRP_IU_TASK_MGMT: 1298 status = srpt_ch_srp_task_mgmt(ch, iu); 1299 return; 1300 1301 case SRP_IU_I_LOGOUT: 1302 SRPT_DPRINTF_L3("ch_process_iu, SRP INITIATOR LOGOUT"); 1303 /* 1304 * Initiators should logout by issuing a CM disconnect 1305 * request (DREQ) with the logout IU in the private data; 1306 * however some initiators have been known to send the 1307 * IU in-band, if this happens just initiate the logout. 1308 * Note that we do not return a response as per the 1309 * specification. 1310 */ 1311 srpt_stp_logout(ch); 1312 break; 1313 1314 case SRP_IU_AER_RSP: 1315 case SRP_IU_CRED_RSP: 1316 default: 1317 /* 1318 * We don't send asynchronous events or ask for credit 1319 * adjustments, so nothing need be done. Log we got an 1320 * unexpected IU but then just repost the IU to the SRQ. 1321 */ 1322 SRPT_DPRINTF_L2("ch_process_iu, invalid IU from initiator," 1323 " IU opcode(%d)", iud->rx_iu.srp_op); 1324 break; 1325 } 1326 1327 if (status == 0) { 1328 return; 1329 } 1330 1331 repost_iu: 1332 SRPT_DPRINTF_L4("process_iu: reposting iu %p", (void *)iu); 1333 mutex_enter(&iu->iu_lock); 1334 srpt_ioc_repost_recv_iu(iu->iu_ioc, iu); 1335 mutex_exit(&iu->iu_lock); 1336 srpt_ch_release_ref(ch, 0); 1337 } 1338 1339 /* 1340 * srpt_ch_post_send 1341 */ 1342 ibt_status_t 1343 srpt_ch_post_send(srpt_channel_t *ch, srpt_iu_t *iu, uint32_t len, 1344 uint_t fence) 1345 { 1346 ibt_status_t status; 1347 ibt_send_wr_t wr; 1348 ibt_wr_ds_t ds; 1349 uint_t posted; 1350 1351 ASSERT(ch != NULL); 1352 ASSERT(iu != NULL); 1353 ASSERT(mutex_owned(&iu->iu_lock)); 1354 1355 rw_enter(&ch->ch_rwlock, RW_READER); 1356 if (ch->ch_state == SRPT_CHANNEL_DISCONNECTING) { 1357 rw_exit(&ch->ch_rwlock); 1358 SRPT_DPRINTF_L2("ch_post_send, bad ch state (%d)", 1359 ch->ch_state); 1360 return (IBT_FAILURE); 1361 } 1362 rw_exit(&ch->ch_rwlock); 1363 1364 wr.wr_id = srpt_ch_alloc_swqe_wrid(ch, SRPT_SWQE_TYPE_RESP, 1365 (void *)iu); 1366 if (wr.wr_id == 0) { 1367 SRPT_DPRINTF_L2("ch_post_send, queue full"); 1368 return (IBT_FAILURE); 1369 } 1370 1371 atomic_inc_32(&iu->iu_sq_posted_cnt); 1372 1373 wr.wr_flags = IBT_WR_SEND_SIGNAL; 1374 if (fence == SRPT_FENCE_SEND) { 1375 wr.wr_flags |= IBT_WR_SEND_FENCE; 1376 } 1377 wr.wr_opcode = IBT_WRC_SEND; 1378 wr.wr_trans = IBT_RC_SRV; 1379 wr.wr_nds = 1; 1380 wr.wr_sgl = &ds; 1381 1382 ds.ds_va = iu->iu_sge.ds_va; 1383 ds.ds_key = iu->iu_sge.ds_key; 1384 ds.ds_len = len; 1385 1386 SRPT_DPRINTF_L4("ch_post_send, posting SRP response to channel" 1387 " ds.ds_va (0x%16llx), ds.ds_key (0x%08x), " 1388 " ds.ds_len (%d)", 1389 (u_longlong_t)ds.ds_va, ds.ds_key, ds.ds_len); 1390 1391 status = ibt_post_send(ch->ch_chan_hdl, &wr, 1, &posted); 1392 if (status != IBT_SUCCESS) { 1393 SRPT_DPRINTF_L2("ch_post_send, post_send failed (%d)", 1394 status); 1395 atomic_dec_32(&iu->iu_sq_posted_cnt); 1396 srpt_ch_free_swqe_wrid(ch, wr.wr_id); 1397 return (status); 1398 } 1399 1400 return (IBT_SUCCESS); 1401 } 1402 1403 /* 1404 * srpt_ch_alloc_swqe_wrid() 1405 */ 1406 ibt_wrid_t 1407 srpt_ch_alloc_swqe_wrid(srpt_channel_t *ch, 1408 srpt_swqe_type_t wqe_type, void *addr) 1409 { 1410 ibt_wrid_t wrid; 1411 1412 mutex_enter(&ch->ch_swqe_lock); 1413 if (ch->ch_head == ch->ch_tail) { 1414 mutex_exit(&ch->ch_swqe_lock); 1415 return ((ibt_wrid_t)0); 1416 } 1417 wrid = (ibt_wrid_t)ch->ch_head; 1418 ch->ch_swqe[ch->ch_head].sw_type = wqe_type; 1419 ch->ch_swqe[ch->ch_head].sw_addr = addr; 1420 ch->ch_head = ch->ch_swqe[ch->ch_head].sw_next; 1421 ch->ch_swqe_posted++; 1422 mutex_exit(&ch->ch_swqe_lock); 1423 return (wrid); 1424 } 1425 1426 /* 1427 * srpt_ch_free_swqe_wrid() 1428 */ 1429 void 1430 srpt_ch_free_swqe_wrid(srpt_channel_t *ch, ibt_wrid_t id) 1431 { 1432 mutex_enter(&ch->ch_swqe_lock); 1433 ch->ch_swqe[ch->ch_tail].sw_next = id; 1434 ch->ch_tail = (uint32_t)id; 1435 ch->ch_swqe_posted--; 1436 mutex_exit(&ch->ch_swqe_lock); 1437 } 1438