1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/ddi.h> 28 #include <sys/types.h> 29 #include <sys/socket.h> 30 #include <netinet/in.h> 31 #include <sys/sunddi.h> 32 #include <sys/sdt.h> 33 #include <sys/ib/ibtl/ibti.h> 34 #include <sys/ib/ibtl/ibtl_types.h> 35 36 #include <sys/ib/clients/iser/iser.h> 37 38 /* 39 * iser_cq.c 40 * Routines for completion queue handlers for iSER. 41 */ 42 static void iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg); 43 int iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg); 44 static int iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, 45 iser_chan_t *iser_chan); 46 static int iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, 47 iser_chan_t *iser_chan); 48 49 void 50 iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg) 51 { 52 iser_chan_t *iser_chan; 53 ibt_status_t status; 54 55 iser_chan = (iser_chan_t *)arg; 56 57 /* 58 * Poll for work request completion while successful. If the 59 * queue empties or otherwise becomes invalid, stop polling. 60 */ 61 do { 62 status = iser_ib_poll_send_completions(cq_hdl, iser_chan); 63 } while (status == IBT_SUCCESS); 64 65 if (status == IBT_CQ_EMPTY) { 66 /* We've emptied the CQ, rearm it before we're done here */ 67 status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION); 68 if (status != IBT_SUCCESS) { 69 /* Unexpected error */ 70 ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: " 71 "ibt_enable_cq_notify error (%d)", status); 72 return; 73 } 74 75 /* Now, check for more completions after the rearm */ 76 do { 77 status = iser_ib_poll_send_completions( 78 cq_hdl, iser_chan); 79 } while (status == IBT_SUCCESS); 80 } 81 } 82 83 static int 84 iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan) 85 { 86 ibt_wc_t wc[ISER_IB_SCQ_POLL_MAX]; 87 ibt_wrid_t wrid; 88 idm_buf_t *idb = NULL; 89 idm_task_t *idt = NULL; 90 iser_wr_t *wr = NULL; 91 int i; 92 uint_t npoll = 0; 93 ibt_status_t status; 94 iser_conn_t *iser_conn; 95 idm_status_t idm_status; 96 iser_mr_t *mr; 97 98 iser_conn = iser_chan->ic_conn; 99 100 /* Poll ISER_IB_SCQ_POLL_MAX completions from the CQ */ 101 status = ibt_poll_cq(cq_hdl, wc, ISER_IB_SCQ_POLL_MAX, &npoll); 102 103 if (status != IBT_SUCCESS) { 104 if (status != IBT_CQ_EMPTY) { 105 /* Unexpected error */ 106 ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: ibt_poll_cq " 107 "unexpected error (%d)", status); 108 } 109 /* CQ is empty. Either way, move along... */ 110 return (status); 111 } 112 113 /* 114 * Handle each of the completions we've polled 115 */ 116 for (i = 0; i < npoll; i++) { 117 118 DTRACE_PROBE3(iser__send__cqe, iser_chan_t *, iser_chan, 119 ibt_wc_t *, &wc[i], ibt_wc_status_t, wc[i].wc_status); 120 121 /* Grab the wrid of the completion */ 122 wrid = wc[i].wc_id; 123 124 /* Decrement this channel's SQ posted count */ 125 mutex_enter(&iser_chan->ic_sq_post_lock); 126 iser_chan->ic_sq_post_count--; 127 mutex_exit(&iser_chan->ic_sq_post_lock); 128 129 /* Pull in the wr handle */ 130 wr = (iser_wr_t *)(uintptr_t)wrid; 131 ASSERT(wr != NULL); 132 133 /* Set an idm_status for return to IDM */ 134 idm_status = (wc[i].wc_status == IBT_WC_SUCCESS) ? 135 IDM_STATUS_SUCCESS : IDM_STATUS_FAIL; 136 137 /* 138 * A non-success status here indicates the QP went 139 * into an error state while this WR was being 140 * processed. This can also happen when the 141 * channel is closed on the remote end. Clean up 142 * the resources, then push CE_TRANSPORT_FAIL 143 * into IDM. 144 */ 145 if (wc[i].wc_status != IBT_WC_SUCCESS) { 146 /* 147 * Free the resources attached to this 148 * completion. 149 */ 150 if (wr->iw_msg != NULL) { 151 /* Free iser_msg handle */ 152 iser_msg_free(wr->iw_msg); 153 } 154 155 if (wr->iw_pdu != NULL) { 156 /* Complete the PDU */ 157 idm_pdu_complete(wr->iw_pdu, idm_status); 158 } 159 160 if (wr->iw_buf != NULL) { 161 /* Invoke buffer callback */ 162 idb = wr->iw_buf; 163 mr = ((iser_buf_t *) 164 idb->idb_buf_private)->iser_mr; 165 #ifdef DEBUG 166 bcopy(&wc[i], 167 &((iser_buf_t *)idb->idb_buf_private)-> 168 buf_wc, sizeof (ibt_wc_t)); 169 #endif 170 idt = idb->idb_task_binding; 171 mutex_enter(&idt->idt_mutex); 172 if (wr->iw_type == ISER_WR_RDMAW) { 173 DTRACE_ISCSI_8(xfer__done, 174 idm_conn_t *, idt->idt_ic, 175 uintptr_t, idb->idb_buf, 176 uint32_t, idb->idb_bufoffset, 177 uint64_t, mr->is_mrva, uint32_t, 0, 178 uint32_t, mr->is_mrrkey, 179 uint32_t, idb->idb_xfer_len, 180 int, XFER_BUF_TX_TO_INI); 181 idm_buf_tx_to_ini_done(idt, idb, 182 IDM_STATUS_FAIL); 183 } else { /* ISER_WR_RDMAR */ 184 DTRACE_ISCSI_8(xfer__done, 185 idm_conn_t *, idt->idt_ic, 186 uintptr_t, idb->idb_buf, 187 uint32_t, idb->idb_bufoffset, 188 uint64_t, mr->is_mrva, uint32_t, 0, 189 uint32_t, mr->is_mrrkey, 190 uint32_t, idb->idb_xfer_len, 191 int, XFER_BUF_RX_FROM_INI); 192 idm_buf_rx_from_ini_done(idt, idb, 193 IDM_STATUS_FAIL); 194 } 195 } 196 197 /* Free the iser wr handle */ 198 iser_wr_free(wr); 199 200 /* 201 * Tell IDM that the channel has gone down, 202 * unless he already knows. 203 */ 204 mutex_enter(&iser_conn->ic_lock); 205 switch (iser_conn->ic_stage) { 206 case ISER_CONN_STAGE_IC_DISCONNECTED: 207 case ISER_CONN_STAGE_IC_FREED: 208 case ISER_CONN_STAGE_CLOSING: 209 case ISER_CONN_STAGE_CLOSED: 210 break; 211 212 default: 213 idm_conn_event(iser_conn->ic_idmc, 214 CE_TRANSPORT_FAIL, idm_status); 215 iser_conn->ic_stage = ISER_CONN_STAGE_CLOSING; 216 } 217 mutex_exit(&iser_conn->ic_lock); 218 219 /* Move onto the next completion */ 220 continue; 221 } 222 223 /* 224 * For a success status, just invoke the PDU or 225 * buffer completion. We use our WR handle's 226 * "iw_type" here so that we can properly process 227 * because the CQE's opcode is invalid if the status 228 * is failed. 229 */ 230 switch (wr->iw_type) { 231 case ISER_WR_SEND: 232 /* Free the msg handle */ 233 ASSERT(wr->iw_msg != NULL); 234 iser_msg_free(wr->iw_msg); 235 236 if (wr->iw_pdu == NULL) { 237 /* This is a hello exchange message */ 238 mutex_enter(&iser_conn->ic_lock); 239 if (iser_conn->ic_stage == 240 ISER_CONN_STAGE_HELLOREPLY_SENT) { 241 /* 242 * We're on the target side, 243 * and have just successfully 244 * sent the HelloReply msg. 245 */ 246 iser_conn->ic_stage = 247 ISER_CONN_STAGE_LOGGED_IN; 248 } 249 mutex_exit(&iser_conn->ic_lock); 250 } else { 251 /* This is a normal control message */ 252 idm_pdu_complete(wr->iw_pdu, idm_status); 253 } 254 255 /* Free the wr handle */ 256 iser_wr_free(wr); 257 258 break; 259 260 case ISER_WR_RDMAW: 261 case ISER_WR_RDMAR: 262 /* 263 * Invoke the appropriate callback; 264 * the buffer will be freed there. 265 */ 266 idb = wr->iw_buf; 267 mr = ((iser_buf_t *)idb->idb_buf_private)->iser_mr; 268 #ifdef DEBUG 269 bcopy(&wc[i], 270 &((iser_buf_t *)idb->idb_buf_private)->buf_wc, 271 sizeof (ibt_wc_t)); 272 #endif 273 idt = idb->idb_task_binding; 274 275 mutex_enter(&idt->idt_mutex); 276 if (wr->iw_type == ISER_WR_RDMAW) { 277 DTRACE_ISCSI_8(xfer__done, 278 idm_conn_t *, idt->idt_ic, 279 uintptr_t, idb->idb_buf, 280 uint32_t, idb->idb_bufoffset, 281 uint64_t, mr->is_mrva, uint32_t, 0, 282 uint32_t, mr->is_mrrkey, 283 uint32_t, idb->idb_xfer_len, 284 int, XFER_BUF_TX_TO_INI); 285 idm_buf_tx_to_ini_done(idt, idb, idm_status); 286 } else { 287 DTRACE_ISCSI_8(xfer__done, 288 idm_conn_t *, idt->idt_ic, 289 uintptr_t, idb->idb_buf, 290 uint32_t, idb->idb_bufoffset, 291 uint64_t, mr->is_mrva, uint32_t, 0, 292 uint32_t, mr->is_mrrkey, 293 uint32_t, idb->idb_xfer_len, 294 int, XFER_BUF_RX_FROM_INI); 295 idm_buf_rx_from_ini_done(idt, idb, idm_status); 296 } 297 298 /* Free the wr handle */ 299 iser_wr_free(wr); 300 301 break; 302 303 default: 304 ASSERT(0); 305 break; 306 } 307 } 308 309 return (status); 310 } 311 312 void 313 iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg) 314 { 315 iser_chan_t *iser_chan; 316 ibt_status_t status; 317 318 iser_chan = (iser_chan_t *)arg; 319 320 /* 321 * Poll for work request completion while successful. If the 322 * queue empties or otherwise becomes invalid, stop polling. 323 */ 324 do { 325 status = iser_ib_poll_recv_completions(cq_hdl, iser_chan); 326 } while (status == IBT_SUCCESS); 327 328 if (status == IBT_CQ_EMPTY) { 329 /* We've emptied the CQ, rearm it before we're done here */ 330 status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION); 331 if (status != IBT_SUCCESS) { 332 /* Unexpected error */ 333 ISER_LOG(CE_NOTE, "iser_ib_recvcq_handler: " 334 "ibt_enable_cq_notify error (%d)", status); 335 return; 336 } 337 338 /* Now, check for more completions after the rearm */ 339 do { 340 status = iser_ib_poll_recv_completions( 341 cq_hdl, iser_chan); 342 } while (status == IBT_SUCCESS); 343 } 344 } 345 346 static int 347 iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan) 348 { 349 ibt_wc_t wc; 350 iser_msg_t *msg; 351 iser_qp_t *iser_qp; 352 int status; 353 354 iser_qp = &(iser_chan->ic_qp); 355 356 bzero(&wc, sizeof (ibt_wc_t)); 357 status = ibt_poll_cq(cq_hdl, &wc, 1, NULL); 358 if (status == IBT_CQ_EMPTY) { 359 /* CQ is empty, return */ 360 return (status); 361 } 362 363 if (status != IBT_SUCCESS) { 364 /* Unexpected error */ 365 ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: " 366 "ibt_poll_cq error (%d)", status); 367 mutex_enter(&iser_qp->qp_lock); 368 iser_qp->rq_level--; 369 mutex_exit(&iser_qp->qp_lock); 370 /* Free the msg handle (if we got it back) */ 371 if ((msg = (iser_msg_t *)(uintptr_t)wc.wc_id) != NULL) { 372 iser_msg_free(msg); 373 } 374 return (status); 375 } 376 377 /* Retrieve the iSER msg handle */ 378 msg = (iser_msg_t *)(uintptr_t)wc.wc_id; 379 ASSERT(msg != NULL); 380 381 /* 382 * Decrement the posted level in the RQ, then check 383 * to see if we need to fill the RQ back up (or if 384 * we are already on the taskq). 385 */ 386 mutex_enter(&iser_chan->ic_conn->ic_lock); 387 mutex_enter(&iser_qp->qp_lock); 388 iser_qp->rq_level--; 389 390 if ((iser_qp->rq_taskqpending == B_FALSE) && 391 (iser_qp->rq_level <= iser_qp->rq_lwm) && 392 (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) && 393 (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) { 394 /* Set the pending flag and fire off a post_recv */ 395 iser_qp->rq_taskqpending = B_TRUE; 396 mutex_exit(&iser_qp->qp_lock); 397 398 status = iser_ib_post_recv_async(iser_chan->ic_chanhdl); 399 400 if (status != DDI_SUCCESS) { 401 ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: " 402 "task dispatch failed"); 403 /* Failure to launch, unset the pending flag */ 404 mutex_enter(&iser_qp->qp_lock); 405 iser_qp->rq_taskqpending = B_FALSE; 406 mutex_exit(&iser_qp->qp_lock); 407 } 408 } else { 409 mutex_exit(&iser_qp->qp_lock); 410 } 411 412 DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan, 413 ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status); 414 if (wc.wc_status != IBT_WC_SUCCESS) { 415 /* 416 * Tell IDM that the channel has gone down, 417 * unless he already knows. 418 */ 419 switch (iser_chan->ic_conn->ic_stage) { 420 case ISER_CONN_STAGE_IC_DISCONNECTED: 421 case ISER_CONN_STAGE_IC_FREED: 422 case ISER_CONN_STAGE_CLOSING: 423 case ISER_CONN_STAGE_CLOSED: 424 break; 425 426 default: 427 idm_conn_event(iser_chan->ic_conn->ic_idmc, 428 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 429 iser_chan->ic_conn->ic_stage = 430 ISER_CONN_STAGE_CLOSING; 431 } 432 mutex_exit(&iser_chan->ic_conn->ic_lock); 433 434 iser_msg_free(msg); 435 return (DDI_SUCCESS); 436 } else { 437 mutex_exit(&iser_chan->ic_conn->ic_lock); 438 439 /* 440 * We have an iSER message in, let's handle it. 441 * We will free the iser_msg_t later in this path, 442 * depending upon the action required. 443 */ 444 iser_msg_handle(iser_chan, msg); 445 return (DDI_SUCCESS); 446 } 447 } 448 449 static void 450 iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg) 451 { 452 int opcode; 453 iser_ctrl_hdr_t *hdr = NULL; 454 iser_conn_t *iser_conn = chan->ic_conn; 455 int status; 456 457 hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 458 ASSERT(hdr != NULL); 459 460 opcode = hdr->opcode; 461 if (opcode == ISER_OPCODE_CTRL_TYPE_PDU) { 462 /* 463 * Handle an iSCSI Control PDU iSER message. 464 * Note we'll free the msg handle in the PDU callback. 465 */ 466 status = iser_iscsihdr_handle(chan, msg); 467 if (status != DDI_SUCCESS) { 468 /* 469 * We are unable to handle this message, and 470 * have no way to recover from this. Fail the 471 * transport. 472 */ 473 ISER_LOG(CE_NOTE, "iser_msg_handle: failed " 474 "iser_iscsihdr_handle"); 475 iser_msg_free(msg); 476 idm_conn_event(iser_conn->ic_idmc, 477 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 478 } 479 } else if (opcode == ISER_OPCODE_HELLO_MSG) { /* at the target */ 480 /* 481 * We are currently not supporting Hello Exchange, 482 * since OFED iSER does not. May be revisited. 483 */ 484 ASSERT(opcode != ISER_OPCODE_HELLO_MSG); 485 486 if (iser_conn->ic_type != ISER_CONN_TYPE_TGT) { 487 idm_conn_event(iser_conn->ic_idmc, 488 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 489 } 490 491 iser_hello_hdr_t *hello_hdr = (iser_hello_hdr_t *)hdr; 492 493 ISER_LOG(CE_NOTE, "received Hello message: opcode[%d], " 494 "maxver[%d], minver[%d], iser_ird[%d], msg (0x%p)", 495 hello_hdr->opcode, hello_hdr->maxver, hello_hdr->minver, 496 ntohs(hello_hdr->iser_ird), (void *)msg); 497 498 mutex_enter(&iser_conn->ic_lock); 499 500 if (iser_conn->ic_stage != ISER_CONN_STAGE_HELLO_WAIT) { 501 /* target is not expected to receive a Hello */ 502 idm_conn_event(iser_conn->ic_idmc, 503 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 504 } 505 506 iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_SENT; 507 mutex_exit(&iser_conn->ic_lock); 508 509 /* Prepare and send a HelloReply message */ 510 status = iser_xfer_helloreply_msg(chan); 511 if (status != ISER_STATUS_SUCCESS) { 512 513 mutex_enter(&iser_conn->ic_lock); 514 iser_conn->ic_stage = 515 ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL; 516 mutex_exit(&iser_conn->ic_lock); 517 518 idm_conn_event(iser_conn->ic_idmc, 519 CE_TRANSPORT_FAIL, status); 520 } 521 522 /* Free this msg handle */ 523 iser_msg_free(msg); 524 525 } else if (opcode == ISER_OPCODE_HELLOREPLY_MSG) { /* at initiator */ 526 527 /* 528 * We are currently not supporting Hello Exchange, 529 * since OFED iSER does not. May be revisited. 530 */ 531 ASSERT(opcode != ISER_OPCODE_HELLOREPLY_MSG); 532 533 if (iser_conn->ic_type != ISER_CONN_TYPE_INI) { 534 idm_conn_event(iser_conn->ic_idmc, 535 CE_TRANSPORT_FAIL, status); 536 } 537 538 iser_helloreply_hdr_t *hello_hdr = (iser_helloreply_hdr_t *)hdr; 539 540 ISER_LOG(CE_NOTE, "received Hello Reply message: opcode[%d], " 541 "maxver[%d], curver[%d], iser_ord[%d], msg (0x%p)", 542 hello_hdr->opcode, hello_hdr->maxver, hello_hdr->curver, 543 ntohs(hello_hdr->iser_ord), (void *)msg); 544 545 /* Free this msg handle */ 546 iser_msg_free(msg); 547 548 /* 549 * Signal the receipt of HelloReply to the waiting thread 550 * so that the initiator can proceed to the Full Feature 551 * Phase. 552 */ 553 mutex_enter(&iser_conn->ic_lock); 554 iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_RCV; 555 cv_signal(&iser_conn->ic_stage_cv); 556 mutex_exit(&iser_conn->ic_lock); 557 } else { 558 /* Protocol error: free the msg handle and fail the session */ 559 ISER_LOG(CE_NOTE, "iser_msg_handle: unsupported opcode (0x%x): " 560 "terminating session on IDM handle (0x%p)", opcode, 561 (void *) iser_conn->ic_idmc); 562 563 iser_msg_free(msg); 564 idm_conn_event(iser_conn->ic_idmc, CE_TRANSPORT_FAIL, 565 IDM_STATUS_FAIL); 566 } 567 } 568 569 #define IDM_PDU_OPCODE(PDU) \ 570 ((PDU)->isp_hdr->opcode & ISCSI_OPCODE_MASK) 571 572 /* network to host translation for 24b integers */ 573 static uint32_t 574 n2h24(uchar_t *ptr) 575 { 576 return ((ptr[0] << 16) | (ptr[1] << 8) | ptr[2]); 577 } 578 579 /* ARGSUSED */ 580 static void 581 iser_rx_pdu_cb(idm_pdu_t *pdu, idm_status_t status) 582 { 583 /* Free the iser msg handle and the PDU handle */ 584 iser_msg_free((iser_msg_t *)pdu->isp_transport_private); 585 idm_pdu_free(pdu); 586 } 587 588 int 589 iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg) 590 { 591 idm_pdu_t *pdu; 592 uint8_t *iser_hdrp; 593 uint8_t *iscsi_hdrp; 594 iscsi_hdr_t *bhs; 595 596 pdu = idm_pdu_alloc_nosleep(sizeof (iscsi_hdr_t), 0); 597 pdu->isp_ic = chan->ic_conn->ic_idmc; 598 ASSERT(pdu->isp_ic != NULL); 599 600 /* Set the iser_msg handle into the transport-private field */ 601 pdu->isp_transport_private = (void *)msg; 602 603 /* Set up a pointer in the pdu handle to the iSER header */ 604 iser_hdrp = (uint8_t *)(uintptr_t)msg->msg_ds.ds_va; 605 if (iser_hdrp == NULL) { 606 ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iser_hdrp is NULL"); 607 idm_pdu_free(pdu); 608 return (ISER_STATUS_FAIL); 609 } 610 pdu->isp_transport_hdr = (void *)iser_hdrp; 611 pdu->isp_transport_hdrlen = ISER_HEADER_LENGTH; 612 613 /* 614 * Set up a pointer to the iSCSI header, which is directly 615 * after the iSER header in the message. 616 */ 617 iscsi_hdrp = ((uint8_t *)(uintptr_t)msg->msg_ds.ds_va) + 618 ISER_HEADER_LENGTH; 619 if (iscsi_hdrp == NULL) { 620 ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iscsi_hdrp is NULL"); 621 idm_pdu_free(pdu); 622 return (ISER_STATUS_FAIL); 623 } 624 pdu->isp_hdr = (iscsi_hdr_t *)(uintptr_t)iscsi_hdrp; 625 626 /* Fill in the BHS */ 627 bhs = pdu->isp_hdr; 628 pdu->isp_hdrlen = sizeof (iscsi_hdr_t) + 629 (bhs->hlength * sizeof (uint32_t)); 630 pdu->isp_datalen = n2h24(bhs->dlength); 631 pdu->isp_callback = iser_rx_pdu_cb; 632 633 /* 634 * If datalen > 0, then non-scsi data may be present. Allocate 635 * space in the PDU handle and set a pointer to the data. 636 */ 637 if (pdu->isp_datalen) { 638 pdu->isp_data = ((uint8_t *)(uintptr_t)pdu->isp_hdr) + 639 pdu->isp_hdrlen; 640 } 641 642 /* Process RX PDU */ 643 idm_pdu_rx(pdu->isp_ic, pdu); 644 645 return (DDI_SUCCESS); 646 } 647