130e7468fSPeter Dunlap /*
230e7468fSPeter Dunlap * CDDL HEADER START
330e7468fSPeter Dunlap *
430e7468fSPeter Dunlap * The contents of this file are subject to the terms of the
530e7468fSPeter Dunlap * Common Development and Distribution License (the "License").
630e7468fSPeter Dunlap * You may not use this file except in compliance with the License.
730e7468fSPeter Dunlap *
830e7468fSPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
930e7468fSPeter Dunlap * or http://www.opensolaris.org/os/licensing.
1030e7468fSPeter Dunlap * See the License for the specific language governing permissions
1130e7468fSPeter Dunlap * and limitations under the License.
1230e7468fSPeter Dunlap *
1330e7468fSPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each
1430e7468fSPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1530e7468fSPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the
1630e7468fSPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying
1730e7468fSPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner]
1830e7468fSPeter Dunlap *
1930e7468fSPeter Dunlap * CDDL HEADER END
2030e7468fSPeter Dunlap */
2130e7468fSPeter Dunlap /*
2230e7468fSPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
2330e7468fSPeter Dunlap * Use is subject to license terms.
24*48bbca81SDaniel Hoffman * Copyright (c) 2016 by Delphix. All rights reserved.
2530e7468fSPeter Dunlap */
2630e7468fSPeter Dunlap
2730e7468fSPeter Dunlap #include <sys/types.h>
2830e7468fSPeter Dunlap #include <sys/ddi.h>
2930e7468fSPeter Dunlap #include <sys/types.h>
3030e7468fSPeter Dunlap #include <sys/socket.h>
3130e7468fSPeter Dunlap #include <netinet/in.h>
3230e7468fSPeter Dunlap #include <sys/sunddi.h>
3330e7468fSPeter Dunlap #include <sys/sdt.h>
3430e7468fSPeter Dunlap #include <sys/ib/ibtl/ibti.h>
3530e7468fSPeter Dunlap #include <sys/ib/ibtl/ibtl_types.h>
3630e7468fSPeter Dunlap
3730e7468fSPeter Dunlap #include <sys/ib/clients/iser/iser.h>
3830e7468fSPeter Dunlap
3930e7468fSPeter Dunlap /*
4030e7468fSPeter Dunlap * iser_cq.c
4130e7468fSPeter Dunlap * Routines for completion queue handlers for iSER.
4230e7468fSPeter Dunlap */
4330e7468fSPeter Dunlap static void iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg);
4430e7468fSPeter Dunlap int iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg);
4530e7468fSPeter Dunlap static int iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl,
4630e7468fSPeter Dunlap iser_chan_t *iser_chan);
4730e7468fSPeter Dunlap static int iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl,
4830e7468fSPeter Dunlap iser_chan_t *iser_chan);
4930e7468fSPeter Dunlap
5030e7468fSPeter Dunlap void
iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl,void * arg)5130e7468fSPeter Dunlap iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
5230e7468fSPeter Dunlap {
5330e7468fSPeter Dunlap iser_chan_t *iser_chan;
5430e7468fSPeter Dunlap ibt_status_t status;
5530e7468fSPeter Dunlap
5630e7468fSPeter Dunlap iser_chan = (iser_chan_t *)arg;
5730e7468fSPeter Dunlap
58a82ec3cfSPriya Krishnan /*
59a82ec3cfSPriya Krishnan * Poll for work request completion while successful. If the
60a82ec3cfSPriya Krishnan * queue empties or otherwise becomes invalid, stop polling.
61a82ec3cfSPriya Krishnan */
6230e7468fSPeter Dunlap do {
6330e7468fSPeter Dunlap status = iser_ib_poll_send_completions(cq_hdl, iser_chan);
64a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS);
6530e7468fSPeter Dunlap
66a82ec3cfSPriya Krishnan if (status == IBT_CQ_EMPTY) {
6730e7468fSPeter Dunlap /* We've emptied the CQ, rearm it before we're done here */
6830e7468fSPeter Dunlap status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
6930e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
7030e7468fSPeter Dunlap /* Unexpected error */
7130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: "
7230e7468fSPeter Dunlap "ibt_enable_cq_notify error (%d)", status);
7330e7468fSPeter Dunlap return;
7430e7468fSPeter Dunlap }
7530e7468fSPeter Dunlap
7630e7468fSPeter Dunlap /* Now, check for more completions after the rearm */
7730e7468fSPeter Dunlap do {
78a82ec3cfSPriya Krishnan status = iser_ib_poll_send_completions(
79a82ec3cfSPriya Krishnan cq_hdl, iser_chan);
80a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS);
81a82ec3cfSPriya Krishnan }
8230e7468fSPeter Dunlap }
8330e7468fSPeter Dunlap
8430e7468fSPeter Dunlap static int
iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl,iser_chan_t * iser_chan)8530e7468fSPeter Dunlap iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
8630e7468fSPeter Dunlap {
8730e7468fSPeter Dunlap ibt_wc_t wc[ISER_IB_SCQ_POLL_MAX];
8830e7468fSPeter Dunlap ibt_wrid_t wrid;
8930e7468fSPeter Dunlap idm_buf_t *idb = NULL;
9030e7468fSPeter Dunlap idm_task_t *idt = NULL;
9130e7468fSPeter Dunlap iser_wr_t *wr = NULL;
9230e7468fSPeter Dunlap int i;
9330e7468fSPeter Dunlap uint_t npoll = 0;
9430e7468fSPeter Dunlap ibt_status_t status;
9530e7468fSPeter Dunlap iser_conn_t *iser_conn;
9630e7468fSPeter Dunlap idm_status_t idm_status;
97a668b114SPriya Krishnan iser_mr_t *mr;
9830e7468fSPeter Dunlap
9930e7468fSPeter Dunlap iser_conn = iser_chan->ic_conn;
10030e7468fSPeter Dunlap
101a82ec3cfSPriya Krishnan /* Poll ISER_IB_SCQ_POLL_MAX completions from the CQ */
10230e7468fSPeter Dunlap status = ibt_poll_cq(cq_hdl, wc, ISER_IB_SCQ_POLL_MAX, &npoll);
10330e7468fSPeter Dunlap
10430e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
10530e7468fSPeter Dunlap if (status != IBT_CQ_EMPTY) {
10630e7468fSPeter Dunlap /* Unexpected error */
10730e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: ibt_poll_cq "
108a82ec3cfSPriya Krishnan "unexpected error (%d)", status);
10930e7468fSPeter Dunlap }
11030e7468fSPeter Dunlap /* CQ is empty. Either way, move along... */
11130e7468fSPeter Dunlap return (status);
11230e7468fSPeter Dunlap }
11330e7468fSPeter Dunlap
11430e7468fSPeter Dunlap /*
11530e7468fSPeter Dunlap * Handle each of the completions we've polled
11630e7468fSPeter Dunlap */
11730e7468fSPeter Dunlap for (i = 0; i < npoll; i++) {
11830e7468fSPeter Dunlap
11930e7468fSPeter Dunlap DTRACE_PROBE3(iser__send__cqe, iser_chan_t *, iser_chan,
12030e7468fSPeter Dunlap ibt_wc_t *, &wc[i], ibt_wc_status_t, wc[i].wc_status);
12130e7468fSPeter Dunlap
12230e7468fSPeter Dunlap /* Grab the wrid of the completion */
12330e7468fSPeter Dunlap wrid = wc[i].wc_id;
12430e7468fSPeter Dunlap
12530e7468fSPeter Dunlap /* Decrement this channel's SQ posted count */
12630e7468fSPeter Dunlap mutex_enter(&iser_chan->ic_sq_post_lock);
12730e7468fSPeter Dunlap iser_chan->ic_sq_post_count--;
12830e7468fSPeter Dunlap mutex_exit(&iser_chan->ic_sq_post_lock);
12930e7468fSPeter Dunlap
13030e7468fSPeter Dunlap /* Pull in the wr handle */
13130e7468fSPeter Dunlap wr = (iser_wr_t *)(uintptr_t)wrid;
13230e7468fSPeter Dunlap ASSERT(wr != NULL);
13330e7468fSPeter Dunlap
13430e7468fSPeter Dunlap /* Set an idm_status for return to IDM */
13530e7468fSPeter Dunlap idm_status = (wc[i].wc_status == IBT_WC_SUCCESS) ?
13630e7468fSPeter Dunlap IDM_STATUS_SUCCESS : IDM_STATUS_FAIL;
13730e7468fSPeter Dunlap
13830e7468fSPeter Dunlap /*
13930e7468fSPeter Dunlap * A non-success status here indicates the QP went
14030e7468fSPeter Dunlap * into an error state while this WR was being
14130e7468fSPeter Dunlap * processed. This can also happen when the
14230e7468fSPeter Dunlap * channel is closed on the remote end. Clean up
14330e7468fSPeter Dunlap * the resources, then push CE_TRANSPORT_FAIL
14430e7468fSPeter Dunlap * into IDM.
14530e7468fSPeter Dunlap */
14630e7468fSPeter Dunlap if (wc[i].wc_status != IBT_WC_SUCCESS) {
14730e7468fSPeter Dunlap /*
14830e7468fSPeter Dunlap * Free the resources attached to this
14930e7468fSPeter Dunlap * completion.
15030e7468fSPeter Dunlap */
15130e7468fSPeter Dunlap if (wr->iw_msg != NULL) {
15230e7468fSPeter Dunlap /* Free iser_msg handle */
15330e7468fSPeter Dunlap iser_msg_free(wr->iw_msg);
15430e7468fSPeter Dunlap }
15530e7468fSPeter Dunlap
15630e7468fSPeter Dunlap if (wr->iw_pdu != NULL) {
15730e7468fSPeter Dunlap /* Complete the PDU */
15830e7468fSPeter Dunlap idm_pdu_complete(wr->iw_pdu, idm_status);
15930e7468fSPeter Dunlap }
16030e7468fSPeter Dunlap
16130e7468fSPeter Dunlap if (wr->iw_buf != NULL) {
16230e7468fSPeter Dunlap /* Invoke buffer callback */
16330e7468fSPeter Dunlap idb = wr->iw_buf;
164a668b114SPriya Krishnan mr = ((iser_buf_t *)
165a668b114SPriya Krishnan idb->idb_buf_private)->iser_mr;
16630e7468fSPeter Dunlap #ifdef DEBUG
16730e7468fSPeter Dunlap bcopy(&wc[i],
16830e7468fSPeter Dunlap &((iser_buf_t *)idb->idb_buf_private)->
16930e7468fSPeter Dunlap buf_wc, sizeof (ibt_wc_t));
17030e7468fSPeter Dunlap #endif
17130e7468fSPeter Dunlap idt = idb->idb_task_binding;
17230e7468fSPeter Dunlap mutex_enter(&idt->idt_mutex);
17330e7468fSPeter Dunlap if (wr->iw_type == ISER_WR_RDMAW) {
174a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done,
175a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic,
176a668b114SPriya Krishnan uintptr_t, idb->idb_buf,
177a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset,
178a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0,
179a668b114SPriya Krishnan uint32_t, mr->is_mrrkey,
180a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len,
181a668b114SPriya Krishnan int, XFER_BUF_TX_TO_INI);
18230e7468fSPeter Dunlap idm_buf_tx_to_ini_done(idt, idb,
18330e7468fSPeter Dunlap IDM_STATUS_FAIL);
18430e7468fSPeter Dunlap } else { /* ISER_WR_RDMAR */
185a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done,
186a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic,
187a668b114SPriya Krishnan uintptr_t, idb->idb_buf,
188a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset,
189a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0,
190a668b114SPriya Krishnan uint32_t, mr->is_mrrkey,
191a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len,
192a668b114SPriya Krishnan int, XFER_BUF_RX_FROM_INI);
19330e7468fSPeter Dunlap idm_buf_rx_from_ini_done(idt, idb,
19430e7468fSPeter Dunlap IDM_STATUS_FAIL);
19530e7468fSPeter Dunlap }
19630e7468fSPeter Dunlap }
19730e7468fSPeter Dunlap
19830e7468fSPeter Dunlap /* Free the iser wr handle */
19930e7468fSPeter Dunlap iser_wr_free(wr);
20030e7468fSPeter Dunlap
20130e7468fSPeter Dunlap /*
20230e7468fSPeter Dunlap * Tell IDM that the channel has gone down,
203*48bbca81SDaniel Hoffman * unless it already knows.
20430e7468fSPeter Dunlap */
20530e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock);
20630e7468fSPeter Dunlap switch (iser_conn->ic_stage) {
20730e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_DISCONNECTED:
20830e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_FREED:
20930e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSING:
21030e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSED:
21130e7468fSPeter Dunlap break;
21230e7468fSPeter Dunlap
21330e7468fSPeter Dunlap default:
21430e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc,
21530e7468fSPeter Dunlap CE_TRANSPORT_FAIL, idm_status);
21630e7468fSPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_CLOSING;
21730e7468fSPeter Dunlap }
21830e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock);
21930e7468fSPeter Dunlap
22030e7468fSPeter Dunlap /* Move onto the next completion */
22130e7468fSPeter Dunlap continue;
22230e7468fSPeter Dunlap }
22330e7468fSPeter Dunlap
22430e7468fSPeter Dunlap /*
22530e7468fSPeter Dunlap * For a success status, just invoke the PDU or
22630e7468fSPeter Dunlap * buffer completion. We use our WR handle's
22730e7468fSPeter Dunlap * "iw_type" here so that we can properly process
22830e7468fSPeter Dunlap * because the CQE's opcode is invalid if the status
22930e7468fSPeter Dunlap * is failed.
23030e7468fSPeter Dunlap */
23130e7468fSPeter Dunlap switch (wr->iw_type) {
23230e7468fSPeter Dunlap case ISER_WR_SEND:
23330e7468fSPeter Dunlap /* Free the msg handle */
23430e7468fSPeter Dunlap ASSERT(wr->iw_msg != NULL);
23530e7468fSPeter Dunlap iser_msg_free(wr->iw_msg);
23630e7468fSPeter Dunlap
23730e7468fSPeter Dunlap if (wr->iw_pdu == NULL) {
23830e7468fSPeter Dunlap /* This is a hello exchange message */
23930e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock);
24030e7468fSPeter Dunlap if (iser_conn->ic_stage ==
24130e7468fSPeter Dunlap ISER_CONN_STAGE_HELLOREPLY_SENT) {
24230e7468fSPeter Dunlap /*
24330e7468fSPeter Dunlap * We're on the target side,
24430e7468fSPeter Dunlap * and have just successfully
24530e7468fSPeter Dunlap * sent the HelloReply msg.
24630e7468fSPeter Dunlap */
24730e7468fSPeter Dunlap iser_conn->ic_stage =
24830e7468fSPeter Dunlap ISER_CONN_STAGE_LOGGED_IN;
24930e7468fSPeter Dunlap }
25030e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock);
25130e7468fSPeter Dunlap } else {
25230e7468fSPeter Dunlap /* This is a normal control message */
25330e7468fSPeter Dunlap idm_pdu_complete(wr->iw_pdu, idm_status);
25430e7468fSPeter Dunlap }
25530e7468fSPeter Dunlap
25630e7468fSPeter Dunlap /* Free the wr handle */
25730e7468fSPeter Dunlap iser_wr_free(wr);
25830e7468fSPeter Dunlap
25930e7468fSPeter Dunlap break;
26030e7468fSPeter Dunlap
26130e7468fSPeter Dunlap case ISER_WR_RDMAW:
26230e7468fSPeter Dunlap case ISER_WR_RDMAR:
26330e7468fSPeter Dunlap /*
26430e7468fSPeter Dunlap * Invoke the appropriate callback;
26530e7468fSPeter Dunlap * the buffer will be freed there.
26630e7468fSPeter Dunlap */
26730e7468fSPeter Dunlap idb = wr->iw_buf;
268a668b114SPriya Krishnan mr = ((iser_buf_t *)idb->idb_buf_private)->iser_mr;
26930e7468fSPeter Dunlap #ifdef DEBUG
27030e7468fSPeter Dunlap bcopy(&wc[i],
27130e7468fSPeter Dunlap &((iser_buf_t *)idb->idb_buf_private)->buf_wc,
27230e7468fSPeter Dunlap sizeof (ibt_wc_t));
27330e7468fSPeter Dunlap #endif
27430e7468fSPeter Dunlap idt = idb->idb_task_binding;
27530e7468fSPeter Dunlap
27630e7468fSPeter Dunlap mutex_enter(&idt->idt_mutex);
27730e7468fSPeter Dunlap if (wr->iw_type == ISER_WR_RDMAW) {
278a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done,
279a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic,
280a668b114SPriya Krishnan uintptr_t, idb->idb_buf,
281a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset,
282a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0,
283a668b114SPriya Krishnan uint32_t, mr->is_mrrkey,
284a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len,
285a668b114SPriya Krishnan int, XFER_BUF_TX_TO_INI);
28630e7468fSPeter Dunlap idm_buf_tx_to_ini_done(idt, idb, idm_status);
28730e7468fSPeter Dunlap } else {
288a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done,
289a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic,
290a668b114SPriya Krishnan uintptr_t, idb->idb_buf,
291a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset,
292a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0,
293a668b114SPriya Krishnan uint32_t, mr->is_mrrkey,
294a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len,
295a668b114SPriya Krishnan int, XFER_BUF_RX_FROM_INI);
29630e7468fSPeter Dunlap idm_buf_rx_from_ini_done(idt, idb, idm_status);
29730e7468fSPeter Dunlap }
29830e7468fSPeter Dunlap
29930e7468fSPeter Dunlap /* Free the wr handle */
30030e7468fSPeter Dunlap iser_wr_free(wr);
30130e7468fSPeter Dunlap
30230e7468fSPeter Dunlap break;
30330e7468fSPeter Dunlap
30430e7468fSPeter Dunlap default:
30530e7468fSPeter Dunlap ASSERT(0);
30630e7468fSPeter Dunlap break;
30730e7468fSPeter Dunlap }
30830e7468fSPeter Dunlap }
30930e7468fSPeter Dunlap
31030e7468fSPeter Dunlap return (status);
31130e7468fSPeter Dunlap }
31230e7468fSPeter Dunlap
31330e7468fSPeter Dunlap void
iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl,void * arg)31430e7468fSPeter Dunlap iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
31530e7468fSPeter Dunlap {
31630e7468fSPeter Dunlap iser_chan_t *iser_chan;
31730e7468fSPeter Dunlap ibt_status_t status;
31830e7468fSPeter Dunlap
31930e7468fSPeter Dunlap iser_chan = (iser_chan_t *)arg;
32030e7468fSPeter Dunlap
321a82ec3cfSPriya Krishnan /*
322a82ec3cfSPriya Krishnan * Poll for work request completion while successful. If the
323a82ec3cfSPriya Krishnan * queue empties or otherwise becomes invalid, stop polling.
324a82ec3cfSPriya Krishnan */
32530e7468fSPeter Dunlap do {
32630e7468fSPeter Dunlap status = iser_ib_poll_recv_completions(cq_hdl, iser_chan);
327a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS);
32830e7468fSPeter Dunlap
329a82ec3cfSPriya Krishnan if (status == IBT_CQ_EMPTY) {
33030e7468fSPeter Dunlap /* We've emptied the CQ, rearm it before we're done here */
33130e7468fSPeter Dunlap status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
33230e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
33330e7468fSPeter Dunlap /* Unexpected error */
33430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_recvcq_handler: "
33530e7468fSPeter Dunlap "ibt_enable_cq_notify error (%d)", status);
33630e7468fSPeter Dunlap return;
33730e7468fSPeter Dunlap }
33830e7468fSPeter Dunlap
33930e7468fSPeter Dunlap /* Now, check for more completions after the rearm */
34030e7468fSPeter Dunlap do {
341a82ec3cfSPriya Krishnan status = iser_ib_poll_recv_completions(
342a82ec3cfSPriya Krishnan cq_hdl, iser_chan);
343a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS);
344a82ec3cfSPriya Krishnan }
34530e7468fSPeter Dunlap }
34630e7468fSPeter Dunlap
34730e7468fSPeter Dunlap static int
iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl,iser_chan_t * iser_chan)34830e7468fSPeter Dunlap iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
34930e7468fSPeter Dunlap {
35030e7468fSPeter Dunlap ibt_wc_t wc;
35130e7468fSPeter Dunlap iser_msg_t *msg;
35230e7468fSPeter Dunlap iser_qp_t *iser_qp;
35330e7468fSPeter Dunlap int status;
35430e7468fSPeter Dunlap
35530e7468fSPeter Dunlap iser_qp = &(iser_chan->ic_qp);
35630e7468fSPeter Dunlap
35730e7468fSPeter Dunlap bzero(&wc, sizeof (ibt_wc_t));
35830e7468fSPeter Dunlap status = ibt_poll_cq(cq_hdl, &wc, 1, NULL);
35930e7468fSPeter Dunlap if (status == IBT_CQ_EMPTY) {
36030e7468fSPeter Dunlap /* CQ is empty, return */
36130e7468fSPeter Dunlap return (status);
36230e7468fSPeter Dunlap }
36330e7468fSPeter Dunlap
36430e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
36530e7468fSPeter Dunlap /* Unexpected error */
36630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
36730e7468fSPeter Dunlap "ibt_poll_cq error (%d)", status);
36830e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
36930e7468fSPeter Dunlap iser_qp->rq_level--;
37030e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
37130e7468fSPeter Dunlap /* Free the msg handle (if we got it back) */
37230e7468fSPeter Dunlap if ((msg = (iser_msg_t *)(uintptr_t)wc.wc_id) != NULL) {
37330e7468fSPeter Dunlap iser_msg_free(msg);
37430e7468fSPeter Dunlap }
37530e7468fSPeter Dunlap return (status);
37630e7468fSPeter Dunlap }
37730e7468fSPeter Dunlap
37830e7468fSPeter Dunlap /* Retrieve the iSER msg handle */
37930e7468fSPeter Dunlap msg = (iser_msg_t *)(uintptr_t)wc.wc_id;
38030e7468fSPeter Dunlap ASSERT(msg != NULL);
38130e7468fSPeter Dunlap
38230e7468fSPeter Dunlap /*
38330e7468fSPeter Dunlap * Decrement the posted level in the RQ, then check
38430e7468fSPeter Dunlap * to see if we need to fill the RQ back up (or if
38530e7468fSPeter Dunlap * we are already on the taskq).
38630e7468fSPeter Dunlap */
387cf8c0ebaSPeter Dunlap mutex_enter(&iser_chan->ic_conn->ic_lock);
38830e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
38930e7468fSPeter Dunlap iser_qp->rq_level--;
39030e7468fSPeter Dunlap
39130e7468fSPeter Dunlap if ((iser_qp->rq_taskqpending == B_FALSE) &&
39272cf3143Speter dunlap (iser_qp->rq_level <= iser_qp->rq_lwm) &&
39372cf3143Speter dunlap (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
39472cf3143Speter dunlap (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) {
39530e7468fSPeter Dunlap /* Set the pending flag and fire off a post_recv */
39630e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_TRUE;
39730e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
39830e7468fSPeter Dunlap
399cf8c0ebaSPeter Dunlap status = iser_ib_post_recv_async(iser_chan->ic_chanhdl);
40030e7468fSPeter Dunlap
40130e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
40230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
40330e7468fSPeter Dunlap "task dispatch failed");
40430e7468fSPeter Dunlap /* Failure to launch, unset the pending flag */
40530e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
40630e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_FALSE;
40730e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
40830e7468fSPeter Dunlap }
40930e7468fSPeter Dunlap } else {
41030e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
41130e7468fSPeter Dunlap }
41230e7468fSPeter Dunlap
41330e7468fSPeter Dunlap DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
41430e7468fSPeter Dunlap ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
41530e7468fSPeter Dunlap if (wc.wc_status != IBT_WC_SUCCESS) {
41630e7468fSPeter Dunlap /*
41730e7468fSPeter Dunlap * Tell IDM that the channel has gone down,
418*48bbca81SDaniel Hoffman * unless it already knows.
41930e7468fSPeter Dunlap */
42030e7468fSPeter Dunlap switch (iser_chan->ic_conn->ic_stage) {
42130e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_DISCONNECTED:
42230e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_FREED:
42330e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSING:
42430e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSED:
42530e7468fSPeter Dunlap break;
42630e7468fSPeter Dunlap
42730e7468fSPeter Dunlap default:
42830e7468fSPeter Dunlap idm_conn_event(iser_chan->ic_conn->ic_idmc,
42930e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
43030e7468fSPeter Dunlap iser_chan->ic_conn->ic_stage =
43130e7468fSPeter Dunlap ISER_CONN_STAGE_CLOSING;
43230e7468fSPeter Dunlap }
43330e7468fSPeter Dunlap mutex_exit(&iser_chan->ic_conn->ic_lock);
43430e7468fSPeter Dunlap
43530e7468fSPeter Dunlap iser_msg_free(msg);
43630e7468fSPeter Dunlap return (DDI_SUCCESS);
43730e7468fSPeter Dunlap } else {
43872cf3143Speter dunlap mutex_exit(&iser_chan->ic_conn->ic_lock);
43972cf3143Speter dunlap
44030e7468fSPeter Dunlap /*
44130e7468fSPeter Dunlap * We have an iSER message in, let's handle it.
44230e7468fSPeter Dunlap * We will free the iser_msg_t later in this path,
44330e7468fSPeter Dunlap * depending upon the action required.
44430e7468fSPeter Dunlap */
44530e7468fSPeter Dunlap iser_msg_handle(iser_chan, msg);
44630e7468fSPeter Dunlap return (DDI_SUCCESS);
44730e7468fSPeter Dunlap }
44830e7468fSPeter Dunlap }
44930e7468fSPeter Dunlap
45030e7468fSPeter Dunlap static void
iser_msg_handle(iser_chan_t * chan,iser_msg_t * msg)45130e7468fSPeter Dunlap iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg)
45230e7468fSPeter Dunlap {
45330e7468fSPeter Dunlap int opcode;
45430e7468fSPeter Dunlap iser_ctrl_hdr_t *hdr = NULL;
45530e7468fSPeter Dunlap iser_conn_t *iser_conn = chan->ic_conn;
45630e7468fSPeter Dunlap int status;
45730e7468fSPeter Dunlap
45830e7468fSPeter Dunlap hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
45930e7468fSPeter Dunlap ASSERT(hdr != NULL);
46030e7468fSPeter Dunlap
46130e7468fSPeter Dunlap opcode = hdr->opcode;
46230e7468fSPeter Dunlap if (opcode == ISER_OPCODE_CTRL_TYPE_PDU) {
46330e7468fSPeter Dunlap /*
46430e7468fSPeter Dunlap * Handle an iSCSI Control PDU iSER message.
46530e7468fSPeter Dunlap * Note we'll free the msg handle in the PDU callback.
46630e7468fSPeter Dunlap */
46730e7468fSPeter Dunlap status = iser_iscsihdr_handle(chan, msg);
46830e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
46930e7468fSPeter Dunlap /*
47030e7468fSPeter Dunlap * We are unable to handle this message, and
47130e7468fSPeter Dunlap * have no way to recover from this. Fail the
47230e7468fSPeter Dunlap * transport.
47330e7468fSPeter Dunlap */
47430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_handle: failed "
47530e7468fSPeter Dunlap "iser_iscsihdr_handle");
47630e7468fSPeter Dunlap iser_msg_free(msg);
47730e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc,
47830e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
47930e7468fSPeter Dunlap }
48030e7468fSPeter Dunlap } else if (opcode == ISER_OPCODE_HELLO_MSG) { /* at the target */
48130e7468fSPeter Dunlap /*
48230e7468fSPeter Dunlap * We are currently not supporting Hello Exchange,
48330e7468fSPeter Dunlap * since OFED iSER does not. May be revisited.
48430e7468fSPeter Dunlap */
48530e7468fSPeter Dunlap ASSERT(opcode != ISER_OPCODE_HELLO_MSG);
48630e7468fSPeter Dunlap
48730e7468fSPeter Dunlap if (iser_conn->ic_type != ISER_CONN_TYPE_TGT) {
48830e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc,
48930e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
49030e7468fSPeter Dunlap }
49130e7468fSPeter Dunlap
49230e7468fSPeter Dunlap iser_hello_hdr_t *hello_hdr = (iser_hello_hdr_t *)hdr;
49330e7468fSPeter Dunlap
49430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "received Hello message: opcode[%d], "
49530e7468fSPeter Dunlap "maxver[%d], minver[%d], iser_ird[%d], msg (0x%p)",
49630e7468fSPeter Dunlap hello_hdr->opcode, hello_hdr->maxver, hello_hdr->minver,
49730e7468fSPeter Dunlap ntohs(hello_hdr->iser_ird), (void *)msg);
49830e7468fSPeter Dunlap
49930e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock);
50030e7468fSPeter Dunlap
50130e7468fSPeter Dunlap if (iser_conn->ic_stage != ISER_CONN_STAGE_HELLO_WAIT) {
50230e7468fSPeter Dunlap /* target is not expected to receive a Hello */
50330e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc,
50430e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
50530e7468fSPeter Dunlap }
50630e7468fSPeter Dunlap
50730e7468fSPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_SENT;
50830e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock);
50930e7468fSPeter Dunlap
51030e7468fSPeter Dunlap /* Prepare and send a HelloReply message */
51130e7468fSPeter Dunlap status = iser_xfer_helloreply_msg(chan);
51230e7468fSPeter Dunlap if (status != ISER_STATUS_SUCCESS) {
51330e7468fSPeter Dunlap
51430e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock);
51530e7468fSPeter Dunlap iser_conn->ic_stage =
51630e7468fSPeter Dunlap ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL;
51730e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock);
51830e7468fSPeter Dunlap
51930e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc,
52030e7468fSPeter Dunlap CE_TRANSPORT_FAIL, status);
52130e7468fSPeter Dunlap }
52230e7468fSPeter Dunlap
52330e7468fSPeter Dunlap /* Free this msg handle */
52430e7468fSPeter Dunlap iser_msg_free(msg);
52530e7468fSPeter Dunlap
52630e7468fSPeter Dunlap } else if (opcode == ISER_OPCODE_HELLOREPLY_MSG) { /* at initiator */
52730e7468fSPeter Dunlap
52830e7468fSPeter Dunlap /*
52930e7468fSPeter Dunlap * We are currently not supporting Hello Exchange,
53030e7468fSPeter Dunlap * since OFED iSER does not. May be revisited.
53130e7468fSPeter Dunlap */
53230e7468fSPeter Dunlap ASSERT(opcode != ISER_OPCODE_HELLOREPLY_MSG);
53330e7468fSPeter Dunlap
53430e7468fSPeter Dunlap if (iser_conn->ic_type != ISER_CONN_TYPE_INI) {
53530e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc,
53630e7468fSPeter Dunlap CE_TRANSPORT_FAIL, status);
53730e7468fSPeter Dunlap }
53830e7468fSPeter Dunlap
53930e7468fSPeter Dunlap iser_helloreply_hdr_t *hello_hdr = (iser_helloreply_hdr_t *)hdr;
54030e7468fSPeter Dunlap
54130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "received Hello Reply message: opcode[%d], "
54230e7468fSPeter Dunlap "maxver[%d], curver[%d], iser_ord[%d], msg (0x%p)",
54330e7468fSPeter Dunlap hello_hdr->opcode, hello_hdr->maxver, hello_hdr->curver,
54430e7468fSPeter Dunlap ntohs(hello_hdr->iser_ord), (void *)msg);
54530e7468fSPeter Dunlap
54630e7468fSPeter Dunlap /* Free this msg handle */
54730e7468fSPeter Dunlap iser_msg_free(msg);
54830e7468fSPeter Dunlap
54930e7468fSPeter Dunlap /*
55030e7468fSPeter Dunlap * Signal the receipt of HelloReply to the waiting thread
55130e7468fSPeter Dunlap * so that the initiator can proceed to the Full Feature
55230e7468fSPeter Dunlap * Phase.
55330e7468fSPeter Dunlap */
55430e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock);
55530e7468fSPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_RCV;
55630e7468fSPeter Dunlap cv_signal(&iser_conn->ic_stage_cv);
55730e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock);
55830e7468fSPeter Dunlap } else {
55930e7468fSPeter Dunlap /* Protocol error: free the msg handle and fail the session */
56030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_handle: unsupported opcode (0x%x): "
56130e7468fSPeter Dunlap "terminating session on IDM handle (0x%p)", opcode,
56230e7468fSPeter Dunlap (void *) iser_conn->ic_idmc);
56330e7468fSPeter Dunlap
56430e7468fSPeter Dunlap iser_msg_free(msg);
56530e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, CE_TRANSPORT_FAIL,
56630e7468fSPeter Dunlap IDM_STATUS_FAIL);
56730e7468fSPeter Dunlap }
56830e7468fSPeter Dunlap }
56930e7468fSPeter Dunlap
57030e7468fSPeter Dunlap #define IDM_PDU_OPCODE(PDU) \
57130e7468fSPeter Dunlap ((PDU)->isp_hdr->opcode & ISCSI_OPCODE_MASK)
57230e7468fSPeter Dunlap
57330e7468fSPeter Dunlap /* network to host translation for 24b integers */
57430e7468fSPeter Dunlap static uint32_t
n2h24(uchar_t * ptr)57530e7468fSPeter Dunlap n2h24(uchar_t *ptr)
57630e7468fSPeter Dunlap {
57730e7468fSPeter Dunlap return ((ptr[0] << 16) | (ptr[1] << 8) | ptr[2]);
57830e7468fSPeter Dunlap }
57930e7468fSPeter Dunlap
58030e7468fSPeter Dunlap /* ARGSUSED */
58130e7468fSPeter Dunlap static void
iser_rx_pdu_cb(idm_pdu_t * pdu,idm_status_t status)58230e7468fSPeter Dunlap iser_rx_pdu_cb(idm_pdu_t *pdu, idm_status_t status)
58330e7468fSPeter Dunlap {
58430e7468fSPeter Dunlap /* Free the iser msg handle and the PDU handle */
58530e7468fSPeter Dunlap iser_msg_free((iser_msg_t *)pdu->isp_transport_private);
58630e7468fSPeter Dunlap idm_pdu_free(pdu);
58730e7468fSPeter Dunlap }
58830e7468fSPeter Dunlap
58930e7468fSPeter Dunlap int
iser_iscsihdr_handle(iser_chan_t * chan,iser_msg_t * msg)59030e7468fSPeter Dunlap iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg)
59130e7468fSPeter Dunlap {
59230e7468fSPeter Dunlap idm_pdu_t *pdu;
59330e7468fSPeter Dunlap uint8_t *iser_hdrp;
59430e7468fSPeter Dunlap uint8_t *iscsi_hdrp;
59530e7468fSPeter Dunlap iscsi_hdr_t *bhs;
59630e7468fSPeter Dunlap
59730e7468fSPeter Dunlap pdu = idm_pdu_alloc_nosleep(sizeof (iscsi_hdr_t), 0);
59830e7468fSPeter Dunlap pdu->isp_ic = chan->ic_conn->ic_idmc;
59930e7468fSPeter Dunlap ASSERT(pdu->isp_ic != NULL);
60030e7468fSPeter Dunlap
60130e7468fSPeter Dunlap /* Set the iser_msg handle into the transport-private field */
60230e7468fSPeter Dunlap pdu->isp_transport_private = (void *)msg;
60330e7468fSPeter Dunlap
60430e7468fSPeter Dunlap /* Set up a pointer in the pdu handle to the iSER header */
60530e7468fSPeter Dunlap iser_hdrp = (uint8_t *)(uintptr_t)msg->msg_ds.ds_va;
60630e7468fSPeter Dunlap if (iser_hdrp == NULL) {
60730e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iser_hdrp is NULL");
60830e7468fSPeter Dunlap idm_pdu_free(pdu);
60930e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
61030e7468fSPeter Dunlap }
61130e7468fSPeter Dunlap pdu->isp_transport_hdr = (void *)iser_hdrp;
61230e7468fSPeter Dunlap pdu->isp_transport_hdrlen = ISER_HEADER_LENGTH;
61330e7468fSPeter Dunlap
61430e7468fSPeter Dunlap /*
61530e7468fSPeter Dunlap * Set up a pointer to the iSCSI header, which is directly
61630e7468fSPeter Dunlap * after the iSER header in the message.
61730e7468fSPeter Dunlap */
61830e7468fSPeter Dunlap iscsi_hdrp = ((uint8_t *)(uintptr_t)msg->msg_ds.ds_va) +
61930e7468fSPeter Dunlap ISER_HEADER_LENGTH;
62030e7468fSPeter Dunlap if (iscsi_hdrp == NULL) {
62130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iscsi_hdrp is NULL");
62230e7468fSPeter Dunlap idm_pdu_free(pdu);
62330e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
62430e7468fSPeter Dunlap }
62530e7468fSPeter Dunlap pdu->isp_hdr = (iscsi_hdr_t *)(uintptr_t)iscsi_hdrp;
62630e7468fSPeter Dunlap
62730e7468fSPeter Dunlap /* Fill in the BHS */
62830e7468fSPeter Dunlap bhs = pdu->isp_hdr;
62930e7468fSPeter Dunlap pdu->isp_hdrlen = sizeof (iscsi_hdr_t) +
63030e7468fSPeter Dunlap (bhs->hlength * sizeof (uint32_t));
63130e7468fSPeter Dunlap pdu->isp_datalen = n2h24(bhs->dlength);
63230e7468fSPeter Dunlap pdu->isp_callback = iser_rx_pdu_cb;
63330e7468fSPeter Dunlap
63430e7468fSPeter Dunlap /*
63530e7468fSPeter Dunlap * If datalen > 0, then non-scsi data may be present. Allocate
63630e7468fSPeter Dunlap * space in the PDU handle and set a pointer to the data.
63730e7468fSPeter Dunlap */
63830e7468fSPeter Dunlap if (pdu->isp_datalen) {
63930e7468fSPeter Dunlap pdu->isp_data = ((uint8_t *)(uintptr_t)pdu->isp_hdr) +
64030e7468fSPeter Dunlap pdu->isp_hdrlen;
64130e7468fSPeter Dunlap }
64230e7468fSPeter Dunlap
64330e7468fSPeter Dunlap /* Process RX PDU */
64430e7468fSPeter Dunlap idm_pdu_rx(pdu->isp_ic, pdu);
64530e7468fSPeter Dunlap
64630e7468fSPeter Dunlap return (DDI_SUCCESS);
64730e7468fSPeter Dunlap }
648