130e7468fSPeter Dunlap /*
230e7468fSPeter Dunlap * CDDL HEADER START
330e7468fSPeter Dunlap *
430e7468fSPeter Dunlap * The contents of this file are subject to the terms of the
530e7468fSPeter Dunlap * Common Development and Distribution License (the "License").
630e7468fSPeter Dunlap * You may not use this file except in compliance with the License.
730e7468fSPeter Dunlap *
830e7468fSPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
930e7468fSPeter Dunlap * or http://www.opensolaris.org/os/licensing.
1030e7468fSPeter Dunlap * See the License for the specific language governing permissions
1130e7468fSPeter Dunlap * and limitations under the License.
1230e7468fSPeter Dunlap *
1330e7468fSPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each
1430e7468fSPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1530e7468fSPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the
1630e7468fSPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying
1730e7468fSPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner]
1830e7468fSPeter Dunlap *
1930e7468fSPeter Dunlap * CDDL HEADER END
2030e7468fSPeter Dunlap */
2130e7468fSPeter Dunlap /*
2230e7468fSPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
2330e7468fSPeter Dunlap * Use is subject to license terms.
2430e7468fSPeter Dunlap */
2530e7468fSPeter Dunlap
2630e7468fSPeter Dunlap #include <sys/types.h>
2730e7468fSPeter Dunlap #include <sys/stat.h>
2830e7468fSPeter Dunlap #include <sys/conf.h>
2930e7468fSPeter Dunlap #include <sys/ddi.h>
3030e7468fSPeter Dunlap #include <sys/sunddi.h>
3130e7468fSPeter Dunlap #include <sys/modctl.h>
3230e7468fSPeter Dunlap #include <sys/byteorder.h>
33a668b114SPriya Krishnan #include <sys/sdt.h>
3430e7468fSPeter Dunlap
3530e7468fSPeter Dunlap #include <sys/ib/clients/iser/iser.h>
3630e7468fSPeter Dunlap
3730e7468fSPeter Dunlap /*
3830e7468fSPeter Dunlap * iser_xfer.c
3930e7468fSPeter Dunlap */
4030e7468fSPeter Dunlap
4130e7468fSPeter Dunlap int
iser_xfer_hello_msg(iser_chan_t * chan)4230e7468fSPeter Dunlap iser_xfer_hello_msg(iser_chan_t *chan)
4330e7468fSPeter Dunlap {
4430e7468fSPeter Dunlap iser_hca_t *hca;
4530e7468fSPeter Dunlap iser_wr_t *iser_wr;
4630e7468fSPeter Dunlap iser_msg_t *msg;
4730e7468fSPeter Dunlap ibt_send_wr_t wr;
4830e7468fSPeter Dunlap iser_hello_hdr_t *hdr;
4930e7468fSPeter Dunlap int status;
5030e7468fSPeter Dunlap
5130e7468fSPeter Dunlap ASSERT(chan != NULL);
5230e7468fSPeter Dunlap
5330e7468fSPeter Dunlap hca = (iser_hca_t *)chan->ic_hca;
5430e7468fSPeter Dunlap if (hca == NULL) {
5530e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: no hca handle found");
5630e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
5730e7468fSPeter Dunlap }
5830e7468fSPeter Dunlap
5930e7468fSPeter Dunlap msg = iser_msg_get(hca, 1, NULL);
6030e7468fSPeter Dunlap
6130e7468fSPeter Dunlap if (msg == NULL) {
6230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: iser message cache "
6330e7468fSPeter Dunlap "alloc failed");
6430e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
6530e7468fSPeter Dunlap }
6630e7468fSPeter Dunlap
6730e7468fSPeter Dunlap /* Send iSER Hello Message to declare iSER parameters to the target */
6830e7468fSPeter Dunlap hdr = (iser_hello_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
6930e7468fSPeter Dunlap
7030e7468fSPeter Dunlap hdr->opcode = ISER_OPCODE_HELLO_MSG;
7130e7468fSPeter Dunlap hdr->rsvd1 = 0;
7230e7468fSPeter Dunlap hdr->maxver = 1;
7330e7468fSPeter Dunlap hdr->minver = 1;
7430e7468fSPeter Dunlap hdr->iser_ird = htons(ISER_IB_DEFAULT_IRD);
7530e7468fSPeter Dunlap hdr->rsvd2[0] = 0;
7630e7468fSPeter Dunlap hdr->rsvd2[1] = 0;
7730e7468fSPeter Dunlap
7830e7468fSPeter Dunlap /* Allocate an iSER WR handle and tuck this msg into it */
7930e7468fSPeter Dunlap iser_wr = iser_wr_get();
8030e7468fSPeter Dunlap if (iser_wr == NULL) {
8130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: unable to allocate "
8230e7468fSPeter Dunlap "iser wr handle");
8330e7468fSPeter Dunlap iser_msg_free(msg);
8430e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
8530e7468fSPeter Dunlap }
8630e7468fSPeter Dunlap iser_wr->iw_msg = msg;
8730e7468fSPeter Dunlap iser_wr->iw_type = ISER_WR_SEND;
8830e7468fSPeter Dunlap
8930e7468fSPeter Dunlap /* Use the address of our generic iser_wr handle as our WRID */
9030e7468fSPeter Dunlap wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr;
9130e7468fSPeter Dunlap
9230e7468fSPeter Dunlap /* Populate the rest of the work request */
9330e7468fSPeter Dunlap wr.wr_trans = IBT_RC_SRV;
9430e7468fSPeter Dunlap wr.wr_opcode = IBT_WRC_SEND;
9530e7468fSPeter Dunlap wr.wr_nds = 1;
9630e7468fSPeter Dunlap wr.wr_sgl = &msg->msg_ds;
9730e7468fSPeter Dunlap
98a82ec3cfSPriya Krishnan /*
99a82ec3cfSPriya Krishnan * Avoid race condition by incrementing this channel's
100a82ec3cfSPriya Krishnan * SQ posted count prior to calling ibt_post_send
101a82ec3cfSPriya Krishnan */
10230e7468fSPeter Dunlap mutex_enter(&chan->ic_sq_post_lock);
10330e7468fSPeter Dunlap chan->ic_sq_post_count++;
10430e7468fSPeter Dunlap if (chan->ic_sq_post_count > chan->ic_sq_max_post_count)
10530e7468fSPeter Dunlap chan->ic_sq_max_post_count = chan->ic_sq_post_count;
10630e7468fSPeter Dunlap mutex_exit(&chan->ic_sq_post_lock);
10730e7468fSPeter Dunlap
108a82ec3cfSPriya Krishnan status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL);
109a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) {
110a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: ibt_post_send "
111a82ec3cfSPriya Krishnan "failure (%d)", status);
112a82ec3cfSPriya Krishnan mutex_enter(&chan->ic_sq_post_lock);
113a82ec3cfSPriya Krishnan chan->ic_sq_post_count--;
114a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_sq_post_lock);
115a82ec3cfSPriya Krishnan iser_msg_free(msg);
116a82ec3cfSPriya Krishnan iser_wr_free(iser_wr);
117a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
118a82ec3cfSPriya Krishnan }
119a82ec3cfSPriya Krishnan
12030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "Posting iSER Hello message: chan (0x%p): "
12130e7468fSPeter Dunlap "IP [%x to %x]", (void *)chan, chan->ic_localip.un.ip4addr,
12230e7468fSPeter Dunlap chan->ic_remoteip.un.ip4addr);
12330e7468fSPeter Dunlap
12430e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
12530e7468fSPeter Dunlap }
12630e7468fSPeter Dunlap
12730e7468fSPeter Dunlap int
iser_xfer_helloreply_msg(iser_chan_t * chan)12830e7468fSPeter Dunlap iser_xfer_helloreply_msg(iser_chan_t *chan)
12930e7468fSPeter Dunlap {
13030e7468fSPeter Dunlap iser_hca_t *hca;
13130e7468fSPeter Dunlap iser_wr_t *iser_wr;
13230e7468fSPeter Dunlap ibt_send_wr_t wr;
13330e7468fSPeter Dunlap iser_msg_t *msg;
13430e7468fSPeter Dunlap iser_helloreply_hdr_t *hdr;
13530e7468fSPeter Dunlap int status;
13630e7468fSPeter Dunlap
13730e7468fSPeter Dunlap ASSERT(chan != NULL);
13830e7468fSPeter Dunlap
13930e7468fSPeter Dunlap hca = (iser_hca_t *)chan->ic_hca;
14030e7468fSPeter Dunlap if (hca == NULL) {
14130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: no hca handle "
14230e7468fSPeter Dunlap "found");
14330e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
14430e7468fSPeter Dunlap }
14530e7468fSPeter Dunlap
14630e7468fSPeter Dunlap msg = iser_msg_get(hca, 1, NULL);
14730e7468fSPeter Dunlap
14830e7468fSPeter Dunlap if (msg == NULL) {
14930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: iser message "
15030e7468fSPeter Dunlap "cache alloc failed");
15130e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
15230e7468fSPeter Dunlap }
15330e7468fSPeter Dunlap
15430e7468fSPeter Dunlap /* Use the iSER Hello Reply Message */
15530e7468fSPeter Dunlap hdr = (iser_helloreply_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
15630e7468fSPeter Dunlap
15730e7468fSPeter Dunlap hdr->opcode = ISER_OPCODE_HELLOREPLY_MSG;
15830e7468fSPeter Dunlap hdr->rsvd1 = 0;
15930e7468fSPeter Dunlap hdr->flag = 0;
16030e7468fSPeter Dunlap hdr->maxver = 1;
16130e7468fSPeter Dunlap hdr->curver = 1;
16230e7468fSPeter Dunlap hdr->iser_ord = htons(ISER_IB_DEFAULT_ORD);
16330e7468fSPeter Dunlap hdr->rsvd2[0] = 0;
16430e7468fSPeter Dunlap hdr->rsvd2[1] = 0;
16530e7468fSPeter Dunlap
16630e7468fSPeter Dunlap /* Allocate an iSER WR handle and tuck this msg into it */
16730e7468fSPeter Dunlap iser_wr = iser_wr_get();
16830e7468fSPeter Dunlap if (iser_wr == NULL) {
16930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: unable to "
17030e7468fSPeter Dunlap "allocate iser wr handle");
17130e7468fSPeter Dunlap iser_msg_free(msg);
17230e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
17330e7468fSPeter Dunlap }
17430e7468fSPeter Dunlap iser_wr->iw_msg = msg;
17530e7468fSPeter Dunlap iser_wr->iw_type = ISER_WR_SEND;
17630e7468fSPeter Dunlap
17730e7468fSPeter Dunlap /* Use the address of our generic iser_wr handle as our WRID */
17830e7468fSPeter Dunlap wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr;
17930e7468fSPeter Dunlap
18030e7468fSPeter Dunlap /* Populate the rest of the work request */
18130e7468fSPeter Dunlap wr.wr_trans = IBT_RC_SRV;
18230e7468fSPeter Dunlap wr.wr_opcode = IBT_WRC_SEND;
18330e7468fSPeter Dunlap wr.wr_nds = 1;
18430e7468fSPeter Dunlap wr.wr_sgl = &msg->msg_ds;
18530e7468fSPeter Dunlap
18630e7468fSPeter Dunlap mutex_enter(&chan->ic_sq_post_lock);
18730e7468fSPeter Dunlap chan->ic_sq_post_count++;
18830e7468fSPeter Dunlap if (chan->ic_sq_post_count > chan->ic_sq_max_post_count)
18930e7468fSPeter Dunlap chan->ic_sq_max_post_count = chan->ic_sq_post_count;
190a82ec3cfSPriya Krishnan
19130e7468fSPeter Dunlap mutex_exit(&chan->ic_sq_post_lock);
19230e7468fSPeter Dunlap
193a82ec3cfSPriya Krishnan status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL);
194a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) {
195a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: ibt_post_send "
196a82ec3cfSPriya Krishnan "failure (%d)", status);
197a82ec3cfSPriya Krishnan mutex_enter(&chan->ic_sq_post_lock);
198a82ec3cfSPriya Krishnan chan->ic_sq_post_count--;
199a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_sq_post_lock);
200a82ec3cfSPriya Krishnan iser_msg_free(msg);
201a82ec3cfSPriya Krishnan iser_wr_free(iser_wr);
202a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
203a82ec3cfSPriya Krishnan }
20430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "Posting iSER HelloReply message: chan (0x%p): "
20530e7468fSPeter Dunlap "IP [%x to %x]", (void *)chan, chan->ic_localip.un.ip4addr,
20630e7468fSPeter Dunlap chan->ic_remoteip.un.ip4addr);
20730e7468fSPeter Dunlap
20830e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
20930e7468fSPeter Dunlap }
21030e7468fSPeter Dunlap
21130e7468fSPeter Dunlap /*
21230e7468fSPeter Dunlap * iser_xfer_ctrlpdu
21330e7468fSPeter Dunlap *
21430e7468fSPeter Dunlap * This is iSER's implementation of the 'Send_control' operational primitive.
21530e7468fSPeter Dunlap * This iSER layer uses the Send Message type of RCaP to transfer the iSCSI
21630e7468fSPeter Dunlap * Control-type PDU. A special case is that the transfer of SCSI Data-out PDUs
21730e7468fSPeter Dunlap * carrying unsolicited data are also treated as iSCSI Control-Type PDUs. The
21830e7468fSPeter Dunlap * message payload contains an iSER header followed by the iSCSI Control-type
21930e7468fSPeter Dunlap * the iSCSI Control-type PDU.
22030e7468fSPeter Dunlap * This function is invoked by an initiator iSCSI layer requesting the transfer
22130e7468fSPeter Dunlap * of a iSCSI command PDU or a target iSCSI layer requesting the transfer of a
22230e7468fSPeter Dunlap * iSCSI response PDU.
22330e7468fSPeter Dunlap */
22430e7468fSPeter Dunlap int
iser_xfer_ctrlpdu(iser_chan_t * chan,idm_pdu_t * pdu)22530e7468fSPeter Dunlap iser_xfer_ctrlpdu(iser_chan_t *chan, idm_pdu_t *pdu)
22630e7468fSPeter Dunlap {
22730e7468fSPeter Dunlap iser_hca_t *hca;
22830e7468fSPeter Dunlap iser_ctrl_hdr_t *hdr;
22930e7468fSPeter Dunlap iser_msg_t *msg;
23030e7468fSPeter Dunlap iser_wr_t *iser_wr;
23130e7468fSPeter Dunlap ibt_send_wr_t wr;
23230e7468fSPeter Dunlap int status;
23330e7468fSPeter Dunlap iser_mr_t *mr;
23430e7468fSPeter Dunlap iscsi_data_hdr_t *bhs;
23530e7468fSPeter Dunlap idm_conn_t *ic;
23630e7468fSPeter Dunlap idm_task_t *idt = NULL;
23730e7468fSPeter Dunlap idm_buf_t *buf;
23830e7468fSPeter Dunlap
23930e7468fSPeter Dunlap ASSERT(chan != NULL);
24030e7468fSPeter Dunlap
241a82ec3cfSPriya Krishnan mutex_enter(&chan->ic_conn->ic_lock);
242a82ec3cfSPriya Krishnan /* Bail out if the connection is closed */
243a82ec3cfSPriya Krishnan if ((chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSING) ||
244a82ec3cfSPriya Krishnan (chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSED)) {
245a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_conn->ic_lock);
246a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
247a82ec3cfSPriya Krishnan }
248a82ec3cfSPriya Krishnan
249*60220f10SPriya Krishnan ic = chan->ic_conn->ic_idmc;
250*60220f10SPriya Krishnan
251*60220f10SPriya Krishnan /* Pull the BHS out of the PDU handle */
252*60220f10SPriya Krishnan bhs = (iscsi_data_hdr_t *)pdu->isp_hdr;
253*60220f10SPriya Krishnan
25430e7468fSPeter Dunlap /*
25530e7468fSPeter Dunlap * All SCSI command PDU (except SCSI Read and SCSI Write) and the SCSI
25630e7468fSPeter Dunlap * Response PDU are sent to the remote end using the SendSE Message.
25730e7468fSPeter Dunlap *
258*60220f10SPriya Krishnan * The StatSN may need to be sent (and possibly advanced) at this time
259*60220f10SPriya Krishnan * for some PDUs, identified by the IDM_PDU_SET_STATSN flag.
260*60220f10SPriya Krishnan */
261*60220f10SPriya Krishnan if (pdu->isp_flags & IDM_PDU_SET_STATSN) {
262*60220f10SPriya Krishnan (ic->ic_conn_ops.icb_update_statsn)(NULL, pdu);
263*60220f10SPriya Krishnan }
264*60220f10SPriya Krishnan /*
26530e7468fSPeter Dunlap * Setup a Send Message for carrying the iSCSI control-type PDU
26630e7468fSPeter Dunlap * preceeded by an iSER header.
26730e7468fSPeter Dunlap */
26830e7468fSPeter Dunlap hca = (iser_hca_t *)chan->ic_hca;
26930e7468fSPeter Dunlap if (hca == NULL) {
27030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: no hca handle found");
271a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_conn->ic_lock);
27230e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
27330e7468fSPeter Dunlap }
27430e7468fSPeter Dunlap
27530e7468fSPeter Dunlap msg = iser_msg_get(hca, 1, NULL);
27630e7468fSPeter Dunlap if (msg == NULL) {
27730e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: iser message cache "
27830e7468fSPeter Dunlap "alloc failed");
279a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_conn->ic_lock);
28030e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
28130e7468fSPeter Dunlap }
28230e7468fSPeter Dunlap
28330e7468fSPeter Dunlap hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
28430e7468fSPeter Dunlap
28530e7468fSPeter Dunlap /*
28630e7468fSPeter Dunlap * Initialize header assuming no transfers
28730e7468fSPeter Dunlap */
28830e7468fSPeter Dunlap bzero(hdr, sizeof (*hdr));
28930e7468fSPeter Dunlap hdr->opcode = ISER_OPCODE_CTRL_TYPE_PDU;
29030e7468fSPeter Dunlap
29130e7468fSPeter Dunlap /*
29230e7468fSPeter Dunlap * On the initiator side, the task buffers will be used to identify
29330e7468fSPeter Dunlap * if there are any buffers to be advertised
29430e7468fSPeter Dunlap */
29530e7468fSPeter Dunlap if ((ic->ic_conn_type == CONN_TYPE_INI) &&
29630e7468fSPeter Dunlap ((bhs->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_SCSI_CMD) &&
29730e7468fSPeter Dunlap ((idt = idm_task_find(ic, bhs->itt, bhs->ttt)) != NULL)) {
29830e7468fSPeter Dunlap
29930e7468fSPeter Dunlap if (!list_is_empty(&idt->idt_inbufv)) {
30030e7468fSPeter Dunlap buf = idm_buf_find(&idt->idt_inbufv, 0);
30130e7468fSPeter Dunlap ASSERT(buf != NULL);
30230e7468fSPeter Dunlap
30330e7468fSPeter Dunlap mr = (iser_mr_t *)buf->idb_reg_private;
30430e7468fSPeter Dunlap ASSERT(mr != NULL);
30530e7468fSPeter Dunlap
30630e7468fSPeter Dunlap hdr->rsv_flag = 1;
30730e7468fSPeter Dunlap hdr->rstag = htonl(mr->is_mrrkey);
30830e7468fSPeter Dunlap BE_OUT64(&hdr->rva, mr->is_mrva);
30930e7468fSPeter Dunlap }
31030e7468fSPeter Dunlap
31130e7468fSPeter Dunlap if (!list_is_empty(&idt->idt_outbufv)) {
31230e7468fSPeter Dunlap buf = idm_buf_find(&idt->idt_outbufv, 0);
31330e7468fSPeter Dunlap ASSERT(buf != NULL);
31430e7468fSPeter Dunlap
31530e7468fSPeter Dunlap mr = (iser_mr_t *)buf->idb_reg_private;
31630e7468fSPeter Dunlap ASSERT(mr != NULL);
31730e7468fSPeter Dunlap
31830e7468fSPeter Dunlap hdr->wsv_flag = 1;
31930e7468fSPeter Dunlap hdr->wstag = htonl(mr->is_mrrkey);
32030e7468fSPeter Dunlap BE_OUT64(&hdr->wva, mr->is_mrva);
32130e7468fSPeter Dunlap }
32230e7468fSPeter Dunlap
32330e7468fSPeter Dunlap /* Release our reference on the task */
32430e7468fSPeter Dunlap idm_task_rele(idt);
32530e7468fSPeter Dunlap }
32630e7468fSPeter Dunlap
32730e7468fSPeter Dunlap /* Copy the BHS after the iSER header */
32830e7468fSPeter Dunlap bcopy(pdu->isp_hdr,
32930e7468fSPeter Dunlap (uint8_t *)(uintptr_t)msg->msg_ds.ds_va + ISER_HEADER_LENGTH,
33030e7468fSPeter Dunlap pdu->isp_hdrlen);
33130e7468fSPeter Dunlap
33230e7468fSPeter Dunlap if (pdu->isp_datalen > 0) {
33330e7468fSPeter Dunlap /* Copy the isp_data after the PDU header */
33430e7468fSPeter Dunlap bcopy(pdu->isp_data,
33530e7468fSPeter Dunlap (uint8_t *)(uintptr_t)msg->msg_ds.ds_va +
33630e7468fSPeter Dunlap ISER_HEADER_LENGTH + pdu->isp_hdrlen,
33730e7468fSPeter Dunlap pdu->isp_datalen);
33830e7468fSPeter Dunlap
33930e7468fSPeter Dunlap /* Set the SGE's ds_len */
34030e7468fSPeter Dunlap msg->msg_ds.ds_len = ISER_HEADER_LENGTH + pdu->isp_hdrlen +
34130e7468fSPeter Dunlap pdu->isp_datalen;
34230e7468fSPeter Dunlap } else {
34330e7468fSPeter Dunlap /* No data, so set the SGE's ds_len to the headers length */
34430e7468fSPeter Dunlap msg->msg_ds.ds_len = ISER_HEADER_LENGTH + pdu->isp_hdrlen;
34530e7468fSPeter Dunlap }
34630e7468fSPeter Dunlap
34730e7468fSPeter Dunlap /*
34830e7468fSPeter Dunlap * Build Work Request to be posted on the Send Queue.
34930e7468fSPeter Dunlap */
35030e7468fSPeter Dunlap bzero(&wr, sizeof (wr));
35130e7468fSPeter Dunlap
35230e7468fSPeter Dunlap /* Allocate an iSER WR handle and tuck the msg and pdu into it */
35330e7468fSPeter Dunlap iser_wr = iser_wr_get();
35430e7468fSPeter Dunlap if (iser_wr == NULL) {
35530e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: unable to allocate "
35630e7468fSPeter Dunlap "iser wr handle");
35730e7468fSPeter Dunlap iser_msg_free(msg);
358a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_conn->ic_lock);
35930e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
36030e7468fSPeter Dunlap }
36130e7468fSPeter Dunlap iser_wr->iw_pdu = pdu;
36230e7468fSPeter Dunlap iser_wr->iw_msg = msg;
36330e7468fSPeter Dunlap iser_wr->iw_type = ISER_WR_SEND;
36430e7468fSPeter Dunlap
36530e7468fSPeter Dunlap /*
36630e7468fSPeter Dunlap * Use the address of our generic iser_wr handle as our WRID
36730e7468fSPeter Dunlap * and populate the rest of the work request
36830e7468fSPeter Dunlap */
36930e7468fSPeter Dunlap wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr;
37030e7468fSPeter Dunlap wr.wr_trans = IBT_RC_SRV;
37130e7468fSPeter Dunlap wr.wr_opcode = IBT_WRC_SEND;
37230e7468fSPeter Dunlap wr.wr_nds = 1;
37330e7468fSPeter Dunlap wr.wr_sgl = &msg->msg_ds;
37430e7468fSPeter Dunlap
37530e7468fSPeter Dunlap /* Increment this channel's SQ posted count */
37630e7468fSPeter Dunlap mutex_enter(&chan->ic_sq_post_lock);
37730e7468fSPeter Dunlap chan->ic_sq_post_count++;
37830e7468fSPeter Dunlap if (chan->ic_sq_post_count > chan->ic_sq_max_post_count)
37930e7468fSPeter Dunlap chan->ic_sq_max_post_count = chan->ic_sq_post_count;
38030e7468fSPeter Dunlap mutex_exit(&chan->ic_sq_post_lock);
38130e7468fSPeter Dunlap
382a82ec3cfSPriya Krishnan /* Post Send Work Request on the specified channel */
383a82ec3cfSPriya Krishnan status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL);
384a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) {
385a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: ibt_post_send "
386a82ec3cfSPriya Krishnan "failure (%d)", status);
387a82ec3cfSPriya Krishnan iser_msg_free(msg);
388a82ec3cfSPriya Krishnan iser_wr_free(iser_wr);
389a82ec3cfSPriya Krishnan mutex_enter(&chan->ic_sq_post_lock);
390a82ec3cfSPriya Krishnan chan->ic_sq_post_count--;
391a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_sq_post_lock);
392a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_conn->ic_lock);
393a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
394a82ec3cfSPriya Krishnan }
395a82ec3cfSPriya Krishnan
396a82ec3cfSPriya Krishnan mutex_exit(&chan->ic_conn->ic_lock);
39730e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
39830e7468fSPeter Dunlap }
39930e7468fSPeter Dunlap
40030e7468fSPeter Dunlap /*
40130e7468fSPeter Dunlap * iser_xfer_buf_to_ini
40230e7468fSPeter Dunlap * This is iSER's implementation of the 'Put_Data' operational primitive.
40330e7468fSPeter Dunlap * The iSCSI layer at the target invokes this function when it is ready to
40430e7468fSPeter Dunlap * return the SCSI Read Data to the initiator. This function generates and
40530e7468fSPeter Dunlap * sends an RDMA Write Message containing the read data to the initiator.
40630e7468fSPeter Dunlap */
40730e7468fSPeter Dunlap int
iser_xfer_buf_to_ini(idm_task_t * idt,idm_buf_t * buf)40830e7468fSPeter Dunlap iser_xfer_buf_to_ini(idm_task_t *idt, idm_buf_t *buf)
40930e7468fSPeter Dunlap {
41030e7468fSPeter Dunlap iser_conn_t *iser_conn;
41130e7468fSPeter Dunlap iser_chan_t *iser_chan;
41230e7468fSPeter Dunlap iser_buf_t *iser_buf;
41330e7468fSPeter Dunlap iser_wr_t *iser_wr;
41430e7468fSPeter Dunlap iser_ctrl_hdr_t *iser_hdr;
41530e7468fSPeter Dunlap ibt_send_wr_t wr;
41630e7468fSPeter Dunlap uint64_t reg_raddr;
41730e7468fSPeter Dunlap uint32_t reg_rkey;
41830e7468fSPeter Dunlap int status;
41930e7468fSPeter Dunlap
42030e7468fSPeter Dunlap /* Grab the iSER resources from the task and buf handles */
42130e7468fSPeter Dunlap iser_conn = (iser_conn_t *)idt->idt_ic->ic_transport_private;
42230e7468fSPeter Dunlap iser_chan = iser_conn->ic_chan;
423a82ec3cfSPriya Krishnan
424a82ec3cfSPriya Krishnan mutex_enter(&iser_chan->ic_conn->ic_lock);
425a82ec3cfSPriya Krishnan /* Bail out if the connection is closed */
426a82ec3cfSPriya Krishnan if ((iser_chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSING) ||
427a82ec3cfSPriya Krishnan (iser_chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSED)) {
428a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
429a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
430a82ec3cfSPriya Krishnan }
431a82ec3cfSPriya Krishnan
43230e7468fSPeter Dunlap iser_buf = (iser_buf_t *)buf->idb_buf_private;
43330e7468fSPeter Dunlap iser_hdr = (iser_ctrl_hdr_t *)idt->idt_transport_hdr;
43430e7468fSPeter Dunlap
43530e7468fSPeter Dunlap /* Pull the Read STag data out of the iSER header in the task hdl */
43630e7468fSPeter Dunlap reg_raddr = BE_IN64(&iser_hdr->rva);
43730e7468fSPeter Dunlap reg_rkey = (ntohl(iser_hdr->rstag));
43830e7468fSPeter Dunlap
43930e7468fSPeter Dunlap /* Set up the WR raddr and rkey based upon the Read iSER STag */
44030e7468fSPeter Dunlap bzero(&wr, sizeof (ibt_send_wr_t));
44130e7468fSPeter Dunlap wr.wr.rc.rcwr.rdma.rdma_raddr = reg_raddr + buf->idb_bufoffset;
44230e7468fSPeter Dunlap wr.wr.rc.rcwr.rdma.rdma_rkey = reg_rkey;
44330e7468fSPeter Dunlap
44430e7468fSPeter Dunlap /* Set the transfer length from the IDM buf handle */
44530e7468fSPeter Dunlap iser_buf->buf_ds.ds_len = buf->idb_xfer_len;
44630e7468fSPeter Dunlap
44730e7468fSPeter Dunlap /* Allocate an iSER WR handle and tuck the IDM buf handle into it */
44830e7468fSPeter Dunlap iser_wr = iser_wr_get();
44930e7468fSPeter Dunlap if (iser_wr == NULL) {
45030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_buf_to_ini: unable to allocate "
45130e7468fSPeter Dunlap "iser wr handle");
452a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
45330e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
45430e7468fSPeter Dunlap }
45530e7468fSPeter Dunlap iser_wr->iw_buf = buf;
45630e7468fSPeter Dunlap iser_wr->iw_type = ISER_WR_RDMAW;
45730e7468fSPeter Dunlap
45830e7468fSPeter Dunlap /* Use the address of our generic iser_wr handle as our WRID */
45930e7468fSPeter Dunlap wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr;
46030e7468fSPeter Dunlap
46130e7468fSPeter Dunlap /* Populate the rest of the work request */
46230e7468fSPeter Dunlap wr.wr_flags = IBT_WR_SEND_SIGNAL;
46330e7468fSPeter Dunlap wr.wr_trans = IBT_RC_SRV;
46430e7468fSPeter Dunlap wr.wr_opcode = IBT_WRC_RDMAW;
46530e7468fSPeter Dunlap wr.wr_nds = 1;
46630e7468fSPeter Dunlap wr.wr_sgl = &iser_buf->buf_ds;
46730e7468fSPeter Dunlap
46830e7468fSPeter Dunlap #ifdef DEBUG
46930e7468fSPeter Dunlap bcopy(&wr, &iser_buf->buf_wr, sizeof (ibt_send_wr_t));
47030e7468fSPeter Dunlap #endif
47130e7468fSPeter Dunlap
472a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__start, idm_conn_t *, idt->idt_ic,
473a668b114SPriya Krishnan uintptr_t, buf->idb_buf, uint32_t, buf->idb_bufoffset,
474a668b114SPriya Krishnan uint64_t, reg_raddr, uint32_t, buf->idb_bufoffset,
475a668b114SPriya Krishnan uint32_t, reg_rkey,
476a668b114SPriya Krishnan uint32_t, buf->idb_xfer_len, int, XFER_BUF_TX_TO_INI);
477a668b114SPriya Krishnan
47830e7468fSPeter Dunlap /* Increment this channel's SQ posted count */
47930e7468fSPeter Dunlap mutex_enter(&iser_chan->ic_sq_post_lock);
48030e7468fSPeter Dunlap iser_chan->ic_sq_post_count++;
48130e7468fSPeter Dunlap if (iser_chan->ic_sq_post_count > iser_chan->ic_sq_max_post_count)
48230e7468fSPeter Dunlap iser_chan->ic_sq_max_post_count = iser_chan->ic_sq_post_count;
48330e7468fSPeter Dunlap mutex_exit(&iser_chan->ic_sq_post_lock);
48430e7468fSPeter Dunlap
485a82ec3cfSPriya Krishnan status = ibt_post_send(iser_chan->ic_chanhdl, &wr, 1, NULL);
486a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) {
487a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_xfer_buf_to_ini: ibt_post_send "
488a82ec3cfSPriya Krishnan "failure (%d)", status);
489a82ec3cfSPriya Krishnan iser_wr_free(iser_wr);
490a82ec3cfSPriya Krishnan mutex_enter(&iser_chan->ic_sq_post_lock);
491a82ec3cfSPriya Krishnan iser_chan->ic_sq_post_count--;
492a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_sq_post_lock);
493a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
494a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
495a82ec3cfSPriya Krishnan }
496a82ec3cfSPriya Krishnan
497a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
49830e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
49930e7468fSPeter Dunlap }
50030e7468fSPeter Dunlap
50130e7468fSPeter Dunlap /*
50230e7468fSPeter Dunlap * iser_xfer_buf_from_ini
50330e7468fSPeter Dunlap * This is iSER's implementation of the 'Get_Data' operational primitive.
50430e7468fSPeter Dunlap * The iSCSI layer at the target invokes this function when it is ready to
50530e7468fSPeter Dunlap * receive the SCSI Write Data from the initiator. This function generates and
50630e7468fSPeter Dunlap * sends an RDMA Read Message to get the data from the initiator. No R2T PDUs
50730e7468fSPeter Dunlap * are generated.
50830e7468fSPeter Dunlap */
50930e7468fSPeter Dunlap int
iser_xfer_buf_from_ini(idm_task_t * idt,idm_buf_t * buf)51030e7468fSPeter Dunlap iser_xfer_buf_from_ini(idm_task_t *idt, idm_buf_t *buf)
51130e7468fSPeter Dunlap {
51230e7468fSPeter Dunlap iser_conn_t *iser_conn;
51330e7468fSPeter Dunlap iser_chan_t *iser_chan;
51430e7468fSPeter Dunlap iser_buf_t *iser_buf;
51530e7468fSPeter Dunlap iser_wr_t *iser_wr;
51630e7468fSPeter Dunlap iser_ctrl_hdr_t *iser_hdr;
51730e7468fSPeter Dunlap ibt_send_wr_t wr;
51830e7468fSPeter Dunlap uint64_t reg_raddr;
51930e7468fSPeter Dunlap uint32_t reg_rkey;
52030e7468fSPeter Dunlap int status;
52130e7468fSPeter Dunlap
52230e7468fSPeter Dunlap /* Grab the iSER resources from the task and buf handles */
52330e7468fSPeter Dunlap iser_conn = (iser_conn_t *)idt->idt_ic->ic_transport_private;
52430e7468fSPeter Dunlap iser_chan = iser_conn->ic_chan;
525a82ec3cfSPriya Krishnan
526a82ec3cfSPriya Krishnan mutex_enter(&iser_chan->ic_conn->ic_lock);
527a82ec3cfSPriya Krishnan /* Bail out if the connection is closed */
528a82ec3cfSPriya Krishnan if ((iser_chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSING) ||
529a82ec3cfSPriya Krishnan (iser_chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSED)) {
530a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
531a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
532a82ec3cfSPriya Krishnan }
533a82ec3cfSPriya Krishnan
53430e7468fSPeter Dunlap iser_buf = (iser_buf_t *)buf->idb_buf_private;
53530e7468fSPeter Dunlap iser_hdr = (iser_ctrl_hdr_t *)idt->idt_transport_hdr;
53630e7468fSPeter Dunlap
53730e7468fSPeter Dunlap /* Pull the Write STag data out of the iSER header in the task hdl */
53830e7468fSPeter Dunlap reg_raddr = BE_IN64(&iser_hdr->wva);
53930e7468fSPeter Dunlap reg_rkey = (ntohl(iser_hdr->wstag));
54030e7468fSPeter Dunlap
54130e7468fSPeter Dunlap /* Set up the WR raddr and rkey based upon the iSER Write STag */
54230e7468fSPeter Dunlap bzero(&wr, sizeof (ibt_send_wr_t));
54330e7468fSPeter Dunlap wr.wr.rc.rcwr.rdma.rdma_raddr = reg_raddr + buf->idb_bufoffset;
54430e7468fSPeter Dunlap wr.wr.rc.rcwr.rdma.rdma_rkey = reg_rkey;
54530e7468fSPeter Dunlap
54630e7468fSPeter Dunlap /* Set the transfer length from the IDM buf handle */
54730e7468fSPeter Dunlap iser_buf->buf_ds.ds_len = buf->idb_xfer_len;
54830e7468fSPeter Dunlap
54930e7468fSPeter Dunlap /* Allocate an iSER WR handle and tuck the IDM buf handle into it */
55030e7468fSPeter Dunlap iser_wr = iser_wr_get();
55130e7468fSPeter Dunlap if (iser_wr == NULL) {
55230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_xfer_buf_from_ini: unable to allocate "
55330e7468fSPeter Dunlap "iser wr handle");
554a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
55530e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
55630e7468fSPeter Dunlap }
55730e7468fSPeter Dunlap iser_wr->iw_buf = buf;
55830e7468fSPeter Dunlap iser_wr->iw_type = ISER_WR_RDMAR;
55930e7468fSPeter Dunlap
56030e7468fSPeter Dunlap /* Use the address of our generic iser_wr handle as our WRID */
56130e7468fSPeter Dunlap wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr;
56230e7468fSPeter Dunlap
56330e7468fSPeter Dunlap /* Populate the rest of the work request */
56430e7468fSPeter Dunlap wr.wr_flags = IBT_WR_SEND_SIGNAL;
56530e7468fSPeter Dunlap wr.wr_trans = IBT_RC_SRV;
56630e7468fSPeter Dunlap wr.wr_opcode = IBT_WRC_RDMAR;
56730e7468fSPeter Dunlap wr.wr_nds = 1;
56830e7468fSPeter Dunlap wr.wr_sgl = &iser_buf->buf_ds;
56930e7468fSPeter Dunlap
57030e7468fSPeter Dunlap #ifdef DEBUG
57130e7468fSPeter Dunlap bcopy(&wr, &iser_buf->buf_wr, sizeof (ibt_send_wr_t));
57230e7468fSPeter Dunlap #endif
57330e7468fSPeter Dunlap
574a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__start, idm_conn_t *, idt->idt_ic,
575a668b114SPriya Krishnan uintptr_t, buf->idb_buf, uint32_t, buf->idb_bufoffset,
576a668b114SPriya Krishnan uint64_t, reg_raddr, uint32_t, buf->idb_bufoffset,
577a668b114SPriya Krishnan uint32_t, reg_rkey,
578a668b114SPriya Krishnan uint32_t, buf->idb_xfer_len, int, XFER_BUF_RX_FROM_INI);
579a668b114SPriya Krishnan
58030e7468fSPeter Dunlap /* Increment this channel's SQ posted count */
58130e7468fSPeter Dunlap mutex_enter(&iser_chan->ic_sq_post_lock);
58230e7468fSPeter Dunlap iser_chan->ic_sq_post_count++;
58330e7468fSPeter Dunlap if (iser_chan->ic_sq_post_count > iser_chan->ic_sq_max_post_count)
58430e7468fSPeter Dunlap iser_chan->ic_sq_max_post_count = iser_chan->ic_sq_post_count;
58530e7468fSPeter Dunlap mutex_exit(&iser_chan->ic_sq_post_lock);
58630e7468fSPeter Dunlap
587a82ec3cfSPriya Krishnan status = ibt_post_send(iser_chan->ic_chanhdl, &wr, 1, NULL);
588a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) {
589a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_xfer_buf_from_ini: ibt_post_send "
590a82ec3cfSPriya Krishnan "failure (%d)", status);
591a82ec3cfSPriya Krishnan iser_wr_free(iser_wr);
592a82ec3cfSPriya Krishnan mutex_enter(&iser_chan->ic_sq_post_lock);
593a82ec3cfSPriya Krishnan iser_chan->ic_sq_post_count--;
594a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_sq_post_lock);
595a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
596a82ec3cfSPriya Krishnan return (ISER_STATUS_FAIL);
597a82ec3cfSPriya Krishnan }
598a82ec3cfSPriya Krishnan
599a82ec3cfSPriya Krishnan mutex_exit(&iser_chan->ic_conn->ic_lock);
60030e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
60130e7468fSPeter Dunlap }
602