68 static void idm_sorx_cache_pdu_cb(idm_pdu_t *pdu, idm_status_t status);
69 static void idm_sorx_addl_pdu_cb(idm_pdu_t *pdu, idm_status_t status);
70 static void idm_sotx_cache_pdu_cb(idm_pdu_t *pdu, idm_status_t status);
79 static idm_status_t idm_i_so_tx(idm_pdu_t *pdu);
81 static idm_status_t idm_sorecvdata(idm_conn_t *ic, idm_pdu_t *pdu);
88 static uint32_t idm_fill_iov(idm_pdu_t *pdu, idm_buf_t *idb,
100 static void idm_so_tx(idm_conn_t *ic, idm_pdu_t *pdu);
103 static void idm_so_rx_datain(idm_conn_t *ic, idm_pdu_t *pdu);
104 static void idm_so_rx_rtt(idm_conn_t *ic, idm_pdu_t *pdu);
105 static void idm_so_rx_dataout(idm_conn_t *ic, idm_pdu_t *pdu);
738 idm_dataseglenokay(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_dataseglenokay()  argument
743 	    pdu->isp_datalen > ic->ic_conn_params.max_recv_dataseglen) {  in idm_dataseglenokay()
749 	bhs = pdu->isp_hdr;  in idm_dataseglenokay()
753 	switch (IDM_PDU_OPCODE(pdu)) {  in idm_dataseglenokay()
762 		if (bhs->hlength != 0 || pdu->isp_datalen != 0)  in idm_dataseglenokay()
795 		if (pdu->isp_datalen) {  in idm_dataseglenokay()
800 			    ((pdu->isp_datalen & 0x3) != 0))  in idm_dataseglenokay()
826 		    ((pdu->isp_datalen & 0x3) != 0))  in idm_dataseglenokay()
836 idm_sorecvhdr(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_sorecvhdr()  argument
854 	bhs = pdu->isp_hdr;  in idm_sorecvhdr()
855 	rc = idm_sorecv(so_conn->ic_so, pdu->isp_hdr, sizeof (iscsi_hdr_t));  in idm_sorecvhdr()
863 	if ((IDM_PDU_OPCODE(pdu) != ISCSI_OP_SCSI_CMD) &&  in idm_sorecvhdr()
868 	pdu->isp_hdrlen = sizeof (iscsi_hdr_t) +  in idm_sorecvhdr()
870 	pdu->isp_datalen = n2h24(bhs->dlength);  in idm_sorecvhdr()
872 	if (!idm_dataseglenokay(ic, pdu)) {  in idm_sorecvhdr()
879 		new_hdr = kmem_alloc(pdu->isp_hdrlen, KM_SLEEP);  in idm_sorecvhdr()
880 		bcopy(pdu->isp_hdr, new_hdr, sizeof (iscsi_hdr_t));  in idm_sorecvhdr()
881 		pdu->isp_hdr = new_hdr;  in idm_sorecvhdr()
882 		pdu->isp_flags |= IDM_PDU_ADDL_HDR;  in idm_sorecvhdr()
888 		pdu->isp_callback = idm_sorx_addl_pdu_cb;  in idm_sorecvhdr()
895 		iov[iovlen].iov_base = (caddr_t)(pdu->isp_hdr + 1);  in idm_sorecvhdr()
896 		ahslen = pdu->isp_hdrlen - sizeof (iscsi_hdr_t);  in idm_sorecvhdr()
919 		crc_calculated = idm_crc32c(pdu->isp_hdr,  in idm_sorecvhdr()
1645 idm_so_rx_datain(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_so_rx_datain()  argument
1652 	iscsi_hdr_t		*ihp = (iscsi_hdr_t *)pdu->isp_hdr;  in idm_so_rx_datain()
1656 	ASSERT(pdu != NULL);  in idm_so_rx_datain()
1657 	ASSERT(IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA_RSP);  in idm_so_rx_datain()
1659 	bhs	= (iscsi_data_hdr_t *)pdu->isp_hdr;  in idm_so_rx_datain()
1670 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_datain()
1674 	idb = pdu->isp_sorx_buf;  in idm_so_rx_datain()
1679 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_datain()
1692 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_datain()
1703 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_datain()
1717 	if ((IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA_RSP) &&  in idm_so_rx_datain()
1719 		(*ic->ic_conn_ops.icb_rx_scsi_rsp)(ic, pdu);  in idm_so_rx_datain()
1722 	idm_pdu_complete(pdu, IDM_STATUS_SUCCESS);  in idm_so_rx_datain()
1734 idm_so_rx_dataout(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_so_rx_dataout()  argument
1743 	ASSERT(pdu != NULL);  in idm_so_rx_dataout()
1744 	ASSERT(IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA);  in idm_so_rx_dataout()
1746 	bhs = (iscsi_data_hdr_t *)pdu->isp_hdr;  in idm_so_rx_dataout()
1757 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_dataout()
1761 	idb = pdu->isp_sorx_buf;  in idm_so_rx_dataout()
1766 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_dataout()
1775 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_dataout()
1815 			idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_dataout()
1850 		idm_pdu_complete(pdu, IDM_STATUS_SUCCESS);  in idm_so_rx_dataout()
1855 	idm_pdu_complete(pdu, IDM_STATUS_SUCCESS);  in idm_so_rx_dataout()
1870 idm_so_rx_rtt(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_so_rx_rtt()  argument
1879 	ASSERT(pdu != NULL);  in idm_so_rx_rtt()
1881 	rtt_hdr	= (iscsi_rtt_hdr_t *)pdu->isp_hdr;  in idm_so_rx_rtt()
1888 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_rtt()
1899 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_rtt()
1910 		idm_pdu_rx_protocol_error(ic, pdu);  in idm_so_rx_rtt()
1923 	idm_pdu_complete(pdu, IDM_STATUS_SUCCESS);  in idm_so_rx_rtt()
1929 idm_sorecvdata(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_sorecvdata()  argument
1941 	    (pdu->isp_datalen & (ISCSI_PAD_WORD_LEN - 1))) &  in idm_sorecvdata()
1944 	ASSERT(pdu->isp_iovlen < (PDU_MAX_IOVLEN - 2)); /* pad + data digest */  in idm_sorecvdata()
1946 	total_len = pdu->isp_datalen;  in idm_sorecvdata()
1949 		pdu->isp_iov[pdu->isp_iovlen].iov_base	= (char *)&pad;  in idm_sorecvdata()
1950 		pdu->isp_iov[pdu->isp_iovlen].iov_len	= pad_len;  in idm_sorecvdata()
1952 		pdu->isp_iovlen++;  in idm_sorecvdata()
1957 		pdu->isp_iov[pdu->isp_iovlen].iov_base =  in idm_sorecvdata()
1959 		pdu->isp_iov[pdu->isp_iovlen].iov_len =  in idm_sorecvdata()
1962 		pdu->isp_iovlen++;  in idm_sorecvdata()
1965 	pdu->isp_data = (uint8_t *)(uintptr_t)pdu->isp_iov[0].iov_base;  in idm_sorecvdata()
1967 	if (idm_iov_sorecv(so_conn->ic_so, &pdu->isp_iov[0],  in idm_sorecvdata()
1968 	    pdu->isp_iovlen, total_len) != 0) {  in idm_sorecvdata()
1973 		crc_calculated = idm_crc32c(pdu->isp_data,  in idm_sorecvdata()
1974 		    pdu->isp_datalen);  in idm_sorecvdata()
1999 idm_sorecv_scsidata(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_sorecv_scsidata()  argument
2011 	ASSERT(pdu != NULL);  in idm_sorecv_scsidata()
2013 	bhs	= (iscsi_data_hdr_t *)pdu->isp_hdr;  in idm_sorecv_scsidata()
2016 	opcode	= IDM_PDU_OPCODE(pdu);  in idm_sorecv_scsidata()
2047 	pdu->isp_sorx_buf = idm_buf_find(buflst, offset);  in idm_sorecv_scsidata()
2050 	if (pdu->isp_sorx_buf == NULL) {  in idm_sorecv_scsidata()
2058 	xfer_bytes = idm_fill_iov(pdu, pdu->isp_sorx_buf, offset, dlength);  in idm_sorecv_scsidata()
2070 	status = idm_sorecvdata(ic, pdu);  in idm_sorecv_scsidata()
2078 idm_fill_iov(idm_pdu_t *pdu, idm_buf_t *idb, uint32_t ro, uint32_t dlength)  in idm_fill_iov()  argument
2085 	pdu->isp_iov[pdu->isp_iovlen].iov_base	=  in idm_fill_iov()
2087 	pdu->isp_iov[pdu->isp_iovlen].iov_len	= xfer_len;  in idm_fill_iov()
2088 	pdu->isp_iovlen++;  in idm_fill_iov()
2094 idm_sorecv_nonscsidata(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_sorecv_nonscsidata()  argument
2096 	pdu->isp_data = kmem_alloc(pdu->isp_datalen, KM_SLEEP);  in idm_sorecv_nonscsidata()
2097 	ASSERT(pdu->isp_data != NULL);  in idm_sorecv_nonscsidata()
2099 	pdu->isp_databuflen = pdu->isp_datalen;  in idm_sorecv_nonscsidata()
2100 	pdu->isp_iov[0].iov_base = (caddr_t)pdu->isp_data;  in idm_sorecv_nonscsidata()
2101 	pdu->isp_iov[0].iov_len = pdu->isp_datalen;  in idm_sorecv_nonscsidata()
2102 	pdu->isp_iovlen = 1;  in idm_sorecv_nonscsidata()
2108 	pdu->isp_flags |= IDM_PDU_ADDL_DATA;  in idm_sorecv_nonscsidata()
2109 	pdu->isp_callback = idm_sorx_addl_pdu_cb;  in idm_sorecv_nonscsidata()
2111 	return (idm_sorecvdata(ic, pdu));  in idm_sorecv_nonscsidata()
2120 	idm_pdu_t	*pdu;  in idm_sorx_thread()  local
2141 		pdu = kmem_cache_alloc(idm.idm_sorx_pdu_cache, KM_SLEEP);  in idm_sorx_thread()
2142 		pdu->isp_ic = ic;  in idm_sorx_thread()
2143 		pdu->isp_flags = 0;  in idm_sorx_thread()
2144 		pdu->isp_transport_hdrlen = 0;  in idm_sorx_thread()
2146 		if ((rc = idm_sorecvhdr(ic, pdu)) != 0) {  in idm_sorx_thread()
2152 			idm_pdu_complete(pdu, IDM_STATUS_FAIL);  in idm_sorx_thread()
2178 		if (pdu->isp_datalen != 0) {  in idm_sorx_thread()
2179 			if ((IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA) ||  in idm_sorx_thread()
2180 			    (IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA_RSP)) {  in idm_sorx_thread()
2181 				rc = idm_sorecv_scsidata(ic, pdu);  in idm_sorx_thread()
2196 				rc = idm_sorecv_nonscsidata(ic, pdu);  in idm_sorx_thread()
2204 				idm_pdu_complete(pdu, IDM_STATUS_FAIL);  in idm_sorx_thread()
2227 		idm_pdu_rx(ic, pdu);  in idm_sorx_thread()
2269 idm_so_tx(idm_conn_t *ic, idm_pdu_t *pdu)  in idm_so_tx()  argument
2273 	ASSERT(pdu->isp_ic == ic);  in idm_so_tx()
2278 		idm_pdu_complete(pdu, IDM_STATUS_ABORTED);  in idm_so_tx()
2282 	list_insert_tail(&so_conn->ic_tx_list, (void *)pdu);  in idm_so_tx()
2288 idm_i_so_tx(idm_pdu_t *pdu)  in idm_i_so_tx()  argument
2290 	idm_conn_t	*ic = pdu->isp_ic;  in idm_i_so_tx()
2304 	iov[iovlen].iov_base	= (caddr_t)pdu->isp_hdr;  in idm_i_so_tx()
2305 	iov[iovlen].iov_len	= pdu->isp_hdrlen;  in idm_i_so_tx()
2310 	if (((pdu->isp_flags & IDM_PDU_LOGIN_TX) == 0) &&  in idm_i_so_tx()
2312 		hdr_digest_crc = idm_crc32c(pdu->isp_hdr, pdu->isp_hdrlen);  in idm_i_so_tx()
2321 	if (pdu->isp_datalen) {  in idm_i_so_tx()
2325 		ihp = (iscsi_data_hdr_t *)pdu->isp_hdr;  in idm_i_so_tx()
2328 		    (IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_CMD ||  in idm_i_so_tx()
2329 		    IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA)) {  in idm_i_so_tx()
2347 					idb->idb_xfer_len += pdu->isp_datalen;  in idm_i_so_tx()
2352 		iov[iovlen].iov_base = (caddr_t)pdu->isp_data;  in idm_i_so_tx()
2353 		iov[iovlen].iov_len  = pdu->isp_datalen;  in idm_i_so_tx()
2360 	    (pdu->isp_datalen & (ISCSI_PAD_WORD_LEN - 1))) &  in idm_i_so_tx()
2376 	    ((pdu->isp_flags & IDM_PDU_LOGIN_TX) == 0) &&  in idm_i_so_tx()
2377 	    (pdu->isp_datalen || pad_len)) {  in idm_i_so_tx()
2382 		if (pdu->isp_datalen) {  in idm_i_so_tx()
2383 			data_digest_crc = idm_crc32c(pdu->isp_data,  in idm_i_so_tx()
2384 			    pdu->isp_datalen);  in idm_i_so_tx()
2404 		    (void *) pdu->isp_data);  in idm_i_so_tx()
2412 	idm_pdu_complete(pdu, status);  in idm_i_so_tx()
2510 	idm_pdu_t		*pdu;  in idm_so_buf_rx_from_ini()  local
2520 	pdu = kmem_cache_alloc(idm.idm_sotx_pdu_cache, KM_SLEEP);  in idm_so_buf_rx_from_ini()
2521 	pdu->isp_ic = idt->idt_ic;  in idm_so_buf_rx_from_ini()
2522 	pdu->isp_flags = IDM_PDU_SET_STATSN;  in idm_so_buf_rx_from_ini()
2523 	bzero(pdu->isp_hdr, sizeof (iscsi_rtt_hdr_t));  in idm_so_buf_rx_from_ini()
2526 	(*idt->idt_ic->ic_conn_ops.icb_build_hdr)(idt, pdu, ISCSI_OP_RTT_RSP);  in idm_so_buf_rx_from_ini()
2529 	rtt = (iscsi_rtt_hdr_t *)(pdu->isp_hdr);  in idm_so_buf_rx_from_ini()
2544 	idm_pdu_tx(pdu);  in idm_so_buf_rx_from_ini()
2697 	idm_pdu_t		*pdu;  in idm_so_send_buf_region()  local
2722 		pdu = kmem_cache_alloc(idm.idm_sotx_pdu_cache, KM_SLEEP);  in idm_so_send_buf_region()
2723 		pdu->isp_ic = ic;  in idm_so_send_buf_region()
2724 		pdu->isp_flags = 0;	/* initialize isp_flags */  in idm_so_send_buf_region()
2732 		bcopy(&idb->idb_data_hdr_tmpl, pdu->isp_hdr,  in idm_so_send_buf_region()
2739 		bhs = (iscsi_data_hdr_t *)(pdu->isp_hdr);  in idm_so_send_buf_region()
2747 		pdu->isp_data	=  (uint8_t *)idb->idb_buf + data_offset;  in idm_so_send_buf_region()
2748 		pdu->isp_datalen = (uint_t)chunk;  in idm_so_send_buf_region()
2754 				pdu->isp_flags |= IDM_PDU_SET_STATSN |  in idm_so_send_buf_region()
2757 				    (idt, pdu);  in idm_so_send_buf_region()
2768 		if (IDM_PDU_OPCODE(pdu) == ISCSI_OP_SCSI_DATA_RSP) {  in idm_so_send_buf_region()
2772 			    (iscsi_data_rsp_hdr_t *)pdu->isp_hdr);  in idm_so_send_buf_region()
2787 		if ((tx_status = idm_i_so_tx(pdu)) != IDM_STATUS_SUCCESS) {  in idm_so_send_buf_region()
2806 	idm_pdu_t	*pdu = hdl;  in idm_sotx_pdu_constructor()  local
2808 	bzero(pdu, sizeof (idm_pdu_t));  in idm_sotx_pdu_constructor()
2809 	pdu->isp_hdr = (iscsi_hdr_t *)(pdu + 1); /* Ptr arithmetic */  in idm_sotx_pdu_constructor()
2810 	pdu->isp_hdrlen = sizeof (iscsi_hdr_t);  in idm_sotx_pdu_constructor()
2811 	pdu->isp_callback = idm_sotx_cache_pdu_cb;  in idm_sotx_pdu_constructor()
2812 	pdu->isp_magic = IDM_PDU_MAGIC;  in idm_sotx_pdu_constructor()
2813 	bzero(pdu->isp_hdr, sizeof (iscsi_hdr_t));  in idm_sotx_pdu_constructor()
2820 idm_sotx_cache_pdu_cb(idm_pdu_t *pdu, idm_status_t status)  in idm_sotx_cache_pdu_cb()  argument
2823 	pdu->isp_datalen = 0;  in idm_sotx_cache_pdu_cb()
2825 	kmem_cache_free(idm.idm_sotx_pdu_cache, pdu);  in idm_sotx_cache_pdu_cb()
2835 	idm_pdu_t	*pdu = hdl;  in idm_sorx_pdu_constructor()  local
2837 	bzero(pdu, sizeof (idm_pdu_t));  in idm_sorx_pdu_constructor()
2838 	pdu->isp_magic = IDM_PDU_MAGIC;  in idm_sorx_pdu_constructor()
2839 	pdu->isp_hdr = (iscsi_hdr_t *)(pdu + 1); /* Ptr arithmetic */  in idm_sorx_pdu_constructor()
2840 	pdu->isp_callback = idm_sorx_cache_pdu_cb;  in idm_sorx_pdu_constructor()
2847 idm_sorx_cache_pdu_cb(idm_pdu_t *pdu, idm_status_t status)  in idm_sorx_cache_pdu_cb()  argument
2849 	pdu->isp_iovlen = 0;  in idm_sorx_cache_pdu_cb()
2850 	pdu->isp_sorx_buf = 0;  in idm_sorx_cache_pdu_cb()
2851 	kmem_cache_free(idm.idm_sorx_pdu_cache, pdu);  in idm_sorx_cache_pdu_cb()
2855 idm_sorx_addl_pdu_cb(idm_pdu_t *pdu, idm_status_t status)  in idm_sorx_addl_pdu_cb()  argument
2862 	if (pdu->isp_flags & IDM_PDU_ADDL_HDR) {  in idm_sorx_addl_pdu_cb()
2863 		kmem_free(pdu->isp_hdr, pdu->isp_hdrlen);  in idm_sorx_addl_pdu_cb()
2865 	if (pdu->isp_flags & IDM_PDU_ADDL_DATA) {  in idm_sorx_addl_pdu_cb()
2866 		kmem_free(pdu->isp_data, pdu->isp_datalen);  in idm_sorx_addl_pdu_cb()
2868 	pdu->isp_hdr = (iscsi_hdr_t *)(pdu + 1);  in idm_sorx_addl_pdu_cb()
2869 	pdu->isp_hdrlen = sizeof (iscsi_hdr_t);  in idm_sorx_addl_pdu_cb()
2870 	pdu->isp_data = NULL;  in idm_sorx_addl_pdu_cb()
2871 	pdu->isp_datalen = 0;  in idm_sorx_addl_pdu_cb()
2872 	pdu->isp_sorx_buf = 0;  in idm_sorx_addl_pdu_cb()
2873 	pdu->isp_callback = idm_sorx_cache_pdu_cb;  in idm_sorx_addl_pdu_cb()
2874 	idm_sorx_cache_pdu_cb(pdu, status);  in idm_sorx_addl_pdu_cb()
2917 			idm_pdu_t *pdu = (idm_pdu_t *)object;  in idm_sotx_thread()  local
2921 			if (pdu->isp_flags & IDM_PDU_SET_STATSN) {  in idm_sotx_thread()
2923 				(ic->ic_conn_ops.icb_update_statsn)(NULL, pdu);  in idm_sotx_thread()