144     "Maximum size of data payload in a transmitted PDU");
306 nvmf_tcp_validate_pdu(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_validate_pdu()  argument
309 	struct mbuf *m = pdu->m;  in nvmf_tcp_validate_pdu()
316 	/* Determine how large of a PDU header to return for errors. */  in nvmf_tcp_validate_pdu()
317 	ch = pdu->hdr;  in nvmf_tcp_validate_pdu()
346 	pdu->data_digest_mismatch = false;  in nvmf_tcp_validate_pdu()
353 			pdu->data_digest_mismatch = true;  in nvmf_tcp_validate_pdu()
357 	pdu->data_len = data_len;  in nvmf_tcp_validate_pdu()
362 nvmf_tcp_free_pdu(struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_free_pdu()  argument
364 	m_freem(pdu->m);  in nvmf_tcp_free_pdu()
365 	pdu->m = NULL;  in nvmf_tcp_free_pdu()
366 	pdu->hdr = NULL;  in nvmf_tcp_free_pdu()
370 nvmf_tcp_handle_term_req(struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_handle_term_req()  argument
374 	hdr = (const void *)pdu->hdr;  in nvmf_tcp_handle_term_req()
378 	nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_term_req()
384     struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_save_command_capsule()  argument
390 	cmd = (const void *)pdu->hdr;  in nvmf_tcp_save_command_capsule()
395 	tc->rx_pdu = *pdu;  in nvmf_tcp_save_command_capsule()
403     struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_save_response_capsule()  argument
409 	rsp = (const void *)pdu->hdr;  in nvmf_tcp_save_response_capsule()
415 	tc->rx_pdu = *pdu;  in nvmf_tcp_save_response_capsule()
429  * Construct a PDU that contains an optional data payload.  This
637 nvmf_tcp_handle_h2c_data(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_handle_h2c_data()  argument
644 	h2c = (const void *)pdu->hdr;  in nvmf_tcp_handle_h2c_data()
648 		    pdu->m, pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
649 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
661 		    offsetof(struct nvme_tcp_h2c_data_hdr, ttag), pdu->m,  in nvmf_tcp_handle_h2c_data()
662 		    pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
663 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
673 		    offsetof(struct nvme_tcp_h2c_data_hdr, ttag), pdu->m,  in nvmf_tcp_handle_h2c_data()
674 		    pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
675 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
681 	if (pdu->data_digest_mismatch) {  in nvmf_tcp_handle_h2c_data()
685 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
690 	if (data_len != pdu->data_len) {  in nvmf_tcp_handle_h2c_data()
694 		    offsetof(struct nvme_tcp_h2c_data_hdr, datal), pdu->m,  in nvmf_tcp_handle_h2c_data()
695 		    pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
696 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
705 		    NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE, 0, pdu->m,  in nvmf_tcp_handle_h2c_data()
706 		    pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
707 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
714 		    NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,  in nvmf_tcp_handle_h2c_data()
715 		    pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
716 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
721 	    ((pdu->hdr->flags & NVME_TCP_H2C_DATA_FLAGS_LAST_PDU) != 0)) {  in nvmf_tcp_handle_h2c_data()
724 		    NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,  in nvmf_tcp_handle_h2c_data()
725 		    pdu->hdr->hlen);  in nvmf_tcp_handle_h2c_data()
726 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
739 	mbuf_copyto_io(pdu->m, pdu->hdr->pdo, data_len, &cb->io, data_offset);  in nvmf_tcp_handle_h2c_data()
742 	nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_h2c_data()
747 nvmf_tcp_handle_c2h_data(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_handle_c2h_data()  argument
753 	c2h = (const void *)pdu->hdr;  in nvmf_tcp_handle_c2h_data()
760 		 * XXX: Could be PDU sequence error if cccid is for a  in nvmf_tcp_handle_c2h_data()
765 		    offsetof(struct nvme_tcp_c2h_data_hdr, cccid), pdu->m,  in nvmf_tcp_handle_c2h_data()
766 		    pdu->hdr->hlen);  in nvmf_tcp_handle_c2h_data()
767 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
772 	if (pdu->data_digest_mismatch) {  in nvmf_tcp_handle_c2h_data()
777 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
782 	if (data_len != pdu->data_len) {  in nvmf_tcp_handle_c2h_data()
786 		    offsetof(struct nvme_tcp_c2h_data_hdr, datal), pdu->m,  in nvmf_tcp_handle_c2h_data()
787 		    pdu->hdr->hlen);  in nvmf_tcp_handle_c2h_data()
788 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
798 		    pdu->m, pdu->hdr->hlen);  in nvmf_tcp_handle_c2h_data()
799 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
806 		    NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,  in nvmf_tcp_handle_c2h_data()
807 		    pdu->hdr->hlen);  in nvmf_tcp_handle_c2h_data()
808 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
813 	    ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_LAST_PDU) != 0)) {  in nvmf_tcp_handle_c2h_data()
816 		    NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,  in nvmf_tcp_handle_c2h_data()
817 		    pdu->hdr->hlen);  in nvmf_tcp_handle_c2h_data()
818 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
830 	mbuf_copyto_io(pdu->m, pdu->hdr->pdo, data_len, &cb->io, data_offset);  in nvmf_tcp_handle_c2h_data()
834 	if ((pdu->hdr->flags & NVME_TCP_C2H_DATA_FLAGS_SUCCESS) != 0) {  in nvmf_tcp_handle_c2h_data()
847 	nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_c2h_data()
942 nvmf_tcp_handle_r2t(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_handle_r2t()  argument
948 	r2t = (const void *)pdu->hdr;  in nvmf_tcp_handle_r2t()
956 		    offsetof(struct nvme_tcp_r2t_hdr, cccid), pdu->m,  in nvmf_tcp_handle_r2t()
957 		    pdu->hdr->hlen);  in nvmf_tcp_handle_r2t()
958 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_r2t()
966 		    NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR, 0, pdu->m,  in nvmf_tcp_handle_r2t()
967 		    pdu->hdr->hlen);  in nvmf_tcp_handle_r2t()
968 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_r2t()
981 		    pdu->m, pdu->hdr->hlen);  in nvmf_tcp_handle_r2t()
982 		nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_r2t()
1012 	nvmf_tcp_free_pdu(pdu);  in nvmf_tcp_handle_r2t()
1050     const struct nvme_tcp_common_pdu_hdr *ch, struct nvmf_tcp_rxpdu *pdu)  in nvmf_tcp_dispatch_pdu()  argument
1052 	/* Ensure the PDU header is contiguous. */  in nvmf_tcp_dispatch_pdu()
1053 	pdu->m = pullup_pdu_hdr(pdu->m, ch->hlen);  in nvmf_tcp_dispatch_pdu()
1054 	pdu->hdr = mtod(pdu->m, const void *);  in nvmf_tcp_dispatch_pdu()
1062 		return (nvmf_tcp_handle_term_req(pdu));  in nvmf_tcp_dispatch_pdu()
1064 		return (nvmf_tcp_save_command_capsule(qp, pdu));  in nvmf_tcp_dispatch_pdu()
1066 		return (nvmf_tcp_save_response_capsule(qp, pdu));  in nvmf_tcp_dispatch_pdu()
1068 		return (nvmf_tcp_handle_h2c_data(qp, pdu));  in nvmf_tcp_dispatch_pdu()
1070 		return (nvmf_tcp_handle_c2h_data(qp, pdu));  in nvmf_tcp_dispatch_pdu()
1072 		return (nvmf_tcp_handle_r2t(qp, pdu));  in nvmf_tcp_dispatch_pdu()
1081 	struct nvmf_tcp_rxpdu pdu;  in nvmf_tcp_receive()  local
1180 		pdu.m = m;  in nvmf_tcp_receive()
1182 		pdu.hdr = &ch;  in nvmf_tcp_receive()
1183 		error = nvmf_tcp_validate_pdu(qp, &pdu);  in nvmf_tcp_receive()
1185 			m_freem(pdu.m);  in nvmf_tcp_receive()
1187 			error = nvmf_tcp_dispatch_pdu(qp, &ch, &pdu);  in nvmf_tcp_receive()
1323 			/* Next PDU to send. */  in nvmf_tcp_send()
1332 			/* Convert a capsule into a PDU. */  in nvmf_tcp_send()