179 	struct nvmf_tcp_command_buffer *cb;  in tcp_alloc_command_buffer()  local
181 	cb = malloc(sizeof(*cb), M_NVMF_TCP, M_WAITOK);  in tcp_alloc_command_buffer()
182 	cb->qp = qp;  in tcp_alloc_command_buffer()
183 	cb->io = *io;  in tcp_alloc_command_buffer()
184 	cb->data_offset = data_offset;  in tcp_alloc_command_buffer()
185 	cb->data_len = data_len;  in tcp_alloc_command_buffer()
186 	cb->data_xfered = 0;  in tcp_alloc_command_buffer()
187 	refcount_init(&cb->refs, 1);  in tcp_alloc_command_buffer()
188 	cb->error = 0;  in tcp_alloc_command_buffer()
189 	cb->cid = cid;  in tcp_alloc_command_buffer()
190 	cb->ttag = 0;  in tcp_alloc_command_buffer()
191 	cb->tc = NULL;  in tcp_alloc_command_buffer()
193 	return (cb);  in tcp_alloc_command_buffer()
197 tcp_hold_command_buffer(struct nvmf_tcp_command_buffer *cb)  in tcp_hold_command_buffer()  argument
199 	refcount_acquire(&cb->refs);  in tcp_hold_command_buffer()
203 tcp_free_command_buffer(struct nvmf_tcp_command_buffer *cb)  in tcp_free_command_buffer()  argument
205 	nvmf_complete_io_request(&cb->io, cb->data_xfered, cb->error);  in tcp_free_command_buffer()
206 	if (cb->tc != NULL)  in tcp_free_command_buffer()
207 		tcp_release_capsule(cb->tc);  in tcp_free_command_buffer()
208 	free(cb, M_NVMF_TCP);  in tcp_free_command_buffer()
212 tcp_release_command_buffer(struct nvmf_tcp_command_buffer *cb)  in tcp_release_command_buffer()  argument
214 	if (refcount_release(&cb->refs))  in tcp_release_command_buffer()
215 		tcp_free_command_buffer(cb);  in tcp_release_command_buffer()
220     struct nvmf_tcp_command_buffer *cb)  in tcp_add_command_buffer()  argument
223 	TAILQ_INSERT_HEAD(&list->head, cb, link);  in tcp_add_command_buffer()
230 	struct nvmf_tcp_command_buffer *cb;  in tcp_find_command_buffer()  local
233 	TAILQ_FOREACH(cb, &list->head, link) {  in tcp_find_command_buffer()
234 		if (cb->cid == cid && cb->ttag == ttag)  in tcp_find_command_buffer()
235 			return (cb);  in tcp_find_command_buffer()
242     struct nvmf_tcp_command_buffer *cb)  in tcp_remove_command_buffer()  argument
245 	TAILQ_REMOVE(&list->head, cb, link);  in tcp_remove_command_buffer()
252 	struct nvmf_tcp_command_buffer *cb;  in tcp_purge_command_buffer()  local
255 	cb = tcp_find_command_buffer(list, cid, ttag);  in tcp_purge_command_buffer()
256 	if (cb != NULL) {  in tcp_purge_command_buffer()
257 		tcp_remove_command_buffer(list, cb);  in tcp_purge_command_buffer()
259 		tcp_release_command_buffer(cb);  in tcp_purge_command_buffer()
508 	struct nvmf_tcp_command_buffer *cb;  in nvmf_tcp_next_r2t()  local
513 	TAILQ_FOREACH(cb, &qp->rx_buffers.head, link) {  in nvmf_tcp_next_r2t()
515 		if (cb->tc->active_r2ts > qp->maxr2t)  in nvmf_tcp_next_r2t()
518 		cb->tc->pending_r2ts--;  in nvmf_tcp_next_r2t()
520 		TAILQ_REMOVE(&qp->rx_buffers.head, cb, link);  in nvmf_tcp_next_r2t()
521 		return (cb);  in nvmf_tcp_next_r2t()
529     struct nvmf_tcp_command_buffer *cb)  in nvmf_tcp_allocate_ttag()  argument
550 	cb->tc->active_r2ts++;  in nvmf_tcp_allocate_ttag()
552 	qp->open_ttags[ttag] = cb;  in nvmf_tcp_allocate_ttag()
558 	cb->ttag = ttag;  in nvmf_tcp_allocate_ttag()
587     struct nvmf_tcp_command_buffer *cb)  in nvmf_tcp_send_next_r2t()  argument
592 	MPASS(qp->open_ttags[cb->ttag] == cb);  in nvmf_tcp_send_next_r2t()
595 	qp->open_ttags[cb->ttag] = NULL;  in nvmf_tcp_send_next_r2t()
597 	cb->tc->active_r2ts--;  in nvmf_tcp_send_next_r2t()
640 	struct nvmf_tcp_command_buffer *cb;  in nvmf_tcp_handle_h2c_data()  local
668 	cb = qp->open_ttags[ttag];  in nvmf_tcp_handle_h2c_data()
669 	if (cb == NULL) {  in nvmf_tcp_handle_h2c_data()
678 	MPASS(cb->ttag == ttag);  in nvmf_tcp_handle_h2c_data()
682 		nvmf_tcp_send_next_r2t(qp, cb);  in nvmf_tcp_handle_h2c_data()
683 		cb->error = EINTEGRITY;  in nvmf_tcp_handle_h2c_data()
684 		tcp_release_command_buffer(cb);  in nvmf_tcp_handle_h2c_data()
701 	if (data_offset < cb->data_offset ||  in nvmf_tcp_handle_h2c_data()
702 	    data_offset + data_len > cb->data_offset + cb->data_len) {  in nvmf_tcp_handle_h2c_data()
711 	if (data_offset != cb->data_offset + cb->data_xfered) {  in nvmf_tcp_handle_h2c_data()
720 	if ((cb->data_xfered + data_len == cb->data_len) !=  in nvmf_tcp_handle_h2c_data()
730 	cb->data_xfered += data_len;  in nvmf_tcp_handle_h2c_data()
731 	data_offset -= cb->data_offset;  in nvmf_tcp_handle_h2c_data()
732 	if (cb->data_xfered == cb->data_len) {  in nvmf_tcp_handle_h2c_data()
733 		nvmf_tcp_send_next_r2t(qp, cb);  in nvmf_tcp_handle_h2c_data()
735 		tcp_hold_command_buffer(cb);  in nvmf_tcp_handle_h2c_data()
739 	mbuf_copyto_io(pdu->m, pdu->hdr->pdo, data_len, &cb->io, data_offset);  in nvmf_tcp_handle_h2c_data()
741 	tcp_release_command_buffer(cb);  in nvmf_tcp_handle_h2c_data()
750 	struct nvmf_tcp_command_buffer *cb;  in nvmf_tcp_handle_c2h_data()  local
756 	cb = tcp_find_command_buffer(&qp->rx_buffers, c2h->cccid, 0);  in nvmf_tcp_handle_c2h_data()
757 	if (cb == NULL) {  in nvmf_tcp_handle_c2h_data()
773 		cb->error = EINTEGRITY;  in nvmf_tcp_handle_c2h_data()
774 		tcp_remove_command_buffer(&qp->rx_buffers, cb);  in nvmf_tcp_handle_c2h_data()
776 		tcp_release_command_buffer(cb);  in nvmf_tcp_handle_c2h_data()
793 	if (data_offset < cb->data_offset ||  in nvmf_tcp_handle_c2h_data()
794 	    data_offset + data_len > cb->data_offset + cb->data_len) {  in nvmf_tcp_handle_c2h_data()
803 	if (data_offset != cb->data_offset + cb->data_xfered) {  in nvmf_tcp_handle_c2h_data()
812 	if ((cb->data_xfered + data_len == cb->data_len) !=  in nvmf_tcp_handle_c2h_data()
822 	cb->data_xfered += data_len;  in nvmf_tcp_handle_c2h_data()
823 	data_offset -= cb->data_offset;  in nvmf_tcp_handle_c2h_data()
824 	if (cb->data_xfered == cb->data_len)  in nvmf_tcp_handle_c2h_data()
825 		tcp_remove_command_buffer(&qp->rx_buffers, cb);  in nvmf_tcp_handle_c2h_data()
827 		tcp_hold_command_buffer(cb);  in nvmf_tcp_handle_c2h_data()
830 	mbuf_copyto_io(pdu->m, pdu->hdr->pdo, data_len, &cb->io, data_offset);  in nvmf_tcp_handle_c2h_data()
832 	tcp_release_command_buffer(cb);  in nvmf_tcp_handle_c2h_data()
855 	struct nvmf_tcp_command_buffer *cb = m->m_ext.ext_arg1;  in nvmf_tcp_mbuf_done()  local
857 	tcp_free_command_buffer(cb);  in nvmf_tcp_mbuf_done()
863 	struct nvmf_tcp_command_buffer *cb = arg;  in nvmf_tcp_mbuf()  local
868 	m_extaddref(m, data, len, &cb->refs, nvmf_tcp_mbuf_done, cb, NULL);  in nvmf_tcp_mbuf()
876 	struct nvmf_tcp_command_buffer *cb = m->m_ext.ext_arg1;  in nvmf_tcp_free_mext_pg()  local
879 	tcp_release_command_buffer(cb);  in nvmf_tcp_free_mext_pg()
885 	struct nvmf_tcp_command_buffer *cb = arg;  in nvmf_tcp_mext_pg()  local
889 	m->m_ext.ext_arg1 = cb;  in nvmf_tcp_mext_pg()
890 	tcp_hold_command_buffer(cb);  in nvmf_tcp_mext_pg()
905 nvmf_tcp_command_buffer_mbuf(struct nvmf_tcp_command_buffer *cb,  in nvmf_tcp_command_buffer_mbuf()  argument
912 	m = memdesc_alloc_ext_mbufs(&cb->io.io_mem, nvmf_tcp_mbuf,  in nvmf_tcp_command_buffer_mbuf()
913 	    nvmf_tcp_mext_pg, cb, M_WAITOK, data_offset, data_len, &len,  in nvmf_tcp_command_buffer_mbuf()
945 	struct nvmf_tcp_command_buffer *cb;  in nvmf_tcp_handle_r2t()  local
951 	cb = tcp_find_command_buffer(&qp->tx_buffers, r2t->cccid, 0);  in nvmf_tcp_handle_r2t()
952 	if (cb == NULL) {  in nvmf_tcp_handle_r2t()
963 	if (data_offset != cb->data_xfered) {  in nvmf_tcp_handle_r2t()
977 	if (data_offset + data_len > cb->data_len) {  in nvmf_tcp_handle_r2t()
986 	cb->data_xfered += data_len;  in nvmf_tcp_handle_r2t()
987 	if (cb->data_xfered == cb->data_len)  in nvmf_tcp_handle_r2t()
988 		tcp_remove_command_buffer(&qp->tx_buffers, cb);  in nvmf_tcp_handle_r2t()
990 		tcp_hold_command_buffer(cb);  in nvmf_tcp_handle_r2t()
1002 		m = nvmf_tcp_command_buffer_mbuf(cb, data_offset, todo, &sent,  in nvmf_tcp_handle_r2t()
1011 	tcp_release_command_buffer(cb);  in nvmf_tcp_handle_r2t()
1222 	struct nvmf_tcp_command_buffer *cb;  in tcp_command_pdu()  local
1229 	cb = NULL;  in tcp_command_pdu()
1233 		cb = tcp_alloc_command_buffer(qp, &nc->nc_data, 0,  in tcp_command_pdu()
1238 			m = nvmf_tcp_command_buffer_mbuf(cb, 0,  in tcp_command_pdu()
1240 			cb->data_xfered = nc->nc_data.io_len;  in tcp_command_pdu()
1241 			tcp_release_command_buffer(cb);  in tcp_command_pdu()
1244 			tcp_add_command_buffer(&qp->tx_buffers, cb);  in tcp_command_pdu()
1248 			tcp_add_command_buffer(&qp->rx_buffers, cb);  in tcp_command_pdu()
1530 	struct nvmf_tcp_command_buffer *ncb, *cb;  in tcp_free_qpair()  local
1566 			cb = qp->open_ttags[i];  in tcp_free_qpair()
1567 			if (cb != NULL) {  in tcp_free_qpair()
1568 				cb->tc->active_r2ts--;  in tcp_free_qpair()
1569 				cb->error = ECONNABORTED;  in tcp_free_qpair()
1570 				tcp_release_command_buffer(cb);  in tcp_free_qpair()
1577 	TAILQ_FOREACH_SAFE(cb, &qp->rx_buffers.head, link, ncb) {  in tcp_free_qpair()
1578 		tcp_remove_command_buffer(&qp->rx_buffers, cb);  in tcp_free_qpair()
1581 		if (cb->tc != NULL)  in tcp_free_qpair()
1582 			cb->tc->pending_r2ts--;  in tcp_free_qpair()
1584 		cb->error = ECONNABORTED;  in tcp_free_qpair()
1585 		tcp_release_command_buffer(cb);  in tcp_free_qpair()
1591 	TAILQ_FOREACH_SAFE(cb, &qp->tx_buffers.head, link, ncb) {  in tcp_free_qpair()
1592 		tcp_remove_command_buffer(&qp->tx_buffers, cb);  in tcp_free_qpair()
1594 		cb->error = ECONNABORTED;  in tcp_free_qpair()
1595 		tcp_release_command_buffer(cb);  in tcp_free_qpair()
1707 	struct nvmf_tcp_command_buffer *cb;  in tcp_receive_r2t_data()  local
1709 	cb = tcp_alloc_command_buffer(qp, io, data_offset, io->io_len,  in tcp_receive_r2t_data()
1712 	cb->tc = tc;  in tcp_receive_r2t_data()
1726 		TAILQ_INSERT_TAIL(&qp->rx_buffers.head, cb, link);  in tcp_receive_r2t_data()
1731 	nvmf_tcp_allocate_ttag(qp, cb);  in tcp_receive_r2t_data()
1734 	tcp_send_r2t(qp, nc->nc_sqe.cid, cb->ttag, data_offset, io->io_len);  in tcp_receive_r2t_data()