85     struct ddp_rcv_buffer *drb);
87     struct sge_wrq *wrq, int tid, struct ddp_rcv_buffer *drb);
181 free_ddp_rcv_buffer(struct toepcb *toep, struct ddp_rcv_buffer *drb)  in free_ddp_rcv_buffer()  argument
183 	t4_free_page_pods(&drb->prsv);  in free_ddp_rcv_buffer()
184 	free(drb->buf, M_CXGBE);  in free_ddp_rcv_buffer()
185 	free(drb, M_CXGBE);  in free_ddp_rcv_buffer()
191 recycle_ddp_rcv_buffer(struct toepcb *toep, struct ddp_rcv_buffer *drb)  in recycle_ddp_rcv_buffer()  argument
196 		TAILQ_INSERT_HEAD(&toep->ddp.cached_buffers, drb, link);  in recycle_ddp_rcv_buffer()
201 		free_ddp_rcv_buffer(toep, drb);  in recycle_ddp_rcv_buffer()
208 	struct ddp_rcv_buffer *drb;  in alloc_cached_ddp_rcv_buffer()  local
212 		drb = TAILQ_FIRST(&toep->ddp.cached_buffers);  in alloc_cached_ddp_rcv_buffer()
213 		TAILQ_REMOVE(&toep->ddp.cached_buffers, drb, link);  in alloc_cached_ddp_rcv_buffer()
217 		drb = NULL;  in alloc_cached_ddp_rcv_buffer()
219 	return (drb);  in alloc_cached_ddp_rcv_buffer()
227 	struct ddp_rcv_buffer *drb;  in alloc_ddp_rcv_buffer()  local
230 	drb = malloc(sizeof(*drb), M_CXGBE, how | M_ZERO);  in alloc_ddp_rcv_buffer()
231 	if (drb == NULL)  in alloc_ddp_rcv_buffer()
234 	drb->buf = contigmalloc(t4_ddp_rcvbuf_len, M_CXGBE, how, 0, ~0,  in alloc_ddp_rcv_buffer()
236 	if (drb->buf == NULL) {  in alloc_ddp_rcv_buffer()
237 		free(drb, M_CXGBE);  in alloc_ddp_rcv_buffer()
240 	drb->len = t4_ddp_rcvbuf_len;  in alloc_ddp_rcv_buffer()
241 	drb->refs = 1;  in alloc_ddp_rcv_buffer()
243 	error = t4_alloc_page_pods_for_rcvbuf(&td->pr, drb);  in alloc_ddp_rcv_buffer()
245 		free(drb->buf, M_CXGBE);  in alloc_ddp_rcv_buffer()
246 		free(drb, M_CXGBE);  in alloc_ddp_rcv_buffer()
250 	error = t4_write_page_pods_for_rcvbuf(sc, toep->ctrlq, toep->tid, drb);  in alloc_ddp_rcv_buffer()
252 		t4_free_page_pods(&drb->prsv);  in alloc_ddp_rcv_buffer()
253 		free(drb->buf, M_CXGBE);  in alloc_ddp_rcv_buffer()
254 		free(drb, M_CXGBE);  in alloc_ddp_rcv_buffer()
260 	return (drb);  in alloc_ddp_rcv_buffer()
267 		if (db->drb != NULL)  in free_ddp_buffer()
268 			free_ddp_rcv_buffer(toep, db->drb);  in free_ddp_buffer()
270 		db->drb = NULL;  in free_ddp_buffer()
318 	struct ddp_rcv_buffer *drb;  in release_ddp_resources()  local
338 		while ((drb = TAILQ_FIRST(&toep->ddp.cached_buffers)) != NULL) {  in release_ddp_resources()
339 			TAILQ_REMOVE(&toep->ddp.cached_buffers, drb, link);  in release_ddp_resources()
340 			free_ddp_rcv_buffer(toep, drb);  in release_ddp_resources()
359 			MPASS(toep->ddp.db[i].drb == NULL);  in ddp_assert_empty()
374 	struct ddp_rcv_buffer *drb;  in complete_ddp_buffer()  local
384 				KASSERT(toep->ddp.db[db_idx ^ 1].drb == NULL,  in complete_ddp_buffer()
405 		drb = db->drb;  in complete_ddp_buffer()
406 		if (atomic_fetchadd_int(&drb->refs, -1) == 1)  in complete_ddp_buffer()
407 			recycle_ddp_rcv_buffer(toep, drb);  in complete_ddp_buffer()
408 		db->drb = NULL;  in complete_ddp_buffer()
424 	struct ddp_rcv_buffer *drb = m->m_ext.ext_arg2;  in ddp_rcv_mbuf_done()  local
426 	recycle_ddp_rcv_buffer(toep, drb);  in ddp_rcv_mbuf_done()
435 	struct ddp_rcv_buffer *drb;  in queue_ddp_rcvbuf_mbuf()  local
446 	drb = db->drb;  in queue_ddp_rcvbuf_mbuf()
447 	m_extaddref(m, (char *)drb->buf + db->placed, len, &drb->refs,  in queue_ddp_rcvbuf_mbuf()
448 	    ddp_rcv_mbuf_done, toep, drb);  in queue_ddp_rcvbuf_mbuf()
497 			if (placed > db->drb->len - db->placed)  in insert_ddp_data()
498 				placed = db->drb->len - db->placed;  in insert_ddp_data()
763 queue_ddp_rcvbuf(struct toepcb *toep, struct ddp_rcv_buffer *drb)  in queue_ddp_rcvbuf()  argument
778 	if (toep->ddp.db[0].drb == NULL) {  in queue_ddp_rcvbuf()
781 		MPASS(toep->ddp.db[1].drb == NULL);  in queue_ddp_rcvbuf()
822 	wr = mk_update_tcb_for_ddp(sc, toep, db_idx, &drb->prsv, 0, drb->len,  in queue_ddp_rcvbuf()
825 		recycle_ddp_rcv_buffer(toep, drb);  in queue_ddp_rcvbuf()
839 	drb->refs = 1;  in queue_ddp_rcvbuf()
844 	db->drb = drb;  in queue_ddp_rcvbuf()
866 	struct ddp_rcv_buffer *drb;  in handle_ddp_data_rcvbuf()  local
950 		KASSERT(db->placed < db->drb->len,  in handle_ddp_data_rcvbuf()
954 		drb = alloc_cached_ddp_rcv_buffer(toep);  in handle_ddp_data_rcvbuf()
955 		if (drb == NULL)  in handle_ddp_data_rcvbuf()
956 			drb = alloc_ddp_rcv_buffer(toep, M_NOWAIT);  in handle_ddp_data_rcvbuf()
957 		if (drb == NULL)  in handle_ddp_data_rcvbuf()
960 			if (!queue_ddp_rcvbuf(toep, drb)) {  in handle_ddp_data_rcvbuf()
1125 			if (placed > db->drb->len - db->placed)  in handle_ddp_close()
1126 				placed = db->drb->len - db->placed;  in handle_ddp_close()
1515     struct ddp_rcv_buffer *drb)  in t4_alloc_page_pods_for_rcvbuf()  argument
1517 	struct ppod_reservation *prsv = &drb->prsv;  in t4_alloc_page_pods_for_rcvbuf()
1522 	return (t4_alloc_page_pods_for_buf(pr, (vm_offset_t)drb->buf, drb->len,  in t4_alloc_page_pods_for_rcvbuf()
1700     struct ddp_rcv_buffer *drb)  in t4_write_page_pods_for_rcvbuf()  argument
1709 	struct ppod_reservation *prsv = &drb->prsv;  in t4_write_page_pods_for_rcvbuf()
1722 	offset = (uintptr_t)drb->buf & PAGE_MASK;  in t4_write_page_pods_for_rcvbuf()
1724 	pva = trunc_page((uintptr_t)drb->buf);  in t4_write_page_pods_for_rcvbuf()
1725 	end_pva = trunc_page((uintptr_t)drb->buf + drb->len - 1);  in t4_write_page_pods_for_rcvbuf()
1752 			ppod->len_offset = htobe64(V_PPOD_LEN(drb->len) |  in t4_write_page_pods_for_rcvbuf()
2861 	struct ddp_rcv_buffer *drb;  in ddp_rcvbuf_requeue()  local
2879 	drb = alloc_cached_ddp_rcv_buffer(toep);  in ddp_rcvbuf_requeue()
2882 	if (drb == NULL) {  in ddp_rcvbuf_requeue()
2883 		drb = alloc_ddp_rcv_buffer(toep, M_WAITOK);  in ddp_rcvbuf_requeue()
2884 		if (drb == NULL) {  in ddp_rcvbuf_requeue()
2894 		recycle_ddp_rcv_buffer(toep, drb);  in ddp_rcvbuf_requeue()
2902 		recycle_ddp_rcv_buffer(toep, drb);  in ddp_rcvbuf_requeue()
2909 		recycle_ddp_rcv_buffer(toep, drb);  in ddp_rcvbuf_requeue()
2914 	if (!queue_ddp_rcvbuf(toep, drb)) {  in ddp_rcvbuf_requeue()