138 zbuf_free(struct zbuf *zb)  in zbuf_free()  argument
142 	for (i = 0; i < zb->zb_numpages; i++) {  in zbuf_free()
143 		if (zb->zb_pages[i] != NULL)  in zbuf_free()
144 			zbuf_sfbuf_free(zb->zb_pages[i]);  in zbuf_free()
146 	free(zb->zb_pages, M_BPF);  in zbuf_free()
147 	free(zb, M_BPF);  in zbuf_free()
180 	struct zbuf *zb;  in zbuf_setup()  local
208 	zb = malloc(sizeof(*zb), M_BPF, M_ZERO | M_WAITOK);  in zbuf_setup()
209 	zb->zb_uaddr = uaddr;  in zbuf_setup()
210 	zb->zb_size = len;  in zbuf_setup()
211 	zb->zb_numpages = len / PAGE_SIZE;  in zbuf_setup()
212 	zb->zb_pages = malloc(sizeof(struct sf_buf *) *  in zbuf_setup()
213 	    zb->zb_numpages, M_BPF, M_ZERO | M_WAITOK);  in zbuf_setup()
215 	for (i = 0; i < zb->zb_numpages; i++) {  in zbuf_setup()
216 		zb->zb_pages[i] = zbuf_sfbuf_get(map,  in zbuf_setup()
218 		if (zb->zb_pages[i] == NULL) {  in zbuf_setup()
223 	zb->zb_header =  in zbuf_setup()
224 	    (struct bpf_zbuf_header *)sf_buf_kva(zb->zb_pages[0]);  in zbuf_setup()
225 	bzero(zb->zb_header, sizeof(*zb->zb_header));  in zbuf_setup()
226 	*zbp = zb;  in zbuf_setup()
230 	zbuf_free(zb);  in zbuf_setup()
244 	struct zbuf *zb;  in bpf_zerocopy_append_bytes()  local
251 	zb = (struct zbuf *)buf;  in bpf_zerocopy_append_bytes()
253 	KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,  in bpf_zerocopy_append_bytes()
264 		KASSERT(page < zb->zb_numpages, ("bpf_zerocopy_append_bytes:"  in bpf_zerocopy_append_bytes()
265 		   " page overflow (%d p %d np)\n", page, zb->zb_numpages));  in bpf_zerocopy_append_bytes()
268 		bcopy(src_bytes, ((u_char *)sf_buf_kva(zb->zb_pages[page])) +  in bpf_zerocopy_append_bytes()
296 	struct zbuf *zb;  in bpf_zerocopy_append_mbuf()  local
303 	zb = (struct zbuf *)buf;  in bpf_zerocopy_append_mbuf()
305 	KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,  in bpf_zerocopy_append_mbuf()
320 		KASSERT(page < zb->zb_numpages,  in bpf_zerocopy_append_mbuf()
322 		    "np)\n", page, zb->zb_numpages));  in bpf_zerocopy_append_mbuf()
329 		    ((u_char *)sf_buf_kva(zb->zb_pages[page])) + poffset,  in bpf_zerocopy_append_mbuf()
357 	struct zbuf *zb;  in bpf_zerocopy_buffull()  local
362 	zb = (struct zbuf *)d->bd_sbuf;  in bpf_zerocopy_buffull()
363 	KASSERT(zb != NULL, ("bpf_zerocopy_buffull: zb == NULL"));  in bpf_zerocopy_buffull()
365 	if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {  in bpf_zerocopy_buffull()
366 		zb->zb_flags |= ZBUF_FLAG_ASSIGNED;  in bpf_zerocopy_buffull()
367 		zb->zb_header->bzh_kernel_len = d->bd_slen;  in bpf_zerocopy_buffull()
368 		atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);  in bpf_zerocopy_buffull()
384 	struct zbuf *zb;  in bpf_zerocopy_bufheld()  local
389 	zb = (struct zbuf *)d->bd_hbuf;  in bpf_zerocopy_bufheld()
390 	KASSERT(zb != NULL, ("bpf_zerocopy_bufheld: zb == NULL"));  in bpf_zerocopy_bufheld()
392 	if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {  in bpf_zerocopy_bufheld()
393 		zb->zb_flags |= ZBUF_FLAG_ASSIGNED;  in bpf_zerocopy_bufheld()
394 		zb->zb_header->bzh_kernel_len = d->bd_hlen;  in bpf_zerocopy_bufheld()
395 		atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);  in bpf_zerocopy_bufheld()
407 	struct zbuf *zb;  in bpf_zerocopy_buf_reclaimed()  local
414 	zb = (struct zbuf *)d->bd_fbuf;  in bpf_zerocopy_buf_reclaimed()
415 	zb->zb_flags &= ~ZBUF_FLAG_ASSIGNED;  in bpf_zerocopy_buf_reclaimed()
427 	struct zbuf *zb;  in bpf_zerocopy_canfreebuf()  local
432 	zb = (struct zbuf *)d->bd_hbuf;  in bpf_zerocopy_canfreebuf()
433 	if (zb == NULL)  in bpf_zerocopy_canfreebuf()
435 	if (zb->zb_header->bzh_kernel_gen ==  in bpf_zerocopy_canfreebuf()
436 	    atomic_load_acq_int(&zb->zb_header->bzh_user_gen))  in bpf_zerocopy_canfreebuf()
450 	struct zbuf *zb;  in bpf_zerocopy_canwritebuf()  local
455 	zb = (struct zbuf *)d->bd_sbuf;  in bpf_zerocopy_canwritebuf()
456 	KASSERT(zb != NULL, ("bpf_zerocopy_canwritebuf: bd_sbuf NULL"));  in bpf_zerocopy_canwritebuf()
458 	if (zb->zb_flags & ZBUF_FLAG_ASSIGNED)  in bpf_zerocopy_canwritebuf()
469 	struct zbuf *zb;  in bpf_zerocopy_free()  local
474 	zb = (struct zbuf *)d->bd_sbuf;  in bpf_zerocopy_free()
475 	if (zb != NULL)  in bpf_zerocopy_free()
476 		zbuf_free(zb);  in bpf_zerocopy_free()
477 	zb = (struct zbuf *)d->bd_hbuf;  in bpf_zerocopy_free()
478 	if (zb != NULL)  in bpf_zerocopy_free()
479 		zbuf_free(zb);  in bpf_zerocopy_free()
480 	zb = (struct zbuf *)d->bd_fbuf;  in bpf_zerocopy_free()
481 	if (zb != NULL)  in bpf_zerocopy_free()
482 		zbuf_free(zb);  in bpf_zerocopy_free()