xref: /freebsd/sys/net/bpf.c (revision b3aaa0cc21c63d388230c7ef2a80abd631ff20d5)
1 /*-
2  * Copyright (c) 1990, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from the Stanford/CMU enet packet filter,
6  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
7  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
8  * Berkeley Laboratory.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *      @(#)bpf.c	8.4 (Berkeley) 1/9/95
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include "opt_bpf.h"
41 #include "opt_mac.h"
42 #include "opt_netgraph.h"
43 
44 #include <sys/types.h>
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/conf.h>
48 #include <sys/fcntl.h>
49 #include <sys/malloc.h>
50 #include <sys/mbuf.h>
51 #include <sys/time.h>
52 #include <sys/priv.h>
53 #include <sys/proc.h>
54 #include <sys/signalvar.h>
55 #include <sys/filio.h>
56 #include <sys/sockio.h>
57 #include <sys/ttycom.h>
58 #include <sys/uio.h>
59 #include <sys/vimage.h>
60 
61 #include <sys/event.h>
62 #include <sys/file.h>
63 #include <sys/poll.h>
64 #include <sys/proc.h>
65 
66 #include <sys/socket.h>
67 
68 #include <net/if.h>
69 #include <net/bpf.h>
70 #include <net/bpf_buffer.h>
71 #ifdef BPF_JITTER
72 #include <net/bpf_jitter.h>
73 #endif
74 #include <net/bpf_zerocopy.h>
75 #include <net/bpfdesc.h>
76 
77 #include <netinet/in.h>
78 #include <netinet/if_ether.h>
79 #include <sys/kernel.h>
80 #include <sys/sysctl.h>
81 
82 #include <net80211/ieee80211_freebsd.h>
83 
84 #include <security/mac/mac_framework.h>
85 
86 MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
87 
88 #if defined(DEV_BPF) || defined(NETGRAPH_BPF)
89 
90 #define PRINET  26			/* interruptible */
91 
92 /*
93  * bpf_iflist is a list of BPF interface structures, each corresponding to a
94  * specific DLT.  The same network interface might have several BPF interface
95  * structures registered by different layers in the stack (i.e., 802.11
96  * frames, ethernet frames, etc).
97  */
98 static LIST_HEAD(, bpf_if)	bpf_iflist;
99 static struct mtx	bpf_mtx;		/* bpf global lock */
100 static int		bpf_bpfd_cnt;
101 
102 static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
103 static void	bpf_detachd(struct bpf_d *);
104 static void	bpf_freed(struct bpf_d *);
105 static int	bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **,
106 		    struct sockaddr *, int *, struct bpf_insn *);
107 static int	bpf_setif(struct bpf_d *, struct ifreq *);
108 static void	bpf_timed_out(void *);
109 static __inline void
110 		bpf_wakeup(struct bpf_d *);
111 static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
112 		    void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
113 		    struct timeval *);
114 static void	reset_d(struct bpf_d *);
115 static int	 bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
116 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
117 static int	bpf_setdlt(struct bpf_d *, u_int);
118 static void	filt_bpfdetach(struct knote *);
119 static int	filt_bpfread(struct knote *, long);
120 static void	bpf_drvinit(void *);
121 static int	bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
122 
123 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl");
124 int bpf_maxinsns = BPF_MAXINSNS;
125 SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
126     &bpf_maxinsns, 0, "Maximum bpf program instructions");
127 static int bpf_zerocopy_enable = 0;
128 SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
129     &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
130 SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_RW,
131     bpf_stats_sysctl, "bpf statistics portal");
132 
133 static	d_open_t	bpfopen;
134 static	d_read_t	bpfread;
135 static	d_write_t	bpfwrite;
136 static	d_ioctl_t	bpfioctl;
137 static	d_poll_t	bpfpoll;
138 static	d_kqfilter_t	bpfkqfilter;
139 
140 static struct cdevsw bpf_cdevsw = {
141 	.d_version =	D_VERSION,
142 	.d_open =	bpfopen,
143 	.d_read =	bpfread,
144 	.d_write =	bpfwrite,
145 	.d_ioctl =	bpfioctl,
146 	.d_poll =	bpfpoll,
147 	.d_name =	"bpf",
148 	.d_kqfilter =	bpfkqfilter,
149 };
150 
151 static struct filterops bpfread_filtops =
152 	{ 1, NULL, filt_bpfdetach, filt_bpfread };
153 
154 /*
155  * Wrapper functions for various buffering methods.  If the set of buffer
156  * modes expands, we will probably want to introduce a switch data structure
157  * similar to protosw, et.
158  */
159 static void
160 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
161     u_int len)
162 {
163 
164 	BPFD_LOCK_ASSERT(d);
165 
166 	switch (d->bd_bufmode) {
167 	case BPF_BUFMODE_BUFFER:
168 		return (bpf_buffer_append_bytes(d, buf, offset, src, len));
169 
170 	case BPF_BUFMODE_ZBUF:
171 		d->bd_zcopy++;
172 		return (bpf_zerocopy_append_bytes(d, buf, offset, src, len));
173 
174 	default:
175 		panic("bpf_buf_append_bytes");
176 	}
177 }
178 
179 static void
180 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
181     u_int len)
182 {
183 
184 	BPFD_LOCK_ASSERT(d);
185 
186 	switch (d->bd_bufmode) {
187 	case BPF_BUFMODE_BUFFER:
188 		return (bpf_buffer_append_mbuf(d, buf, offset, src, len));
189 
190 	case BPF_BUFMODE_ZBUF:
191 		d->bd_zcopy++;
192 		return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len));
193 
194 	default:
195 		panic("bpf_buf_append_mbuf");
196 	}
197 }
198 
199 /*
200  * This function gets called when the free buffer is re-assigned.
201  */
202 static void
203 bpf_buf_reclaimed(struct bpf_d *d)
204 {
205 
206 	BPFD_LOCK_ASSERT(d);
207 
208 	switch (d->bd_bufmode) {
209 	case BPF_BUFMODE_BUFFER:
210 		return;
211 
212 	case BPF_BUFMODE_ZBUF:
213 		bpf_zerocopy_buf_reclaimed(d);
214 		return;
215 
216 	default:
217 		panic("bpf_buf_reclaimed");
218 	}
219 }
220 
221 /*
222  * If the buffer mechanism has a way to decide that a held buffer can be made
223  * free, then it is exposed via the bpf_canfreebuf() interface.  (1) is
224  * returned if the buffer can be discarded, (0) is returned if it cannot.
225  */
226 static int
227 bpf_canfreebuf(struct bpf_d *d)
228 {
229 
230 	BPFD_LOCK_ASSERT(d);
231 
232 	switch (d->bd_bufmode) {
233 	case BPF_BUFMODE_ZBUF:
234 		return (bpf_zerocopy_canfreebuf(d));
235 	}
236 	return (0);
237 }
238 
239 /*
240  * Allow the buffer model to indicate that the current store buffer is
241  * immutable, regardless of the appearance of space.  Return (1) if the
242  * buffer is writable, and (0) if not.
243  */
244 static int
245 bpf_canwritebuf(struct bpf_d *d)
246 {
247 
248 	BPFD_LOCK_ASSERT(d);
249 
250 	switch (d->bd_bufmode) {
251 	case BPF_BUFMODE_ZBUF:
252 		return (bpf_zerocopy_canwritebuf(d));
253 	}
254 	return (1);
255 }
256 
257 /*
258  * Notify buffer model that an attempt to write to the store buffer has
259  * resulted in a dropped packet, in which case the buffer may be considered
260  * full.
261  */
262 static void
263 bpf_buffull(struct bpf_d *d)
264 {
265 
266 	BPFD_LOCK_ASSERT(d);
267 
268 	switch (d->bd_bufmode) {
269 	case BPF_BUFMODE_ZBUF:
270 		bpf_zerocopy_buffull(d);
271 		break;
272 	}
273 }
274 
275 /*
276  * Notify the buffer model that a buffer has moved into the hold position.
277  */
278 void
279 bpf_bufheld(struct bpf_d *d)
280 {
281 
282 	BPFD_LOCK_ASSERT(d);
283 
284 	switch (d->bd_bufmode) {
285 	case BPF_BUFMODE_ZBUF:
286 		bpf_zerocopy_bufheld(d);
287 		break;
288 	}
289 }
290 
291 static void
292 bpf_free(struct bpf_d *d)
293 {
294 
295 	switch (d->bd_bufmode) {
296 	case BPF_BUFMODE_BUFFER:
297 		return (bpf_buffer_free(d));
298 
299 	case BPF_BUFMODE_ZBUF:
300 		return (bpf_zerocopy_free(d));
301 
302 	default:
303 		panic("bpf_buf_free");
304 	}
305 }
306 
307 static int
308 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio)
309 {
310 
311 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
312 		return (EOPNOTSUPP);
313 	return (bpf_buffer_uiomove(d, buf, len, uio));
314 }
315 
316 static int
317 bpf_ioctl_sblen(struct bpf_d *d, u_int *i)
318 {
319 
320 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
321 		return (EOPNOTSUPP);
322 	return (bpf_buffer_ioctl_sblen(d, i));
323 }
324 
325 static int
326 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
327 {
328 
329 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
330 		return (EOPNOTSUPP);
331 	return (bpf_zerocopy_ioctl_getzmax(td, d, i));
332 }
333 
334 static int
335 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
336 {
337 
338 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
339 		return (EOPNOTSUPP);
340 	return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz));
341 }
342 
343 static int
344 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
345 {
346 
347 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
348 		return (EOPNOTSUPP);
349 	return (bpf_zerocopy_ioctl_setzbuf(td, d, bz));
350 }
351 
352 /*
353  * General BPF functions.
354  */
355 static int
356 bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
357     struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter)
358 {
359 	const struct ieee80211_bpf_params *p;
360 	struct ether_header *eh;
361 	struct mbuf *m;
362 	int error;
363 	int len;
364 	int hlen;
365 	int slen;
366 
367 	/*
368 	 * Build a sockaddr based on the data link layer type.
369 	 * We do this at this level because the ethernet header
370 	 * is copied directly into the data field of the sockaddr.
371 	 * In the case of SLIP, there is no header and the packet
372 	 * is forwarded as is.
373 	 * Also, we are careful to leave room at the front of the mbuf
374 	 * for the link level header.
375 	 */
376 	switch (linktype) {
377 
378 	case DLT_SLIP:
379 		sockp->sa_family = AF_INET;
380 		hlen = 0;
381 		break;
382 
383 	case DLT_EN10MB:
384 		sockp->sa_family = AF_UNSPEC;
385 		/* XXX Would MAXLINKHDR be better? */
386 		hlen = ETHER_HDR_LEN;
387 		break;
388 
389 	case DLT_FDDI:
390 		sockp->sa_family = AF_IMPLINK;
391 		hlen = 0;
392 		break;
393 
394 	case DLT_RAW:
395 		sockp->sa_family = AF_UNSPEC;
396 		hlen = 0;
397 		break;
398 
399 	case DLT_NULL:
400 		/*
401 		 * null interface types require a 4 byte pseudo header which
402 		 * corresponds to the address family of the packet.
403 		 */
404 		sockp->sa_family = AF_UNSPEC;
405 		hlen = 4;
406 		break;
407 
408 	case DLT_ATM_RFC1483:
409 		/*
410 		 * en atm driver requires 4-byte atm pseudo header.
411 		 * though it isn't standard, vpi:vci needs to be
412 		 * specified anyway.
413 		 */
414 		sockp->sa_family = AF_UNSPEC;
415 		hlen = 12;	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
416 		break;
417 
418 	case DLT_PPP:
419 		sockp->sa_family = AF_UNSPEC;
420 		hlen = 4;	/* This should match PPP_HDRLEN */
421 		break;
422 
423 	case DLT_IEEE802_11:		/* IEEE 802.11 wireless */
424 		sockp->sa_family = AF_IEEE80211;
425 		hlen = 0;
426 		break;
427 
428 	case DLT_IEEE802_11_RADIO:	/* IEEE 802.11 wireless w/ phy params */
429 		sockp->sa_family = AF_IEEE80211;
430 		sockp->sa_len = 12;	/* XXX != 0 */
431 		hlen = sizeof(struct ieee80211_bpf_params);
432 		break;
433 
434 	default:
435 		return (EIO);
436 	}
437 
438 	len = uio->uio_resid;
439 
440 	if (len - hlen > ifp->if_mtu)
441 		return (EMSGSIZE);
442 
443 	if ((unsigned)len > MJUM16BYTES)
444 		return (EIO);
445 
446 	if (len <= MHLEN)
447 		MGETHDR(m, M_WAIT, MT_DATA);
448 	else if (len <= MCLBYTES)
449 		m = m_getcl(M_WAIT, MT_DATA, M_PKTHDR);
450 	else
451 		m = m_getjcl(M_WAIT, MT_DATA, M_PKTHDR,
452 #if (MJUMPAGESIZE > MCLBYTES)
453 		    len <= MJUMPAGESIZE ? MJUMPAGESIZE :
454 #endif
455 		    (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES));
456 	m->m_pkthdr.len = m->m_len = len;
457 	m->m_pkthdr.rcvif = NULL;
458 	*mp = m;
459 
460 	if (m->m_len < hlen) {
461 		error = EPERM;
462 		goto bad;
463 	}
464 
465 	error = uiomove(mtod(m, u_char *), len, uio);
466 	if (error)
467 		goto bad;
468 
469 	slen = bpf_filter(wfilter, mtod(m, u_char *), len, len);
470 	if (slen == 0) {
471 		error = EPERM;
472 		goto bad;
473 	}
474 
475 	/* Check for multicast destination */
476 	switch (linktype) {
477 	case DLT_EN10MB:
478 		eh = mtod(m, struct ether_header *);
479 		if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
480 			if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost,
481 			    ETHER_ADDR_LEN) == 0)
482 				m->m_flags |= M_BCAST;
483 			else
484 				m->m_flags |= M_MCAST;
485 		}
486 		break;
487 	}
488 
489 	/*
490 	 * Make room for link header, and copy it to sockaddr
491 	 */
492 	if (hlen != 0) {
493 		if (sockp->sa_family == AF_IEEE80211) {
494 			/*
495 			 * Collect true length from the parameter header
496 			 * NB: sockp is known to be zero'd so if we do a
497 			 *     short copy unspecified parameters will be
498 			 *     zero.
499 			 * NB: packet may not be aligned after stripping
500 			 *     bpf params
501 			 * XXX check ibp_vers
502 			 */
503 			p = mtod(m, const struct ieee80211_bpf_params *);
504 			hlen = p->ibp_len;
505 			if (hlen > sizeof(sockp->sa_data)) {
506 				error = EINVAL;
507 				goto bad;
508 			}
509 		}
510 		bcopy(m->m_data, sockp->sa_data, hlen);
511 	}
512 	*hdrlen = hlen;
513 
514 	return (0);
515 bad:
516 	m_freem(m);
517 	return (error);
518 }
519 
520 /*
521  * Attach file to the bpf interface, i.e. make d listen on bp.
522  */
523 static void
524 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
525 {
526 	/*
527 	 * Point d at bp, and add d to the interface's list of listeners.
528 	 * Finally, point the driver's bpf cookie at the interface so
529 	 * it will divert packets to bpf.
530 	 */
531 	BPFIF_LOCK(bp);
532 	d->bd_bif = bp;
533 	LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
534 
535 	bpf_bpfd_cnt++;
536 	BPFIF_UNLOCK(bp);
537 }
538 
539 /*
540  * Detach a file from its interface.
541  */
542 static void
543 bpf_detachd(struct bpf_d *d)
544 {
545 	int error;
546 	struct bpf_if *bp;
547 	struct ifnet *ifp;
548 
549 	bp = d->bd_bif;
550 	BPFIF_LOCK(bp);
551 	BPFD_LOCK(d);
552 	ifp = d->bd_bif->bif_ifp;
553 
554 	/*
555 	 * Remove d from the interface's descriptor list.
556 	 */
557 	LIST_REMOVE(d, bd_next);
558 
559 	bpf_bpfd_cnt--;
560 	d->bd_bif = NULL;
561 	BPFD_UNLOCK(d);
562 	BPFIF_UNLOCK(bp);
563 
564 	/*
565 	 * Check if this descriptor had requested promiscuous mode.
566 	 * If so, turn it off.
567 	 */
568 	if (d->bd_promisc) {
569 		d->bd_promisc = 0;
570 		CURVNET_SET(ifp->if_vnet);
571 		error = ifpromisc(ifp, 0);
572 		CURVNET_RESTORE();
573 		if (error != 0 && error != ENXIO) {
574 			/*
575 			 * ENXIO can happen if a pccard is unplugged
576 			 * Something is really wrong if we were able to put
577 			 * the driver into promiscuous mode, but can't
578 			 * take it out.
579 			 */
580 			if_printf(bp->bif_ifp,
581 				"bpf_detach: ifpromisc failed (%d)\n", error);
582 		}
583 	}
584 }
585 
586 /*
587  * Close the descriptor by detaching it from its interface,
588  * deallocating its buffers, and marking it free.
589  */
590 static void
591 bpf_dtor(void *data)
592 {
593 	struct bpf_d *d = data;
594 
595 	BPFD_LOCK(d);
596 	if (d->bd_state == BPF_WAITING)
597 		callout_stop(&d->bd_callout);
598 	d->bd_state = BPF_IDLE;
599 	BPFD_UNLOCK(d);
600 	funsetown(&d->bd_sigio);
601 	mtx_lock(&bpf_mtx);
602 	if (d->bd_bif)
603 		bpf_detachd(d);
604 	mtx_unlock(&bpf_mtx);
605 	selwakeuppri(&d->bd_sel, PRINET);
606 #ifdef MAC
607 	mac_bpfdesc_destroy(d);
608 #endif /* MAC */
609 	knlist_destroy(&d->bd_sel.si_note);
610 	bpf_freed(d);
611 	free(d, M_BPF);
612 }
613 
614 /*
615  * Open ethernet device.  Returns ENXIO for illegal minor device number,
616  * EBUSY if file is open by another process.
617  */
618 /* ARGSUSED */
619 static	int
620 bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
621 {
622 	struct bpf_d *d;
623 	int error;
624 
625 	d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
626 	error = devfs_set_cdevpriv(d, bpf_dtor);
627 	if (error != 0) {
628 		free(d, M_BPF);
629 		return (error);
630 	}
631 
632 	/*
633 	 * For historical reasons, perform a one-time initialization call to
634 	 * the buffer routines, even though we're not yet committed to a
635 	 * particular buffer method.
636 	 */
637 	bpf_buffer_init(d);
638 	d->bd_bufmode = BPF_BUFMODE_BUFFER;
639 	d->bd_sig = SIGIO;
640 	d->bd_direction = BPF_D_INOUT;
641 	d->bd_pid = td->td_proc->p_pid;
642 #ifdef MAC
643 	mac_bpfdesc_init(d);
644 	mac_bpfdesc_create(td->td_ucred, d);
645 #endif
646 	mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF);
647 	callout_init(&d->bd_callout, CALLOUT_MPSAFE);
648 	knlist_init(&d->bd_sel.si_note, &d->bd_mtx, NULL, NULL, NULL);
649 
650 	return (0);
651 }
652 
653 /*
654  *  bpfread - read next chunk of packets from buffers
655  */
656 static	int
657 bpfread(struct cdev *dev, struct uio *uio, int ioflag)
658 {
659 	struct bpf_d *d;
660 	int timed_out;
661 	int error;
662 
663 	error = devfs_get_cdevpriv((void **)&d);
664 	if (error != 0)
665 		return (error);
666 
667 	/*
668 	 * Restrict application to use a buffer the same size as
669 	 * as kernel buffers.
670 	 */
671 	if (uio->uio_resid != d->bd_bufsize)
672 		return (EINVAL);
673 
674 	BPFD_LOCK(d);
675 	d->bd_pid = curthread->td_proc->p_pid;
676 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER) {
677 		BPFD_UNLOCK(d);
678 		return (EOPNOTSUPP);
679 	}
680 	if (d->bd_state == BPF_WAITING)
681 		callout_stop(&d->bd_callout);
682 	timed_out = (d->bd_state == BPF_TIMED_OUT);
683 	d->bd_state = BPF_IDLE;
684 	/*
685 	 * If the hold buffer is empty, then do a timed sleep, which
686 	 * ends when the timeout expires or when enough packets
687 	 * have arrived to fill the store buffer.
688 	 */
689 	while (d->bd_hbuf == NULL) {
690 		if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
691 			/*
692 			 * A packet(s) either arrived since the previous
693 			 * read or arrived while we were asleep.
694 			 * Rotate the buffers and return what's here.
695 			 */
696 			ROTATE_BUFFERS(d);
697 			break;
698 		}
699 
700 		/*
701 		 * No data is available, check to see if the bpf device
702 		 * is still pointed at a real interface.  If not, return
703 		 * ENXIO so that the userland process knows to rebind
704 		 * it before using it again.
705 		 */
706 		if (d->bd_bif == NULL) {
707 			BPFD_UNLOCK(d);
708 			return (ENXIO);
709 		}
710 
711 		if (ioflag & O_NONBLOCK) {
712 			BPFD_UNLOCK(d);
713 			return (EWOULDBLOCK);
714 		}
715 		error = msleep(d, &d->bd_mtx, PRINET|PCATCH,
716 		     "bpf", d->bd_rtout);
717 		if (error == EINTR || error == ERESTART) {
718 			BPFD_UNLOCK(d);
719 			return (error);
720 		}
721 		if (error == EWOULDBLOCK) {
722 			/*
723 			 * On a timeout, return what's in the buffer,
724 			 * which may be nothing.  If there is something
725 			 * in the store buffer, we can rotate the buffers.
726 			 */
727 			if (d->bd_hbuf)
728 				/*
729 				 * We filled up the buffer in between
730 				 * getting the timeout and arriving
731 				 * here, so we don't need to rotate.
732 				 */
733 				break;
734 
735 			if (d->bd_slen == 0) {
736 				BPFD_UNLOCK(d);
737 				return (0);
738 			}
739 			ROTATE_BUFFERS(d);
740 			break;
741 		}
742 	}
743 	/*
744 	 * At this point, we know we have something in the hold slot.
745 	 */
746 	BPFD_UNLOCK(d);
747 
748 	/*
749 	 * Move data from hold buffer into user space.
750 	 * We know the entire buffer is transferred since
751 	 * we checked above that the read buffer is bpf_bufsize bytes.
752 	 *
753 	 * XXXRW: More synchronization needed here: what if a second thread
754 	 * issues a read on the same fd at the same time?  Don't want this
755 	 * getting invalidated.
756 	 */
757 	error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio);
758 
759 	BPFD_LOCK(d);
760 	d->bd_fbuf = d->bd_hbuf;
761 	d->bd_hbuf = NULL;
762 	d->bd_hlen = 0;
763 	bpf_buf_reclaimed(d);
764 	BPFD_UNLOCK(d);
765 
766 	return (error);
767 }
768 
769 /*
770  * If there are processes sleeping on this descriptor, wake them up.
771  */
772 static __inline void
773 bpf_wakeup(struct bpf_d *d)
774 {
775 
776 	BPFD_LOCK_ASSERT(d);
777 	if (d->bd_state == BPF_WAITING) {
778 		callout_stop(&d->bd_callout);
779 		d->bd_state = BPF_IDLE;
780 	}
781 	wakeup(d);
782 	if (d->bd_async && d->bd_sig && d->bd_sigio)
783 		pgsigio(&d->bd_sigio, d->bd_sig, 0);
784 
785 	selwakeuppri(&d->bd_sel, PRINET);
786 	KNOTE_LOCKED(&d->bd_sel.si_note, 0);
787 }
788 
789 static void
790 bpf_timed_out(void *arg)
791 {
792 	struct bpf_d *d = (struct bpf_d *)arg;
793 
794 	BPFD_LOCK(d);
795 	if (d->bd_state == BPF_WAITING) {
796 		d->bd_state = BPF_TIMED_OUT;
797 		if (d->bd_slen != 0)
798 			bpf_wakeup(d);
799 	}
800 	BPFD_UNLOCK(d);
801 }
802 
803 static int
804 bpf_ready(struct bpf_d *d)
805 {
806 
807 	BPFD_LOCK_ASSERT(d);
808 
809 	if (!bpf_canfreebuf(d) && d->bd_hlen != 0)
810 		return (1);
811 	if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
812 	    d->bd_slen != 0)
813 		return (1);
814 	return (0);
815 }
816 
817 static int
818 bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
819 {
820 	struct bpf_d *d;
821 	struct ifnet *ifp;
822 	struct mbuf *m, *mc;
823 	struct sockaddr dst;
824 	int error, hlen;
825 
826 	error = devfs_get_cdevpriv((void **)&d);
827 	if (error != 0)
828 		return (error);
829 
830 	d->bd_pid = curthread->td_proc->p_pid;
831 	d->bd_wcount++;
832 	if (d->bd_bif == NULL) {
833 		d->bd_wdcount++;
834 		return (ENXIO);
835 	}
836 
837 	ifp = d->bd_bif->bif_ifp;
838 
839 	if ((ifp->if_flags & IFF_UP) == 0) {
840 		d->bd_wdcount++;
841 		return (ENETDOWN);
842 	}
843 
844 	if (uio->uio_resid == 0) {
845 		d->bd_wdcount++;
846 		return (0);
847 	}
848 
849 	bzero(&dst, sizeof(dst));
850 	m = NULL;
851 	hlen = 0;
852 	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp,
853 	    &m, &dst, &hlen, d->bd_wfilter);
854 	if (error) {
855 		d->bd_wdcount++;
856 		return (error);
857 	}
858 	d->bd_wfcount++;
859 	if (d->bd_hdrcmplt)
860 		dst.sa_family = pseudo_AF_HDRCMPLT;
861 
862 	if (d->bd_feedback) {
863 		mc = m_dup(m, M_DONTWAIT);
864 		if (mc != NULL)
865 			mc->m_pkthdr.rcvif = ifp;
866 		/* Set M_PROMISC for outgoing packets to be discarded. */
867 		if (d->bd_direction == BPF_D_INOUT)
868 			m->m_flags |= M_PROMISC;
869 	} else
870 		mc = NULL;
871 
872 	m->m_pkthdr.len -= hlen;
873 	m->m_len -= hlen;
874 	m->m_data += hlen;	/* XXX */
875 
876 #ifdef MAC
877 	BPFD_LOCK(d);
878 	CURVNET_SET(ifp->if_vnet);
879 	mac_bpfdesc_create_mbuf(d, m);
880 	CURVNET_RESTORE();
881 	if (mc != NULL)
882 		mac_bpfdesc_create_mbuf(d, mc);
883 	BPFD_UNLOCK(d);
884 #endif
885 
886 	error = (*ifp->if_output)(ifp, m, &dst, NULL);
887 	if (error)
888 		d->bd_wdcount++;
889 
890 	if (mc != NULL) {
891 		if (error == 0)
892 			(*ifp->if_input)(ifp, mc);
893 		else
894 			m_freem(mc);
895 	}
896 
897 	return (error);
898 }
899 
900 /*
901  * Reset a descriptor by flushing its packet buffer and clearing the
902  * receive and drop counts.
903  */
904 static void
905 reset_d(struct bpf_d *d)
906 {
907 
908 	mtx_assert(&d->bd_mtx, MA_OWNED);
909 	if (d->bd_hbuf) {
910 		/* Free the hold buffer. */
911 		d->bd_fbuf = d->bd_hbuf;
912 		d->bd_hbuf = NULL;
913 		bpf_buf_reclaimed(d);
914 	}
915 	d->bd_slen = 0;
916 	d->bd_hlen = 0;
917 	d->bd_rcount = 0;
918 	d->bd_dcount = 0;
919 	d->bd_fcount = 0;
920 	d->bd_wcount = 0;
921 	d->bd_wfcount = 0;
922 	d->bd_wdcount = 0;
923 	d->bd_zcopy = 0;
924 }
925 
926 /*
927  *  FIONREAD		Check for read packet available.
928  *  SIOCGIFADDR		Get interface address - convenient hook to driver.
929  *  BIOCGBLEN		Get buffer len [for read()].
930  *  BIOCSETF		Set read filter.
931  *  BIOCSETFNR		Set read filter without resetting descriptor.
932  *  BIOCSETWF		Set write filter.
933  *  BIOCFLUSH		Flush read packet buffer.
934  *  BIOCPROMISC		Put interface into promiscuous mode.
935  *  BIOCGDLT		Get link layer type.
936  *  BIOCGETIF		Get interface name.
937  *  BIOCSETIF		Set interface.
938  *  BIOCSRTIMEOUT	Set read timeout.
939  *  BIOCGRTIMEOUT	Get read timeout.
940  *  BIOCGSTATS		Get packet stats.
941  *  BIOCIMMEDIATE	Set immediate mode.
942  *  BIOCVERSION		Get filter language version.
943  *  BIOCGHDRCMPLT	Get "header already complete" flag
944  *  BIOCSHDRCMPLT	Set "header already complete" flag
945  *  BIOCGDIRECTION	Get packet direction flag
946  *  BIOCSDIRECTION	Set packet direction flag
947  *  BIOCLOCK		Set "locked" flag
948  *  BIOCFEEDBACK	Set packet feedback mode.
949  *  BIOCSETZBUF		Set current zero-copy buffer locations.
950  *  BIOCGETZMAX		Get maximum zero-copy buffer size.
951  *  BIOCROTZBUF		Force rotation of zero-copy buffer
952  *  BIOCSETBUFMODE	Set buffer mode.
953  *  BIOCGETBUFMODE	Get current buffer mode.
954  */
955 /* ARGSUSED */
956 static	int
957 bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
958     struct thread *td)
959 {
960 	struct bpf_d *d;
961 	int error;
962 
963 	error = devfs_get_cdevpriv((void **)&d);
964 	if (error != 0)
965 		return (error);
966 
967 	/*
968 	 * Refresh PID associated with this descriptor.
969 	 */
970 	BPFD_LOCK(d);
971 	d->bd_pid = td->td_proc->p_pid;
972 	if (d->bd_state == BPF_WAITING)
973 		callout_stop(&d->bd_callout);
974 	d->bd_state = BPF_IDLE;
975 	BPFD_UNLOCK(d);
976 
977 	if (d->bd_locked == 1) {
978 		switch (cmd) {
979 		case BIOCGBLEN:
980 		case BIOCFLUSH:
981 		case BIOCGDLT:
982 		case BIOCGDLTLIST:
983 		case BIOCGETIF:
984 		case BIOCGRTIMEOUT:
985 		case BIOCGSTATS:
986 		case BIOCVERSION:
987 		case BIOCGRSIG:
988 		case BIOCGHDRCMPLT:
989 		case BIOCFEEDBACK:
990 		case FIONREAD:
991 		case BIOCLOCK:
992 		case BIOCSRTIMEOUT:
993 		case BIOCIMMEDIATE:
994 		case TIOCGPGRP:
995 		case BIOCROTZBUF:
996 			break;
997 		default:
998 			return (EPERM);
999 		}
1000 	}
1001 	CURVNET_SET(TD_TO_VNET(td));
1002 	switch (cmd) {
1003 
1004 	default:
1005 		error = EINVAL;
1006 		break;
1007 
1008 	/*
1009 	 * Check for read packet available.
1010 	 */
1011 	case FIONREAD:
1012 		{
1013 			int n;
1014 
1015 			BPFD_LOCK(d);
1016 			n = d->bd_slen;
1017 			if (d->bd_hbuf)
1018 				n += d->bd_hlen;
1019 			BPFD_UNLOCK(d);
1020 
1021 			*(int *)addr = n;
1022 			break;
1023 		}
1024 
1025 	case SIOCGIFADDR:
1026 		{
1027 			struct ifnet *ifp;
1028 
1029 			if (d->bd_bif == NULL)
1030 				error = EINVAL;
1031 			else {
1032 				ifp = d->bd_bif->bif_ifp;
1033 				error = (*ifp->if_ioctl)(ifp, cmd, addr);
1034 			}
1035 			break;
1036 		}
1037 
1038 	/*
1039 	 * Get buffer len [for read()].
1040 	 */
1041 	case BIOCGBLEN:
1042 		*(u_int *)addr = d->bd_bufsize;
1043 		break;
1044 
1045 	/*
1046 	 * Set buffer length.
1047 	 */
1048 	case BIOCSBLEN:
1049 		error = bpf_ioctl_sblen(d, (u_int *)addr);
1050 		break;
1051 
1052 	/*
1053 	 * Set link layer read filter.
1054 	 */
1055 	case BIOCSETF:
1056 	case BIOCSETFNR:
1057 	case BIOCSETWF:
1058 		error = bpf_setf(d, (struct bpf_program *)addr, cmd);
1059 		break;
1060 
1061 	/*
1062 	 * Flush read packet buffer.
1063 	 */
1064 	case BIOCFLUSH:
1065 		BPFD_LOCK(d);
1066 		reset_d(d);
1067 		BPFD_UNLOCK(d);
1068 		break;
1069 
1070 	/*
1071 	 * Put interface into promiscuous mode.
1072 	 */
1073 	case BIOCPROMISC:
1074 		if (d->bd_bif == NULL) {
1075 			/*
1076 			 * No interface attached yet.
1077 			 */
1078 			error = EINVAL;
1079 			break;
1080 		}
1081 		if (d->bd_promisc == 0) {
1082 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
1083 			if (error == 0)
1084 				d->bd_promisc = 1;
1085 		}
1086 		break;
1087 
1088 	/*
1089 	 * Get current data link type.
1090 	 */
1091 	case BIOCGDLT:
1092 		if (d->bd_bif == NULL)
1093 			error = EINVAL;
1094 		else
1095 			*(u_int *)addr = d->bd_bif->bif_dlt;
1096 		break;
1097 
1098 	/*
1099 	 * Get a list of supported data link types.
1100 	 */
1101 	case BIOCGDLTLIST:
1102 		if (d->bd_bif == NULL)
1103 			error = EINVAL;
1104 		else
1105 			error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
1106 		break;
1107 
1108 	/*
1109 	 * Set data link type.
1110 	 */
1111 	case BIOCSDLT:
1112 		if (d->bd_bif == NULL)
1113 			error = EINVAL;
1114 		else
1115 			error = bpf_setdlt(d, *(u_int *)addr);
1116 		break;
1117 
1118 	/*
1119 	 * Get interface name.
1120 	 */
1121 	case BIOCGETIF:
1122 		if (d->bd_bif == NULL)
1123 			error = EINVAL;
1124 		else {
1125 			struct ifnet *const ifp = d->bd_bif->bif_ifp;
1126 			struct ifreq *const ifr = (struct ifreq *)addr;
1127 
1128 			strlcpy(ifr->ifr_name, ifp->if_xname,
1129 			    sizeof(ifr->ifr_name));
1130 		}
1131 		break;
1132 
1133 	/*
1134 	 * Set interface.
1135 	 */
1136 	case BIOCSETIF:
1137 		error = bpf_setif(d, (struct ifreq *)addr);
1138 		break;
1139 
1140 	/*
1141 	 * Set read timeout.
1142 	 */
1143 	case BIOCSRTIMEOUT:
1144 		{
1145 			struct timeval *tv = (struct timeval *)addr;
1146 
1147 			/*
1148 			 * Subtract 1 tick from tvtohz() since this isn't
1149 			 * a one-shot timer.
1150 			 */
1151 			if ((error = itimerfix(tv)) == 0)
1152 				d->bd_rtout = tvtohz(tv) - 1;
1153 			break;
1154 		}
1155 
1156 	/*
1157 	 * Get read timeout.
1158 	 */
1159 	case BIOCGRTIMEOUT:
1160 		{
1161 			struct timeval *tv = (struct timeval *)addr;
1162 
1163 			tv->tv_sec = d->bd_rtout / hz;
1164 			tv->tv_usec = (d->bd_rtout % hz) * tick;
1165 			break;
1166 		}
1167 
1168 	/*
1169 	 * Get packet stats.
1170 	 */
1171 	case BIOCGSTATS:
1172 		{
1173 			struct bpf_stat *bs = (struct bpf_stat *)addr;
1174 
1175 			/* XXXCSJP overflow */
1176 			bs->bs_recv = d->bd_rcount;
1177 			bs->bs_drop = d->bd_dcount;
1178 			break;
1179 		}
1180 
1181 	/*
1182 	 * Set immediate mode.
1183 	 */
1184 	case BIOCIMMEDIATE:
1185 		d->bd_immediate = *(u_int *)addr;
1186 		break;
1187 
1188 	case BIOCVERSION:
1189 		{
1190 			struct bpf_version *bv = (struct bpf_version *)addr;
1191 
1192 			bv->bv_major = BPF_MAJOR_VERSION;
1193 			bv->bv_minor = BPF_MINOR_VERSION;
1194 			break;
1195 		}
1196 
1197 	/*
1198 	 * Get "header already complete" flag
1199 	 */
1200 	case BIOCGHDRCMPLT:
1201 		*(u_int *)addr = d->bd_hdrcmplt;
1202 		break;
1203 
1204 	/*
1205 	 * Set "header already complete" flag
1206 	 */
1207 	case BIOCSHDRCMPLT:
1208 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1209 		break;
1210 
1211 	/*
1212 	 * Get packet direction flag
1213 	 */
1214 	case BIOCGDIRECTION:
1215 		*(u_int *)addr = d->bd_direction;
1216 		break;
1217 
1218 	/*
1219 	 * Set packet direction flag
1220 	 */
1221 	case BIOCSDIRECTION:
1222 		{
1223 			u_int	direction;
1224 
1225 			direction = *(u_int *)addr;
1226 			switch (direction) {
1227 			case BPF_D_IN:
1228 			case BPF_D_INOUT:
1229 			case BPF_D_OUT:
1230 				d->bd_direction = direction;
1231 				break;
1232 			default:
1233 				error = EINVAL;
1234 			}
1235 		}
1236 		break;
1237 
1238 	case BIOCFEEDBACK:
1239 		d->bd_feedback = *(u_int *)addr;
1240 		break;
1241 
1242 	case BIOCLOCK:
1243 		d->bd_locked = 1;
1244 		break;
1245 
1246 	case FIONBIO:		/* Non-blocking I/O */
1247 		break;
1248 
1249 	case FIOASYNC:		/* Send signal on receive packets */
1250 		d->bd_async = *(int *)addr;
1251 		break;
1252 
1253 	case FIOSETOWN:
1254 		error = fsetown(*(int *)addr, &d->bd_sigio);
1255 		break;
1256 
1257 	case FIOGETOWN:
1258 		*(int *)addr = fgetown(&d->bd_sigio);
1259 		break;
1260 
1261 	/* This is deprecated, FIOSETOWN should be used instead. */
1262 	case TIOCSPGRP:
1263 		error = fsetown(-(*(int *)addr), &d->bd_sigio);
1264 		break;
1265 
1266 	/* This is deprecated, FIOGETOWN should be used instead. */
1267 	case TIOCGPGRP:
1268 		*(int *)addr = -fgetown(&d->bd_sigio);
1269 		break;
1270 
1271 	case BIOCSRSIG:		/* Set receive signal */
1272 		{
1273 			u_int sig;
1274 
1275 			sig = *(u_int *)addr;
1276 
1277 			if (sig >= NSIG)
1278 				error = EINVAL;
1279 			else
1280 				d->bd_sig = sig;
1281 			break;
1282 		}
1283 	case BIOCGRSIG:
1284 		*(u_int *)addr = d->bd_sig;
1285 		break;
1286 
1287 	case BIOCGETBUFMODE:
1288 		*(u_int *)addr = d->bd_bufmode;
1289 		break;
1290 
1291 	case BIOCSETBUFMODE:
1292 		/*
1293 		 * Allow the buffering mode to be changed as long as we
1294 		 * haven't yet committed to a particular mode.  Our
1295 		 * definition of commitment, for now, is whether or not a
1296 		 * buffer has been allocated or an interface attached, since
1297 		 * that's the point where things get tricky.
1298 		 */
1299 		switch (*(u_int *)addr) {
1300 		case BPF_BUFMODE_BUFFER:
1301 			break;
1302 
1303 		case BPF_BUFMODE_ZBUF:
1304 			if (bpf_zerocopy_enable)
1305 				break;
1306 			/* FALLSTHROUGH */
1307 
1308 		default:
1309 			return (EINVAL);
1310 		}
1311 
1312 		BPFD_LOCK(d);
1313 		if (d->bd_sbuf != NULL || d->bd_hbuf != NULL ||
1314 		    d->bd_fbuf != NULL || d->bd_bif != NULL) {
1315 			BPFD_UNLOCK(d);
1316 			return (EBUSY);
1317 		}
1318 		d->bd_bufmode = *(u_int *)addr;
1319 		BPFD_UNLOCK(d);
1320 		break;
1321 
1322 	case BIOCGETZMAX:
1323 		return (bpf_ioctl_getzmax(td, d, (size_t *)addr));
1324 
1325 	case BIOCSETZBUF:
1326 		return (bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr));
1327 
1328 	case BIOCROTZBUF:
1329 		return (bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr));
1330 	}
1331 	CURVNET_RESTORE();
1332 	return (error);
1333 }
1334 
1335 /*
1336  * Set d's packet filter program to fp.  If this file already has a filter,
1337  * free it and replace it.  Returns EINVAL for bogus requests.
1338  */
1339 static int
1340 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1341 {
1342 	struct bpf_insn *fcode, *old;
1343 	u_int wfilter, flen, size;
1344 #ifdef BPF_JITTER
1345 	bpf_jit_filter *ofunc;
1346 #endif
1347 
1348 	if (cmd == BIOCSETWF) {
1349 		old = d->bd_wfilter;
1350 		wfilter = 1;
1351 #ifdef BPF_JITTER
1352 		ofunc = NULL;
1353 #endif
1354 	} else {
1355 		wfilter = 0;
1356 		old = d->bd_rfilter;
1357 #ifdef BPF_JITTER
1358 		ofunc = d->bd_bfilter;
1359 #endif
1360 	}
1361 	if (fp->bf_insns == NULL) {
1362 		if (fp->bf_len != 0)
1363 			return (EINVAL);
1364 		BPFD_LOCK(d);
1365 		if (wfilter)
1366 			d->bd_wfilter = NULL;
1367 		else {
1368 			d->bd_rfilter = NULL;
1369 #ifdef BPF_JITTER
1370 			d->bd_bfilter = NULL;
1371 #endif
1372 			if (cmd == BIOCSETF)
1373 				reset_d(d);
1374 		}
1375 		BPFD_UNLOCK(d);
1376 		if (old != NULL)
1377 			free((caddr_t)old, M_BPF);
1378 #ifdef BPF_JITTER
1379 		if (ofunc != NULL)
1380 			bpf_destroy_jit_filter(ofunc);
1381 #endif
1382 		return (0);
1383 	}
1384 	flen = fp->bf_len;
1385 	if (flen > bpf_maxinsns)
1386 		return (EINVAL);
1387 
1388 	size = flen * sizeof(*fp->bf_insns);
1389 	fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK);
1390 	if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 &&
1391 	    bpf_validate(fcode, (int)flen)) {
1392 		BPFD_LOCK(d);
1393 		if (wfilter)
1394 			d->bd_wfilter = fcode;
1395 		else {
1396 			d->bd_rfilter = fcode;
1397 #ifdef BPF_JITTER
1398 			d->bd_bfilter = bpf_jitter(fcode, flen);
1399 #endif
1400 			if (cmd == BIOCSETF)
1401 				reset_d(d);
1402 		}
1403 		BPFD_UNLOCK(d);
1404 		if (old != NULL)
1405 			free((caddr_t)old, M_BPF);
1406 #ifdef BPF_JITTER
1407 		if (ofunc != NULL)
1408 			bpf_destroy_jit_filter(ofunc);
1409 #endif
1410 
1411 		return (0);
1412 	}
1413 	free((caddr_t)fcode, M_BPF);
1414 	return (EINVAL);
1415 }
1416 
1417 /*
1418  * Detach a file from its current interface (if attached at all) and attach
1419  * to the interface indicated by the name stored in ifr.
1420  * Return an errno or 0.
1421  */
1422 static int
1423 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1424 {
1425 	struct bpf_if *bp;
1426 	struct ifnet *theywant;
1427 
1428 	theywant = ifunit(ifr->ifr_name);
1429 	if (theywant == NULL || theywant->if_bpf == NULL)
1430 		return (ENXIO);
1431 
1432 	bp = theywant->if_bpf;
1433 
1434 	/*
1435 	 * Behavior here depends on the buffering model.  If we're using
1436 	 * kernel memory buffers, then we can allocate them here.  If we're
1437 	 * using zero-copy, then the user process must have registered
1438 	 * buffers by the time we get here.  If not, return an error.
1439 	 *
1440 	 * XXXRW: There are locking issues here with multi-threaded use: what
1441 	 * if two threads try to set the interface at once?
1442 	 */
1443 	switch (d->bd_bufmode) {
1444 	case BPF_BUFMODE_BUFFER:
1445 		if (d->bd_sbuf == NULL)
1446 			bpf_buffer_alloc(d);
1447 		KASSERT(d->bd_sbuf != NULL, ("bpf_setif: bd_sbuf NULL"));
1448 		break;
1449 
1450 	case BPF_BUFMODE_ZBUF:
1451 		if (d->bd_sbuf == NULL)
1452 			return (EINVAL);
1453 		break;
1454 
1455 	default:
1456 		panic("bpf_setif: bufmode %d", d->bd_bufmode);
1457 	}
1458 	if (bp != d->bd_bif) {
1459 		if (d->bd_bif)
1460 			/*
1461 			 * Detach if attached to something else.
1462 			 */
1463 			bpf_detachd(d);
1464 
1465 		bpf_attachd(d, bp);
1466 	}
1467 	BPFD_LOCK(d);
1468 	reset_d(d);
1469 	BPFD_UNLOCK(d);
1470 	return (0);
1471 }
1472 
1473 /*
1474  * Support for select() and poll() system calls
1475  *
1476  * Return true iff the specific operation will not block indefinitely.
1477  * Otherwise, return false but make a note that a selwakeup() must be done.
1478  */
1479 static int
1480 bpfpoll(struct cdev *dev, int events, struct thread *td)
1481 {
1482 	struct bpf_d *d;
1483 	int revents;
1484 
1485 	if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL)
1486 		return (events &
1487 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1488 
1489 	/*
1490 	 * Refresh PID associated with this descriptor.
1491 	 */
1492 	revents = events & (POLLOUT | POLLWRNORM);
1493 	BPFD_LOCK(d);
1494 	d->bd_pid = td->td_proc->p_pid;
1495 	if (events & (POLLIN | POLLRDNORM)) {
1496 		if (bpf_ready(d))
1497 			revents |= events & (POLLIN | POLLRDNORM);
1498 		else {
1499 			selrecord(td, &d->bd_sel);
1500 			/* Start the read timeout if necessary. */
1501 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1502 				callout_reset(&d->bd_callout, d->bd_rtout,
1503 				    bpf_timed_out, d);
1504 				d->bd_state = BPF_WAITING;
1505 			}
1506 		}
1507 	}
1508 	BPFD_UNLOCK(d);
1509 	return (revents);
1510 }
1511 
1512 /*
1513  * Support for kevent() system call.  Register EVFILT_READ filters and
1514  * reject all others.
1515  */
1516 int
1517 bpfkqfilter(struct cdev *dev, struct knote *kn)
1518 {
1519 	struct bpf_d *d;
1520 
1521 	if (devfs_get_cdevpriv((void **)&d) != 0 ||
1522 	    kn->kn_filter != EVFILT_READ)
1523 		return (1);
1524 
1525 	/*
1526 	 * Refresh PID associated with this descriptor.
1527 	 */
1528 	BPFD_LOCK(d);
1529 	d->bd_pid = curthread->td_proc->p_pid;
1530 	kn->kn_fop = &bpfread_filtops;
1531 	kn->kn_hook = d;
1532 	knlist_add(&d->bd_sel.si_note, kn, 1);
1533 	BPFD_UNLOCK(d);
1534 
1535 	return (0);
1536 }
1537 
1538 static void
1539 filt_bpfdetach(struct knote *kn)
1540 {
1541 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1542 
1543 	knlist_remove(&d->bd_sel.si_note, kn, 0);
1544 }
1545 
1546 static int
1547 filt_bpfread(struct knote *kn, long hint)
1548 {
1549 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1550 	int ready;
1551 
1552 	BPFD_LOCK_ASSERT(d);
1553 	ready = bpf_ready(d);
1554 	if (ready) {
1555 		kn->kn_data = d->bd_slen;
1556 		if (d->bd_hbuf)
1557 			kn->kn_data += d->bd_hlen;
1558 	}
1559 	else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1560 		callout_reset(&d->bd_callout, d->bd_rtout,
1561 		    bpf_timed_out, d);
1562 		d->bd_state = BPF_WAITING;
1563 	}
1564 
1565 	return (ready);
1566 }
1567 
1568 /*
1569  * Incoming linkage from device drivers.  Process the packet pkt, of length
1570  * pktlen, which is stored in a contiguous buffer.  The packet is parsed
1571  * by each process' filter, and if accepted, stashed into the corresponding
1572  * buffer.
1573  */
1574 void
1575 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
1576 {
1577 	struct bpf_d *d;
1578 	u_int slen;
1579 	int gottime;
1580 	struct timeval tv;
1581 
1582 	gottime = 0;
1583 	BPFIF_LOCK(bp);
1584 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1585 		BPFD_LOCK(d);
1586 		++d->bd_rcount;
1587 		/*
1588 		 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no
1589 		 * way for the caller to indiciate to us whether this packet
1590 		 * is inbound or outbound.  In the bpf_mtap() routines, we use
1591 		 * the interface pointers on the mbuf to figure it out.
1592 		 */
1593 #ifdef BPF_JITTER
1594 		if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL)
1595 			slen = (*(d->bd_bfilter->func))(pkt, pktlen, pktlen);
1596 		else
1597 #endif
1598 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
1599 		if (slen != 0) {
1600 			d->bd_fcount++;
1601 			if (!gottime) {
1602 				microtime(&tv);
1603 				gottime = 1;
1604 			}
1605 #ifdef MAC
1606 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1607 #endif
1608 				catchpacket(d, pkt, pktlen, slen,
1609 				    bpf_append_bytes, &tv);
1610 		}
1611 		BPFD_UNLOCK(d);
1612 	}
1613 	BPFIF_UNLOCK(bp);
1614 }
1615 
1616 #define	BPF_CHECK_DIRECTION(d, r, i)				\
1617 	    (((d)->bd_direction == BPF_D_IN && (r) != (i)) ||	\
1618 	    ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
1619 
1620 /*
1621  * Incoming linkage from device drivers, when packet is in an mbuf chain.
1622  */
1623 void
1624 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
1625 {
1626 	struct bpf_d *d;
1627 	u_int pktlen, slen;
1628 	int gottime;
1629 	struct timeval tv;
1630 
1631 	/* Skip outgoing duplicate packets. */
1632 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1633 		m->m_flags &= ~M_PROMISC;
1634 		return;
1635 	}
1636 
1637 	gottime = 0;
1638 
1639 	pktlen = m_length(m, NULL);
1640 
1641 	BPFIF_LOCK(bp);
1642 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1643 		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
1644 			continue;
1645 		BPFD_LOCK(d);
1646 		++d->bd_rcount;
1647 #ifdef BPF_JITTER
1648 		/* XXX We cannot handle multiple mbufs. */
1649 		if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL &&
1650 		    m->m_next == NULL)
1651 			slen = (*(d->bd_bfilter->func))(mtod(m, u_char *),
1652 			    pktlen, pktlen);
1653 		else
1654 #endif
1655 		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
1656 		if (slen != 0) {
1657 			d->bd_fcount++;
1658 			if (!gottime) {
1659 				microtime(&tv);
1660 				gottime = 1;
1661 			}
1662 #ifdef MAC
1663 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1664 #endif
1665 				catchpacket(d, (u_char *)m, pktlen, slen,
1666 				    bpf_append_mbuf, &tv);
1667 		}
1668 		BPFD_UNLOCK(d);
1669 	}
1670 	BPFIF_UNLOCK(bp);
1671 }
1672 
1673 /*
1674  * Incoming linkage from device drivers, when packet is in
1675  * an mbuf chain and to be prepended by a contiguous header.
1676  */
1677 void
1678 bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
1679 {
1680 	struct mbuf mb;
1681 	struct bpf_d *d;
1682 	u_int pktlen, slen;
1683 	int gottime;
1684 	struct timeval tv;
1685 
1686 	/* Skip outgoing duplicate packets. */
1687 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1688 		m->m_flags &= ~M_PROMISC;
1689 		return;
1690 	}
1691 
1692 	gottime = 0;
1693 
1694 	pktlen = m_length(m, NULL);
1695 	/*
1696 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
1697 	 * Note that we cut corners here; we only setup what's
1698 	 * absolutely needed--this mbuf should never go anywhere else.
1699 	 */
1700 	mb.m_next = m;
1701 	mb.m_data = data;
1702 	mb.m_len = dlen;
1703 	pktlen += dlen;
1704 
1705 	BPFIF_LOCK(bp);
1706 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1707 		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
1708 			continue;
1709 		BPFD_LOCK(d);
1710 		++d->bd_rcount;
1711 		slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
1712 		if (slen != 0) {
1713 			d->bd_fcount++;
1714 			if (!gottime) {
1715 				microtime(&tv);
1716 				gottime = 1;
1717 			}
1718 #ifdef MAC
1719 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1720 #endif
1721 				catchpacket(d, (u_char *)&mb, pktlen, slen,
1722 				    bpf_append_mbuf, &tv);
1723 		}
1724 		BPFD_UNLOCK(d);
1725 	}
1726 	BPFIF_UNLOCK(bp);
1727 }
1728 
1729 #undef	BPF_CHECK_DIRECTION
1730 
1731 /*
1732  * Move the packet data from interface memory (pkt) into the
1733  * store buffer.  "cpfn" is the routine called to do the actual data
1734  * transfer.  bcopy is passed in to copy contiguous chunks, while
1735  * bpf_append_mbuf is passed in to copy mbuf chains.  In the latter case,
1736  * pkt is really an mbuf.
1737  */
1738 static void
1739 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
1740     void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
1741     struct timeval *tv)
1742 {
1743 	struct bpf_hdr hdr;
1744 	int totlen, curlen;
1745 	int hdrlen = d->bd_bif->bif_hdrlen;
1746 	int do_wakeup = 0;
1747 
1748 	BPFD_LOCK_ASSERT(d);
1749 
1750 	/*
1751 	 * Detect whether user space has released a buffer back to us, and if
1752 	 * so, move it from being a hold buffer to a free buffer.  This may
1753 	 * not be the best place to do it (for example, we might only want to
1754 	 * run this check if we need the space), but for now it's a reliable
1755 	 * spot to do it.
1756 	 */
1757 	if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) {
1758 		d->bd_fbuf = d->bd_hbuf;
1759 		d->bd_hbuf = NULL;
1760 		d->bd_hlen = 0;
1761 		bpf_buf_reclaimed(d);
1762 	}
1763 
1764 	/*
1765 	 * Figure out how many bytes to move.  If the packet is
1766 	 * greater or equal to the snapshot length, transfer that
1767 	 * much.  Otherwise, transfer the whole packet (unless
1768 	 * we hit the buffer size limit).
1769 	 */
1770 	totlen = hdrlen + min(snaplen, pktlen);
1771 	if (totlen > d->bd_bufsize)
1772 		totlen = d->bd_bufsize;
1773 
1774 	/*
1775 	 * Round up the end of the previous packet to the next longword.
1776 	 *
1777 	 * Drop the packet if there's no room and no hope of room
1778 	 * If the packet would overflow the storage buffer or the storage
1779 	 * buffer is considered immutable by the buffer model, try to rotate
1780 	 * the buffer and wakeup pending processes.
1781 	 */
1782 	curlen = BPF_WORDALIGN(d->bd_slen);
1783 	if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) {
1784 		if (d->bd_fbuf == NULL) {
1785 			/*
1786 			 * There's no room in the store buffer, and no
1787 			 * prospect of room, so drop the packet.  Notify the
1788 			 * buffer model.
1789 			 */
1790 			bpf_buffull(d);
1791 			++d->bd_dcount;
1792 			return;
1793 		}
1794 		ROTATE_BUFFERS(d);
1795 		do_wakeup = 1;
1796 		curlen = 0;
1797 	} else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT)
1798 		/*
1799 		 * Immediate mode is set, or the read timeout has already
1800 		 * expired during a select call.  A packet arrived, so the
1801 		 * reader should be woken up.
1802 		 */
1803 		do_wakeup = 1;
1804 
1805 	/*
1806 	 * Append the bpf header.  Note we append the actual header size, but
1807 	 * move forward the length of the header plus padding.
1808 	 */
1809 	bzero(&hdr, sizeof(hdr));
1810 	hdr.bh_tstamp = *tv;
1811 	hdr.bh_datalen = pktlen;
1812 	hdr.bh_hdrlen = hdrlen;
1813 	hdr.bh_caplen = totlen - hdrlen;
1814 	bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr));
1815 
1816 	/*
1817 	 * Copy the packet data into the store buffer and update its length.
1818 	 */
1819 	(*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, hdr.bh_caplen);
1820 	d->bd_slen = curlen + totlen;
1821 
1822 	if (do_wakeup)
1823 		bpf_wakeup(d);
1824 }
1825 
1826 /*
1827  * Free buffers currently in use by a descriptor.
1828  * Called on close.
1829  */
1830 static void
1831 bpf_freed(struct bpf_d *d)
1832 {
1833 
1834 	/*
1835 	 * We don't need to lock out interrupts since this descriptor has
1836 	 * been detached from its interface and it yet hasn't been marked
1837 	 * free.
1838 	 */
1839 	bpf_free(d);
1840 	if (d->bd_rfilter) {
1841 		free((caddr_t)d->bd_rfilter, M_BPF);
1842 #ifdef BPF_JITTER
1843 		bpf_destroy_jit_filter(d->bd_bfilter);
1844 #endif
1845 	}
1846 	if (d->bd_wfilter)
1847 		free((caddr_t)d->bd_wfilter, M_BPF);
1848 	mtx_destroy(&d->bd_mtx);
1849 }
1850 
1851 /*
1852  * Attach an interface to bpf.  dlt is the link layer type; hdrlen is the
1853  * fixed size of the link header (variable length headers not yet supported).
1854  */
1855 void
1856 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
1857 {
1858 
1859 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
1860 }
1861 
1862 /*
1863  * Attach an interface to bpf.  ifp is a pointer to the structure
1864  * defining the interface to be attached, dlt is the link layer type,
1865  * and hdrlen is the fixed size of the link header (variable length
1866  * headers are not yet supporrted).
1867  */
1868 void
1869 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
1870 {
1871 	struct bpf_if *bp;
1872 
1873 	bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO);
1874 	if (bp == NULL)
1875 		panic("bpfattach");
1876 
1877 	LIST_INIT(&bp->bif_dlist);
1878 	bp->bif_ifp = ifp;
1879 	bp->bif_dlt = dlt;
1880 	mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF);
1881 	KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized"));
1882 	*driverp = bp;
1883 
1884 	mtx_lock(&bpf_mtx);
1885 	LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
1886 	mtx_unlock(&bpf_mtx);
1887 
1888 	/*
1889 	 * Compute the length of the bpf header.  This is not necessarily
1890 	 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
1891 	 * that the network layer header begins on a longword boundary (for
1892 	 * performance reasons and to alleviate alignment restrictions).
1893 	 */
1894 	bp->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
1895 
1896 	if (bootverbose)
1897 		if_printf(ifp, "bpf attached\n");
1898 }
1899 
1900 /*
1901  * Detach bpf from an interface.  This involves detaching each descriptor
1902  * associated with the interface, and leaving bd_bif NULL.  Notify each
1903  * descriptor as it's detached so that any sleepers wake up and get
1904  * ENXIO.
1905  */
1906 void
1907 bpfdetach(struct ifnet *ifp)
1908 {
1909 	struct bpf_if	*bp;
1910 	struct bpf_d	*d;
1911 
1912 	/* Locate BPF interface information */
1913 	mtx_lock(&bpf_mtx);
1914 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1915 		if (ifp == bp->bif_ifp)
1916 			break;
1917 	}
1918 
1919 	/* Interface wasn't attached */
1920 	if ((bp == NULL) || (bp->bif_ifp == NULL)) {
1921 		mtx_unlock(&bpf_mtx);
1922 		printf("bpfdetach: %s was not attached\n", ifp->if_xname);
1923 		return;
1924 	}
1925 
1926 	LIST_REMOVE(bp, bif_next);
1927 	mtx_unlock(&bpf_mtx);
1928 
1929 	while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) {
1930 		bpf_detachd(d);
1931 		BPFD_LOCK(d);
1932 		bpf_wakeup(d);
1933 		BPFD_UNLOCK(d);
1934 	}
1935 
1936 	mtx_destroy(&bp->bif_mtx);
1937 	free(bp, M_BPF);
1938 }
1939 
1940 /*
1941  * Get a list of available data link type of the interface.
1942  */
1943 static int
1944 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
1945 {
1946 	int n, error;
1947 	struct ifnet *ifp;
1948 	struct bpf_if *bp;
1949 
1950 	ifp = d->bd_bif->bif_ifp;
1951 	n = 0;
1952 	error = 0;
1953 	mtx_lock(&bpf_mtx);
1954 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1955 		if (bp->bif_ifp != ifp)
1956 			continue;
1957 		if (bfl->bfl_list != NULL) {
1958 			if (n >= bfl->bfl_len) {
1959 				mtx_unlock(&bpf_mtx);
1960 				return (ENOMEM);
1961 			}
1962 			error = copyout(&bp->bif_dlt,
1963 			    bfl->bfl_list + n, sizeof(u_int));
1964 		}
1965 		n++;
1966 	}
1967 	mtx_unlock(&bpf_mtx);
1968 	bfl->bfl_len = n;
1969 	return (error);
1970 }
1971 
1972 /*
1973  * Set the data link type of a BPF instance.
1974  */
1975 static int
1976 bpf_setdlt(struct bpf_d *d, u_int dlt)
1977 {
1978 	int error, opromisc;
1979 	struct ifnet *ifp;
1980 	struct bpf_if *bp;
1981 
1982 	if (d->bd_bif->bif_dlt == dlt)
1983 		return (0);
1984 	ifp = d->bd_bif->bif_ifp;
1985 	mtx_lock(&bpf_mtx);
1986 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1987 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
1988 			break;
1989 	}
1990 	mtx_unlock(&bpf_mtx);
1991 	if (bp != NULL) {
1992 		opromisc = d->bd_promisc;
1993 		bpf_detachd(d);
1994 		bpf_attachd(d, bp);
1995 		BPFD_LOCK(d);
1996 		reset_d(d);
1997 		BPFD_UNLOCK(d);
1998 		if (opromisc) {
1999 			error = ifpromisc(bp->bif_ifp, 1);
2000 			if (error)
2001 				if_printf(bp->bif_ifp,
2002 					"bpf_setdlt: ifpromisc failed (%d)\n",
2003 					error);
2004 			else
2005 				d->bd_promisc = 1;
2006 		}
2007 	}
2008 	return (bp == NULL ? EINVAL : 0);
2009 }
2010 
2011 static void
2012 bpf_drvinit(void *unused)
2013 {
2014 	struct cdev *dev;
2015 
2016 	mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF);
2017 	LIST_INIT(&bpf_iflist);
2018 
2019 	dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf");
2020 	/* For compatibility */
2021 	make_dev_alias(dev, "bpf0");
2022 
2023 }
2024 
2025 static void
2026 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
2027 {
2028 
2029 	bzero(d, sizeof(*d));
2030 	BPFD_LOCK_ASSERT(bd);
2031 	d->bd_structsize = sizeof(*d);
2032 	d->bd_immediate = bd->bd_immediate;
2033 	d->bd_promisc = bd->bd_promisc;
2034 	d->bd_hdrcmplt = bd->bd_hdrcmplt;
2035 	d->bd_direction = bd->bd_direction;
2036 	d->bd_feedback = bd->bd_feedback;
2037 	d->bd_async = bd->bd_async;
2038 	d->bd_rcount = bd->bd_rcount;
2039 	d->bd_dcount = bd->bd_dcount;
2040 	d->bd_fcount = bd->bd_fcount;
2041 	d->bd_sig = bd->bd_sig;
2042 	d->bd_slen = bd->bd_slen;
2043 	d->bd_hlen = bd->bd_hlen;
2044 	d->bd_bufsize = bd->bd_bufsize;
2045 	d->bd_pid = bd->bd_pid;
2046 	strlcpy(d->bd_ifname,
2047 	    bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
2048 	d->bd_locked = bd->bd_locked;
2049 	d->bd_wcount = bd->bd_wcount;
2050 	d->bd_wdcount = bd->bd_wdcount;
2051 	d->bd_wfcount = bd->bd_wfcount;
2052 	d->bd_zcopy = bd->bd_zcopy;
2053 	d->bd_bufmode = bd->bd_bufmode;
2054 }
2055 
2056 static int
2057 bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
2058 {
2059 	struct xbpf_d *xbdbuf, *xbd;
2060 	int index, error;
2061 	struct bpf_if *bp;
2062 	struct bpf_d *bd;
2063 
2064 	/*
2065 	 * XXX This is not technically correct. It is possible for non
2066 	 * privileged users to open bpf devices. It would make sense
2067 	 * if the users who opened the devices were able to retrieve
2068 	 * the statistics for them, too.
2069 	 */
2070 	error = priv_check(req->td, PRIV_NET_BPF);
2071 	if (error)
2072 		return (error);
2073 	if (req->oldptr == NULL)
2074 		return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
2075 	if (bpf_bpfd_cnt == 0)
2076 		return (SYSCTL_OUT(req, 0, 0));
2077 	xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
2078 	mtx_lock(&bpf_mtx);
2079 	if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
2080 		mtx_unlock(&bpf_mtx);
2081 		free(xbdbuf, M_BPF);
2082 		return (ENOMEM);
2083 	}
2084 	index = 0;
2085 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2086 		BPFIF_LOCK(bp);
2087 		LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2088 			xbd = &xbdbuf[index++];
2089 			BPFD_LOCK(bd);
2090 			bpfstats_fill_xbpf(xbd, bd);
2091 			BPFD_UNLOCK(bd);
2092 		}
2093 		BPFIF_UNLOCK(bp);
2094 	}
2095 	mtx_unlock(&bpf_mtx);
2096 	error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
2097 	free(xbdbuf, M_BPF);
2098 	return (error);
2099 }
2100 
2101 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL);
2102 
2103 #else /* !DEV_BPF && !NETGRAPH_BPF */
2104 /*
2105  * NOP stubs to allow bpf-using drivers to load and function.
2106  *
2107  * A 'better' implementation would allow the core bpf functionality
2108  * to be loaded at runtime.
2109  */
2110 static struct bpf_if bp_null;
2111 
2112 void
2113 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2114 {
2115 }
2116 
2117 void
2118 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2119 {
2120 }
2121 
2122 void
2123 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
2124 {
2125 }
2126 
2127 void
2128 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2129 {
2130 
2131 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2132 }
2133 
2134 void
2135 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2136 {
2137 
2138 	*driverp = &bp_null;
2139 }
2140 
2141 void
2142 bpfdetach(struct ifnet *ifp)
2143 {
2144 }
2145 
2146 u_int
2147 bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
2148 {
2149 	return -1;	/* "no filter" behaviour */
2150 }
2151 
2152 int
2153 bpf_validate(const struct bpf_insn *f, int len)
2154 {
2155 	return 0;		/* false */
2156 }
2157 
2158 #endif /* !DEV_BPF && !NETGRAPH_BPF */
2159