xref: /freebsd/sys/net/bpf.c (revision c96ae1968a6ab7056427a739bce81bf07447c2d4)
1 /*-
2  * Copyright (c) 1990, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from the Stanford/CMU enet packet filter,
6  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
7  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
8  * Berkeley Laboratory.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *      @(#)bpf.c	8.4 (Berkeley) 1/9/95
35  *
36  * $FreeBSD$
37  */
38 
39 #include "opt_bpf.h"
40 #include "opt_mac.h"
41 #include "opt_netgraph.h"
42 
43 #include <sys/types.h>
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/conf.h>
47 #include <sys/fcntl.h>
48 #include <sys/malloc.h>
49 #include <sys/mbuf.h>
50 #include <sys/time.h>
51 #include <sys/priv.h>
52 #include <sys/proc.h>
53 #include <sys/signalvar.h>
54 #include <sys/filio.h>
55 #include <sys/sockio.h>
56 #include <sys/ttycom.h>
57 #include <sys/uio.h>
58 
59 #include <sys/event.h>
60 #include <sys/file.h>
61 #include <sys/poll.h>
62 #include <sys/proc.h>
63 
64 #include <sys/socket.h>
65 
66 #include <net/if.h>
67 #include <net/bpf.h>
68 #ifdef BPF_JITTER
69 #include <net/bpf_jitter.h>
70 #endif
71 #include <net/bpfdesc.h>
72 
73 #include <netinet/in.h>
74 #include <netinet/if_ether.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 
78 #include <net80211/ieee80211_freebsd.h>
79 
80 #include <security/mac/mac_framework.h>
81 
82 static MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
83 
84 #if defined(DEV_BPF) || defined(NETGRAPH_BPF)
85 
86 #define PRINET  26			/* interruptible */
87 
88 /*
89  * bpf_iflist is a list of BPF interface structures, each corresponding to a
90  * specific DLT.  The same network interface might have several BPF interface
91  * structures registered by different layers in the stack (i.e., 802.11
92  * frames, ethernet frames, etc).
93  */
94 static LIST_HEAD(, bpf_if)	bpf_iflist;
95 static struct mtx	bpf_mtx;		/* bpf global lock */
96 static int		bpf_bpfd_cnt;
97 
98 static void	bpf_allocbufs(struct bpf_d *);
99 static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
100 static void	bpf_detachd(struct bpf_d *);
101 static void	bpf_freed(struct bpf_d *);
102 static void	bpf_mcopy(const void *, void *, size_t);
103 static int	bpf_movein(struct uio *, int, int,
104 		    struct mbuf **, struct sockaddr *, struct bpf_insn *);
105 static int	bpf_setif(struct bpf_d *, struct ifreq *);
106 static void	bpf_timed_out(void *);
107 static __inline void
108 		bpf_wakeup(struct bpf_d *);
109 static void	catchpacket(struct bpf_d *, u_char *, u_int,
110 		    u_int, void (*)(const void *, void *, size_t),
111 		    struct timeval *);
112 static void	reset_d(struct bpf_d *);
113 static int	 bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
114 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
115 static int	bpf_setdlt(struct bpf_d *, u_int);
116 static void	filt_bpfdetach(struct knote *);
117 static int	filt_bpfread(struct knote *, long);
118 static void	bpf_drvinit(void *);
119 static void	bpf_clone(void *, struct ucred *, char *, int, struct cdev **);
120 static int	bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
121 
122 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl");
123 static int bpf_bufsize = 4096;
124 SYSCTL_INT(_net_bpf, OID_AUTO, bufsize, CTLFLAG_RW,
125     &bpf_bufsize, 0, "Default bpf buffer size");
126 static int bpf_maxbufsize = BPF_MAXBUFSIZE;
127 SYSCTL_INT(_net_bpf, OID_AUTO, maxbufsize, CTLFLAG_RW,
128     &bpf_maxbufsize, 0, "Maximum bpf buffer size");
129 static int bpf_maxinsns = BPF_MAXINSNS;
130 SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
131     &bpf_maxinsns, 0, "Maximum bpf program instructions");
132 SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_RW,
133     bpf_stats_sysctl, "bpf statistics portal");
134 
135 static	d_open_t	bpfopen;
136 static	d_close_t	bpfclose;
137 static	d_read_t	bpfread;
138 static	d_write_t	bpfwrite;
139 static	d_ioctl_t	bpfioctl;
140 static	d_poll_t	bpfpoll;
141 static	d_kqfilter_t	bpfkqfilter;
142 
143 static struct cdevsw bpf_cdevsw = {
144 	.d_version =	D_VERSION,
145 	.d_flags =	D_NEEDGIANT,
146 	.d_open =	bpfopen,
147 	.d_close =	bpfclose,
148 	.d_read =	bpfread,
149 	.d_write =	bpfwrite,
150 	.d_ioctl =	bpfioctl,
151 	.d_poll =	bpfpoll,
152 	.d_name =	"bpf",
153 	.d_kqfilter =	bpfkqfilter,
154 };
155 
156 static struct filterops bpfread_filtops =
157 	{ 1, NULL, filt_bpfdetach, filt_bpfread };
158 
159 static int
160 bpf_movein(struct uio *uio, int linktype, int mtu, struct mbuf **mp,
161     struct sockaddr *sockp, struct bpf_insn *wfilter)
162 {
163 	const struct ieee80211_bpf_params *p;
164 	struct mbuf *m;
165 	int error;
166 	int len;
167 	int hlen;
168 	int slen;
169 
170 	/*
171 	 * Build a sockaddr based on the data link layer type.
172 	 * We do this at this level because the ethernet header
173 	 * is copied directly into the data field of the sockaddr.
174 	 * In the case of SLIP, there is no header and the packet
175 	 * is forwarded as is.
176 	 * Also, we are careful to leave room at the front of the mbuf
177 	 * for the link level header.
178 	 */
179 	switch (linktype) {
180 
181 	case DLT_SLIP:
182 		sockp->sa_family = AF_INET;
183 		hlen = 0;
184 		break;
185 
186 	case DLT_EN10MB:
187 		sockp->sa_family = AF_UNSPEC;
188 		/* XXX Would MAXLINKHDR be better? */
189 		hlen = ETHER_HDR_LEN;
190 		break;
191 
192 	case DLT_FDDI:
193 		sockp->sa_family = AF_IMPLINK;
194 		hlen = 0;
195 		break;
196 
197 	case DLT_RAW:
198 		sockp->sa_family = AF_UNSPEC;
199 		hlen = 0;
200 		break;
201 
202 	case DLT_NULL:
203 		/*
204 		 * null interface types require a 4 byte pseudo header which
205 		 * corresponds to the address family of the packet.
206 		 */
207 		sockp->sa_family = AF_UNSPEC;
208 		hlen = 4;
209 		break;
210 
211 	case DLT_ATM_RFC1483:
212 		/*
213 		 * en atm driver requires 4-byte atm pseudo header.
214 		 * though it isn't standard, vpi:vci needs to be
215 		 * specified anyway.
216 		 */
217 		sockp->sa_family = AF_UNSPEC;
218 		hlen = 12;	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
219 		break;
220 
221 	case DLT_PPP:
222 		sockp->sa_family = AF_UNSPEC;
223 		hlen = 4;	/* This should match PPP_HDRLEN */
224 		break;
225 
226 	case DLT_IEEE802_11:		/* IEEE 802.11 wireless */
227 		sockp->sa_family = AF_IEEE80211;
228 		hlen = 0;
229 		break;
230 
231 	case DLT_IEEE802_11_RADIO:	/* IEEE 802.11 wireless w/ phy params */
232 		sockp->sa_family = AF_IEEE80211;
233 		sockp->sa_len = 12;	/* XXX != 0 */
234 		hlen = sizeof(struct ieee80211_bpf_params);
235 		break;
236 
237 	default:
238 		return (EIO);
239 	}
240 
241 	len = uio->uio_resid;
242 
243 	if (len - hlen > mtu)
244 		return (EMSGSIZE);
245 
246 	if ((unsigned)len > MCLBYTES)
247 		return (EIO);
248 
249 	if (len > MHLEN) {
250 		m = m_getcl(M_TRYWAIT, MT_DATA, M_PKTHDR);
251 	} else {
252 		MGETHDR(m, M_TRYWAIT, MT_DATA);
253 	}
254 	if (m == NULL)
255 		return (ENOBUFS);
256 	m->m_pkthdr.len = m->m_len = len;
257 	m->m_pkthdr.rcvif = NULL;
258 	*mp = m;
259 
260 	if (m->m_len < hlen) {
261 		error = EPERM;
262 		goto bad;
263 	}
264 
265 	error = uiomove(mtod(m, u_char *), len, uio);
266 	if (error)
267 		goto bad;
268 
269 	slen = bpf_filter(wfilter, mtod(m, u_char *), len, len);
270 	if (slen == 0) {
271 		error = EPERM;
272 		goto bad;
273 	}
274 
275 	/*
276 	 * Make room for link header, and copy it to sockaddr
277 	 */
278 	if (hlen != 0) {
279 		if (sockp->sa_family == AF_IEEE80211) {
280 			/*
281 			 * Collect true length from the parameter header
282 			 * NB: sockp is known to be zero'd so if we do a
283 			 *     short copy unspecified parameters will be
284 			 *     zero.
285 			 * NB: packet may not be aligned after stripping
286 			 *     bpf params
287 			 * XXX check ibp_vers
288 			 */
289 			p = mtod(m, const struct ieee80211_bpf_params *);
290 			hlen = p->ibp_len;
291 			if (hlen > sizeof(sockp->sa_data)) {
292 				error = EINVAL;
293 				goto bad;
294 			}
295 		}
296 		bcopy(m->m_data, sockp->sa_data, hlen);
297 		m->m_pkthdr.len -= hlen;
298 		m->m_len -= hlen;
299 #if BSD >= 199103
300 		m->m_data += hlen; /* XXX */
301 #else
302 		m->m_off += hlen;
303 #endif
304 	}
305 
306 	return (0);
307 bad:
308 	m_freem(m);
309 	return (error);
310 }
311 
312 /*
313  * Attach file to the bpf interface, i.e. make d listen on bp.
314  */
315 static void
316 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
317 {
318 	/*
319 	 * Point d at bp, and add d to the interface's list of listeners.
320 	 * Finally, point the driver's bpf cookie at the interface so
321 	 * it will divert packets to bpf.
322 	 */
323 	BPFIF_LOCK(bp);
324 	d->bd_bif = bp;
325 	LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
326 
327 	bpf_bpfd_cnt++;
328 	BPFIF_UNLOCK(bp);
329 }
330 
331 /*
332  * Detach a file from its interface.
333  */
334 static void
335 bpf_detachd(struct bpf_d *d)
336 {
337 	int error;
338 	struct bpf_if *bp;
339 	struct ifnet *ifp;
340 
341 	bp = d->bd_bif;
342 	BPFIF_LOCK(bp);
343 	BPFD_LOCK(d);
344 	ifp = d->bd_bif->bif_ifp;
345 
346 	/*
347 	 * Remove d from the interface's descriptor list.
348 	 */
349 	LIST_REMOVE(d, bd_next);
350 
351 	bpf_bpfd_cnt--;
352 	d->bd_bif = NULL;
353 	BPFD_UNLOCK(d);
354 	BPFIF_UNLOCK(bp);
355 
356 	/*
357 	 * Check if this descriptor had requested promiscuous mode.
358 	 * If so, turn it off.
359 	 */
360 	if (d->bd_promisc) {
361 		d->bd_promisc = 0;
362 		error = ifpromisc(ifp, 0);
363 		if (error != 0 && error != ENXIO) {
364 			/*
365 			 * ENXIO can happen if a pccard is unplugged
366 			 * Something is really wrong if we were able to put
367 			 * the driver into promiscuous mode, but can't
368 			 * take it out.
369 			 */
370 			if_printf(bp->bif_ifp,
371 				"bpf_detach: ifpromisc failed (%d)\n", error);
372 		}
373 	}
374 }
375 
376 /*
377  * Open ethernet device.  Returns ENXIO for illegal minor device number,
378  * EBUSY if file is open by another process.
379  */
380 /* ARGSUSED */
381 static	int
382 bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
383 {
384 	struct bpf_d *d;
385 
386 	mtx_lock(&bpf_mtx);
387 	d = dev->si_drv1;
388 	/*
389 	 * Each minor can be opened by only one process.  If the requested
390 	 * minor is in use, return EBUSY.
391 	 */
392 	if (d != NULL) {
393 		mtx_unlock(&bpf_mtx);
394 		return (EBUSY);
395 	}
396 	dev->si_drv1 = (struct bpf_d *)~0;	/* mark device in use */
397 	mtx_unlock(&bpf_mtx);
398 
399 	if ((dev->si_flags & SI_NAMED) == 0)
400 		make_dev(&bpf_cdevsw, minor(dev), UID_ROOT, GID_WHEEL, 0600,
401 		    "bpf%d", dev2unit(dev));
402 	MALLOC(d, struct bpf_d *, sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
403 	dev->si_drv1 = d;
404 	d->bd_bufsize = bpf_bufsize;
405 	d->bd_sig = SIGIO;
406 	d->bd_seesent = 1;
407 	d->bd_pid = td->td_proc->p_pid;
408 #ifdef MAC
409 	mac_init_bpfdesc(d);
410 	mac_create_bpfdesc(td->td_ucred, d);
411 #endif
412 	mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF);
413 	callout_init(&d->bd_callout, NET_CALLOUT_MPSAFE);
414 	knlist_init(&d->bd_sel.si_note, &d->bd_mtx, NULL, NULL, NULL);
415 
416 	return (0);
417 }
418 
419 /*
420  * Close the descriptor by detaching it from its interface,
421  * deallocating its buffers, and marking it free.
422  */
423 /* ARGSUSED */
424 static	int
425 bpfclose(struct cdev *dev, int flags, int fmt, struct thread *td)
426 {
427 	struct bpf_d *d = dev->si_drv1;
428 
429 	BPFD_LOCK(d);
430 	if (d->bd_state == BPF_WAITING)
431 		callout_stop(&d->bd_callout);
432 	d->bd_state = BPF_IDLE;
433 	BPFD_UNLOCK(d);
434 	funsetown(&d->bd_sigio);
435 	mtx_lock(&bpf_mtx);
436 	if (d->bd_bif)
437 		bpf_detachd(d);
438 	mtx_unlock(&bpf_mtx);
439 	selwakeuppri(&d->bd_sel, PRINET);
440 #ifdef MAC
441 	mac_destroy_bpfdesc(d);
442 #endif /* MAC */
443 	knlist_destroy(&d->bd_sel.si_note);
444 	bpf_freed(d);
445 	dev->si_drv1 = NULL;
446 	free(d, M_BPF);
447 
448 	return (0);
449 }
450 
451 
452 /*
453  * Rotate the packet buffers in descriptor d.  Move the store buffer
454  * into the hold slot, and the free buffer into the store slot.
455  * Zero the length of the new store buffer.
456  */
457 #define ROTATE_BUFFERS(d) \
458 	(d)->bd_hbuf = (d)->bd_sbuf; \
459 	(d)->bd_hlen = (d)->bd_slen; \
460 	(d)->bd_sbuf = (d)->bd_fbuf; \
461 	(d)->bd_slen = 0; \
462 	(d)->bd_fbuf = NULL;
463 /*
464  *  bpfread - read next chunk of packets from buffers
465  */
466 static	int
467 bpfread(struct cdev *dev, struct uio *uio, int ioflag)
468 {
469 	struct bpf_d *d = dev->si_drv1;
470 	int timed_out;
471 	int error;
472 
473 	/*
474 	 * Restrict application to use a buffer the same size as
475 	 * as kernel buffers.
476 	 */
477 	if (uio->uio_resid != d->bd_bufsize)
478 		return (EINVAL);
479 
480 	BPFD_LOCK(d);
481 	if (d->bd_state == BPF_WAITING)
482 		callout_stop(&d->bd_callout);
483 	timed_out = (d->bd_state == BPF_TIMED_OUT);
484 	d->bd_state = BPF_IDLE;
485 	/*
486 	 * If the hold buffer is empty, then do a timed sleep, which
487 	 * ends when the timeout expires or when enough packets
488 	 * have arrived to fill the store buffer.
489 	 */
490 	while (d->bd_hbuf == NULL) {
491 		if ((d->bd_immediate || timed_out) && d->bd_slen != 0) {
492 			/*
493 			 * A packet(s) either arrived since the previous
494 			 * read or arrived while we were asleep.
495 			 * Rotate the buffers and return what's here.
496 			 */
497 			ROTATE_BUFFERS(d);
498 			break;
499 		}
500 
501 		/*
502 		 * No data is available, check to see if the bpf device
503 		 * is still pointed at a real interface.  If not, return
504 		 * ENXIO so that the userland process knows to rebind
505 		 * it before using it again.
506 		 */
507 		if (d->bd_bif == NULL) {
508 			BPFD_UNLOCK(d);
509 			return (ENXIO);
510 		}
511 
512 		if (ioflag & O_NONBLOCK) {
513 			BPFD_UNLOCK(d);
514 			return (EWOULDBLOCK);
515 		}
516 		error = msleep(d, &d->bd_mtx, PRINET|PCATCH,
517 		     "bpf", d->bd_rtout);
518 		if (error == EINTR || error == ERESTART) {
519 			BPFD_UNLOCK(d);
520 			return (error);
521 		}
522 		if (error == EWOULDBLOCK) {
523 			/*
524 			 * On a timeout, return what's in the buffer,
525 			 * which may be nothing.  If there is something
526 			 * in the store buffer, we can rotate the buffers.
527 			 */
528 			if (d->bd_hbuf)
529 				/*
530 				 * We filled up the buffer in between
531 				 * getting the timeout and arriving
532 				 * here, so we don't need to rotate.
533 				 */
534 				break;
535 
536 			if (d->bd_slen == 0) {
537 				BPFD_UNLOCK(d);
538 				return (0);
539 			}
540 			ROTATE_BUFFERS(d);
541 			break;
542 		}
543 	}
544 	/*
545 	 * At this point, we know we have something in the hold slot.
546 	 */
547 	BPFD_UNLOCK(d);
548 
549 	/*
550 	 * Move data from hold buffer into user space.
551 	 * We know the entire buffer is transferred since
552 	 * we checked above that the read buffer is bpf_bufsize bytes.
553 	 */
554 	error = uiomove(d->bd_hbuf, d->bd_hlen, uio);
555 
556 	BPFD_LOCK(d);
557 	d->bd_fbuf = d->bd_hbuf;
558 	d->bd_hbuf = NULL;
559 	d->bd_hlen = 0;
560 	BPFD_UNLOCK(d);
561 
562 	return (error);
563 }
564 
565 
566 /*
567  * If there are processes sleeping on this descriptor, wake them up.
568  */
569 static __inline void
570 bpf_wakeup(struct bpf_d *d)
571 {
572 
573 	BPFD_LOCK_ASSERT(d);
574 	if (d->bd_state == BPF_WAITING) {
575 		callout_stop(&d->bd_callout);
576 		d->bd_state = BPF_IDLE;
577 	}
578 	wakeup(d);
579 	if (d->bd_async && d->bd_sig && d->bd_sigio)
580 		pgsigio(&d->bd_sigio, d->bd_sig, 0);
581 
582 	selwakeuppri(&d->bd_sel, PRINET);
583 	KNOTE_LOCKED(&d->bd_sel.si_note, 0);
584 }
585 
586 static void
587 bpf_timed_out(void *arg)
588 {
589 	struct bpf_d *d = (struct bpf_d *)arg;
590 
591 	BPFD_LOCK(d);
592 	if (d->bd_state == BPF_WAITING) {
593 		d->bd_state = BPF_TIMED_OUT;
594 		if (d->bd_slen != 0)
595 			bpf_wakeup(d);
596 	}
597 	BPFD_UNLOCK(d);
598 }
599 
600 static int
601 bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
602 {
603 	struct bpf_d *d = dev->si_drv1;
604 	struct ifnet *ifp;
605 	struct mbuf *m;
606 	int error;
607 	struct sockaddr dst;
608 
609 	if (d->bd_bif == NULL)
610 		return (ENXIO);
611 
612 	ifp = d->bd_bif->bif_ifp;
613 
614 	if ((ifp->if_flags & IFF_UP) == 0)
615 		return (ENETDOWN);
616 
617 	if (uio->uio_resid == 0)
618 		return (0);
619 
620 	bzero(&dst, sizeof(dst));
621 	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu,
622 	    &m, &dst, d->bd_wfilter);
623 	if (error)
624 		return (error);
625 
626 	if (d->bd_hdrcmplt)
627 		dst.sa_family = pseudo_AF_HDRCMPLT;
628 
629 #ifdef MAC
630 	BPFD_LOCK(d);
631 	mac_create_mbuf_from_bpfdesc(d, m);
632 	BPFD_UNLOCK(d);
633 #endif
634 	NET_LOCK_GIANT();
635 	error = (*ifp->if_output)(ifp, m, &dst, NULL);
636 	NET_UNLOCK_GIANT();
637 	/*
638 	 * The driver frees the mbuf.
639 	 */
640 	return (error);
641 }
642 
643 /*
644  * Reset a descriptor by flushing its packet buffer and clearing the
645  * receive and drop counts.
646  */
647 static void
648 reset_d(struct bpf_d *d)
649 {
650 
651 	mtx_assert(&d->bd_mtx, MA_OWNED);
652 	if (d->bd_hbuf) {
653 		/* Free the hold buffer. */
654 		d->bd_fbuf = d->bd_hbuf;
655 		d->bd_hbuf = NULL;
656 	}
657 	d->bd_slen = 0;
658 	d->bd_hlen = 0;
659 	d->bd_rcount = 0;
660 	d->bd_dcount = 0;
661 	d->bd_fcount = 0;
662 }
663 
664 /*
665  *  FIONREAD		Check for read packet available.
666  *  SIOCGIFADDR		Get interface address - convenient hook to driver.
667  *  BIOCGBLEN		Get buffer len [for read()].
668  *  BIOCSETF		Set ethernet read filter.
669  *  BIOCSETWF		Set ethernet write filter.
670  *  BIOCFLUSH		Flush read packet buffer.
671  *  BIOCPROMISC		Put interface into promiscuous mode.
672  *  BIOCGDLT		Get link layer type.
673  *  BIOCGETIF		Get interface name.
674  *  BIOCSETIF		Set interface.
675  *  BIOCSRTIMEOUT	Set read timeout.
676  *  BIOCGRTIMEOUT	Get read timeout.
677  *  BIOCGSTATS		Get packet stats.
678  *  BIOCIMMEDIATE	Set immediate mode.
679  *  BIOCVERSION		Get filter language version.
680  *  BIOCGHDRCMPLT	Get "header already complete" flag
681  *  BIOCSHDRCMPLT	Set "header already complete" flag
682  *  BIOCGSEESENT	Get "see packets sent" flag
683  *  BIOCSSEESENT	Set "see packets sent" flag
684  *  BIOCLOCK		Set "locked" flag
685  */
686 /* ARGSUSED */
687 static	int
688 bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
689     struct thread *td)
690 {
691 	struct bpf_d *d = dev->si_drv1;
692 	int error = 0;
693 
694 	/*
695 	 * Refresh PID associated with this descriptor.
696 	 */
697 	BPFD_LOCK(d);
698 	d->bd_pid = td->td_proc->p_pid;
699 	if (d->bd_state == BPF_WAITING)
700 		callout_stop(&d->bd_callout);
701 	d->bd_state = BPF_IDLE;
702 	BPFD_UNLOCK(d);
703 
704 	if (d->bd_locked == 1) {
705 		switch (cmd) {
706 		case BIOCGBLEN:
707 		case BIOCFLUSH:
708 		case BIOCGDLT:
709 		case BIOCGDLTLIST:
710 		case BIOCGETIF:
711 		case BIOCGRTIMEOUT:
712 		case BIOCGSTATS:
713 		case BIOCVERSION:
714 		case BIOCGRSIG:
715 		case BIOCGHDRCMPLT:
716 		case FIONREAD:
717 		case BIOCLOCK:
718 		case BIOCSRTIMEOUT:
719 		case BIOCIMMEDIATE:
720 		case TIOCGPGRP:
721 			break;
722 		default:
723 			return (EPERM);
724 		}
725 	}
726 	switch (cmd) {
727 
728 	default:
729 		error = EINVAL;
730 		break;
731 
732 	/*
733 	 * Check for read packet available.
734 	 */
735 	case FIONREAD:
736 		{
737 			int n;
738 
739 			BPFD_LOCK(d);
740 			n = d->bd_slen;
741 			if (d->bd_hbuf)
742 				n += d->bd_hlen;
743 			BPFD_UNLOCK(d);
744 
745 			*(int *)addr = n;
746 			break;
747 		}
748 
749 	case SIOCGIFADDR:
750 		{
751 			struct ifnet *ifp;
752 
753 			if (d->bd_bif == NULL)
754 				error = EINVAL;
755 			else {
756 				ifp = d->bd_bif->bif_ifp;
757 				error = (*ifp->if_ioctl)(ifp, cmd, addr);
758 			}
759 			break;
760 		}
761 
762 	/*
763 	 * Get buffer len [for read()].
764 	 */
765 	case BIOCGBLEN:
766 		*(u_int *)addr = d->bd_bufsize;
767 		break;
768 
769 	/*
770 	 * Set buffer length.
771 	 */
772 	case BIOCSBLEN:
773 		if (d->bd_bif != NULL)
774 			error = EINVAL;
775 		else {
776 			u_int size = *(u_int *)addr;
777 
778 			if (size > bpf_maxbufsize)
779 				*(u_int *)addr = size = bpf_maxbufsize;
780 			else if (size < BPF_MINBUFSIZE)
781 				*(u_int *)addr = size = BPF_MINBUFSIZE;
782 			d->bd_bufsize = size;
783 		}
784 		break;
785 
786 	/*
787 	 * Set link layer read filter.
788 	 */
789 	case BIOCSETF:
790 	case BIOCSETWF:
791 		error = bpf_setf(d, (struct bpf_program *)addr, cmd);
792 		break;
793 
794 	/*
795 	 * Flush read packet buffer.
796 	 */
797 	case BIOCFLUSH:
798 		BPFD_LOCK(d);
799 		reset_d(d);
800 		BPFD_UNLOCK(d);
801 		break;
802 
803 	/*
804 	 * Put interface into promiscuous mode.
805 	 */
806 	case BIOCPROMISC:
807 		if (d->bd_bif == NULL) {
808 			/*
809 			 * No interface attached yet.
810 			 */
811 			error = EINVAL;
812 			break;
813 		}
814 		if (d->bd_promisc == 0) {
815 			mtx_lock(&Giant);
816 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
817 			mtx_unlock(&Giant);
818 			if (error == 0)
819 				d->bd_promisc = 1;
820 		}
821 		break;
822 
823 	/*
824 	 * Get current data link type.
825 	 */
826 	case BIOCGDLT:
827 		if (d->bd_bif == NULL)
828 			error = EINVAL;
829 		else
830 			*(u_int *)addr = d->bd_bif->bif_dlt;
831 		break;
832 
833 	/*
834 	 * Get a list of supported data link types.
835 	 */
836 	case BIOCGDLTLIST:
837 		if (d->bd_bif == NULL)
838 			error = EINVAL;
839 		else
840 			error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
841 		break;
842 
843 	/*
844 	 * Set data link type.
845 	 */
846 	case BIOCSDLT:
847 		if (d->bd_bif == NULL)
848 			error = EINVAL;
849 		else
850 			error = bpf_setdlt(d, *(u_int *)addr);
851 		break;
852 
853 	/*
854 	 * Get interface name.
855 	 */
856 	case BIOCGETIF:
857 		if (d->bd_bif == NULL)
858 			error = EINVAL;
859 		else {
860 			struct ifnet *const ifp = d->bd_bif->bif_ifp;
861 			struct ifreq *const ifr = (struct ifreq *)addr;
862 
863 			strlcpy(ifr->ifr_name, ifp->if_xname,
864 			    sizeof(ifr->ifr_name));
865 		}
866 		break;
867 
868 	/*
869 	 * Set interface.
870 	 */
871 	case BIOCSETIF:
872 		error = bpf_setif(d, (struct ifreq *)addr);
873 		break;
874 
875 	/*
876 	 * Set read timeout.
877 	 */
878 	case BIOCSRTIMEOUT:
879 		{
880 			struct timeval *tv = (struct timeval *)addr;
881 
882 			/*
883 			 * Subtract 1 tick from tvtohz() since this isn't
884 			 * a one-shot timer.
885 			 */
886 			if ((error = itimerfix(tv)) == 0)
887 				d->bd_rtout = tvtohz(tv) - 1;
888 			break;
889 		}
890 
891 	/*
892 	 * Get read timeout.
893 	 */
894 	case BIOCGRTIMEOUT:
895 		{
896 			struct timeval *tv = (struct timeval *)addr;
897 
898 			tv->tv_sec = d->bd_rtout / hz;
899 			tv->tv_usec = (d->bd_rtout % hz) * tick;
900 			break;
901 		}
902 
903 	/*
904 	 * Get packet stats.
905 	 */
906 	case BIOCGSTATS:
907 		{
908 			struct bpf_stat *bs = (struct bpf_stat *)addr;
909 
910 			bs->bs_recv = d->bd_rcount;
911 			bs->bs_drop = d->bd_dcount;
912 			break;
913 		}
914 
915 	/*
916 	 * Set immediate mode.
917 	 */
918 	case BIOCIMMEDIATE:
919 		d->bd_immediate = *(u_int *)addr;
920 		break;
921 
922 	case BIOCVERSION:
923 		{
924 			struct bpf_version *bv = (struct bpf_version *)addr;
925 
926 			bv->bv_major = BPF_MAJOR_VERSION;
927 			bv->bv_minor = BPF_MINOR_VERSION;
928 			break;
929 		}
930 
931 	/*
932 	 * Get "header already complete" flag
933 	 */
934 	case BIOCGHDRCMPLT:
935 		*(u_int *)addr = d->bd_hdrcmplt;
936 		break;
937 
938 	case BIOCLOCK:
939 		d->bd_locked = 1;
940 		break;
941 	/*
942 	 * Set "header already complete" flag
943 	 */
944 	case BIOCSHDRCMPLT:
945 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
946 		break;
947 
948 	/*
949 	 * Get "see sent packets" flag
950 	 */
951 	case BIOCGSEESENT:
952 		*(u_int *)addr = d->bd_seesent;
953 		break;
954 
955 	/*
956 	 * Set "see sent packets" flag
957 	 */
958 	case BIOCSSEESENT:
959 		d->bd_seesent = *(u_int *)addr;
960 		break;
961 
962 	case FIONBIO:		/* Non-blocking I/O */
963 		break;
964 
965 	case FIOASYNC:		/* Send signal on receive packets */
966 		d->bd_async = *(int *)addr;
967 		break;
968 
969 	case FIOSETOWN:
970 		error = fsetown(*(int *)addr, &d->bd_sigio);
971 		break;
972 
973 	case FIOGETOWN:
974 		*(int *)addr = fgetown(&d->bd_sigio);
975 		break;
976 
977 	/* This is deprecated, FIOSETOWN should be used instead. */
978 	case TIOCSPGRP:
979 		error = fsetown(-(*(int *)addr), &d->bd_sigio);
980 		break;
981 
982 	/* This is deprecated, FIOGETOWN should be used instead. */
983 	case TIOCGPGRP:
984 		*(int *)addr = -fgetown(&d->bd_sigio);
985 		break;
986 
987 	case BIOCSRSIG:		/* Set receive signal */
988 		{
989 			u_int sig;
990 
991 			sig = *(u_int *)addr;
992 
993 			if (sig >= NSIG)
994 				error = EINVAL;
995 			else
996 				d->bd_sig = sig;
997 			break;
998 		}
999 	case BIOCGRSIG:
1000 		*(u_int *)addr = d->bd_sig;
1001 		break;
1002 	}
1003 	return (error);
1004 }
1005 
1006 /*
1007  * Set d's packet filter program to fp.  If this file already has a filter,
1008  * free it and replace it.  Returns EINVAL for bogus requests.
1009  */
1010 static int
1011 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1012 {
1013 	struct bpf_insn *fcode, *old;
1014 	u_int wfilter, flen, size;
1015 #ifdef BPF_JITTER
1016 	bpf_jit_filter *ofunc;
1017 #endif
1018 
1019 	if (cmd == BIOCSETWF) {
1020 		old = d->bd_wfilter;
1021 		wfilter = 1;
1022 #ifdef BPF_JITTER
1023 		ofunc = NULL;
1024 #endif
1025 	} else {
1026 		wfilter = 0;
1027 		old = d->bd_rfilter;
1028 #ifdef BPF_JITTER
1029 		ofunc = d->bd_bfilter;
1030 #endif
1031 	}
1032 	if (fp->bf_insns == NULL) {
1033 		if (fp->bf_len != 0)
1034 			return (EINVAL);
1035 		BPFD_LOCK(d);
1036 		if (wfilter)
1037 			d->bd_wfilter = NULL;
1038 		else {
1039 			d->bd_rfilter = NULL;
1040 #ifdef BPF_JITTER
1041 			d->bd_bfilter = NULL;
1042 #endif
1043 		}
1044 		reset_d(d);
1045 		BPFD_UNLOCK(d);
1046 		if (old != NULL)
1047 			free((caddr_t)old, M_BPF);
1048 #ifdef BPF_JITTER
1049 		if (ofunc != NULL)
1050 			bpf_destroy_jit_filter(ofunc);
1051 #endif
1052 		return (0);
1053 	}
1054 	flen = fp->bf_len;
1055 	if (flen > bpf_maxinsns)
1056 		return (EINVAL);
1057 
1058 	size = flen * sizeof(*fp->bf_insns);
1059 	fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK);
1060 	if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 &&
1061 	    bpf_validate(fcode, (int)flen)) {
1062 		BPFD_LOCK(d);
1063 		if (wfilter)
1064 			d->bd_wfilter = fcode;
1065 		else {
1066 			d->bd_rfilter = fcode;
1067 #ifdef BPF_JITTER
1068 			d->bd_bfilter = bpf_jitter(fcode, flen);
1069 #endif
1070 		}
1071 		reset_d(d);
1072 		BPFD_UNLOCK(d);
1073 		if (old != NULL)
1074 			free((caddr_t)old, M_BPF);
1075 #ifdef BPF_JITTER
1076 		if (ofunc != NULL)
1077 			bpf_destroy_jit_filter(ofunc);
1078 #endif
1079 
1080 		return (0);
1081 	}
1082 	free((caddr_t)fcode, M_BPF);
1083 	return (EINVAL);
1084 }
1085 
1086 /*
1087  * Detach a file from its current interface (if attached at all) and attach
1088  * to the interface indicated by the name stored in ifr.
1089  * Return an errno or 0.
1090  */
1091 static int
1092 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1093 {
1094 	struct bpf_if *bp;
1095 	struct ifnet *theywant;
1096 
1097 	theywant = ifunit(ifr->ifr_name);
1098 	if (theywant == NULL || theywant->if_bpf == NULL)
1099 		return (ENXIO);
1100 
1101 	bp = theywant->if_bpf;
1102 	/*
1103 	 * Allocate the packet buffers if we need to.
1104 	 * If we're already attached to requested interface,
1105 	 * just flush the buffer.
1106 	 */
1107 	if (d->bd_sbuf == NULL)
1108 		bpf_allocbufs(d);
1109 	if (bp != d->bd_bif) {
1110 		if (d->bd_bif)
1111 			/*
1112 			 * Detach if attached to something else.
1113 			 */
1114 			bpf_detachd(d);
1115 
1116 		bpf_attachd(d, bp);
1117 	}
1118 	BPFD_LOCK(d);
1119 	reset_d(d);
1120 	BPFD_UNLOCK(d);
1121 	return (0);
1122 }
1123 
1124 /*
1125  * Support for select() and poll() system calls
1126  *
1127  * Return true iff the specific operation will not block indefinitely.
1128  * Otherwise, return false but make a note that a selwakeup() must be done.
1129  */
1130 static int
1131 bpfpoll(struct cdev *dev, int events, struct thread *td)
1132 {
1133 	struct bpf_d *d;
1134 	int revents;
1135 
1136 	d = dev->si_drv1;
1137 	if (d->bd_bif == NULL)
1138 		return (ENXIO);
1139 
1140 	/*
1141 	 * Refresh PID associated with this descriptor.
1142 	 */
1143 	revents = events & (POLLOUT | POLLWRNORM);
1144 	BPFD_LOCK(d);
1145 	d->bd_pid = td->td_proc->p_pid;
1146 	if (events & (POLLIN | POLLRDNORM)) {
1147 		if (bpf_ready(d))
1148 			revents |= events & (POLLIN | POLLRDNORM);
1149 		else {
1150 			selrecord(td, &d->bd_sel);
1151 			/* Start the read timeout if necessary. */
1152 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1153 				callout_reset(&d->bd_callout, d->bd_rtout,
1154 				    bpf_timed_out, d);
1155 				d->bd_state = BPF_WAITING;
1156 			}
1157 		}
1158 	}
1159 	BPFD_UNLOCK(d);
1160 	return (revents);
1161 }
1162 
1163 /*
1164  * Support for kevent() system call.  Register EVFILT_READ filters and
1165  * reject all others.
1166  */
1167 int
1168 bpfkqfilter(struct cdev *dev, struct knote *kn)
1169 {
1170 	struct bpf_d *d = (struct bpf_d *)dev->si_drv1;
1171 
1172 	if (kn->kn_filter != EVFILT_READ)
1173 		return (1);
1174 
1175 	/*
1176 	 * Refresh PID associated with this descriptor.
1177 	 */
1178 	BPFD_LOCK(d);
1179 	d->bd_pid = curthread->td_proc->p_pid;
1180 	kn->kn_fop = &bpfread_filtops;
1181 	kn->kn_hook = d;
1182 	knlist_add(&d->bd_sel.si_note, kn, 1);
1183 	BPFD_UNLOCK(d);
1184 
1185 	return (0);
1186 }
1187 
1188 static void
1189 filt_bpfdetach(struct knote *kn)
1190 {
1191 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1192 
1193 	knlist_remove(&d->bd_sel.si_note, kn, 0);
1194 }
1195 
1196 static int
1197 filt_bpfread(struct knote *kn, long hint)
1198 {
1199 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1200 	int ready;
1201 
1202 	BPFD_LOCK_ASSERT(d);
1203 	ready = bpf_ready(d);
1204 	if (ready) {
1205 		kn->kn_data = d->bd_slen;
1206 		if (d->bd_hbuf)
1207 			kn->kn_data += d->bd_hlen;
1208 	}
1209 	else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1210 		callout_reset(&d->bd_callout, d->bd_rtout,
1211 		    bpf_timed_out, d);
1212 		d->bd_state = BPF_WAITING;
1213 	}
1214 
1215 	return (ready);
1216 }
1217 
1218 /*
1219  * Incoming linkage from device drivers.  Process the packet pkt, of length
1220  * pktlen, which is stored in a contiguous buffer.  The packet is parsed
1221  * by each process' filter, and if accepted, stashed into the corresponding
1222  * buffer.
1223  */
1224 void
1225 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
1226 {
1227 	struct bpf_d *d;
1228 	u_int slen;
1229 	int gottime;
1230 	struct timeval tv;
1231 
1232 	gottime = 0;
1233 	BPFIF_LOCK(bp);
1234 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1235 		BPFD_LOCK(d);
1236 		++d->bd_rcount;
1237 #ifdef BPF_JITTER
1238 		if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL)
1239 			slen = (*(d->bd_bfilter->func))(pkt, pktlen, pktlen);
1240 		else
1241 #endif
1242 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
1243 		if (slen != 0) {
1244 			d->bd_fcount++;
1245 			if (!gottime) {
1246 				microtime(&tv);
1247 				gottime = 1;
1248 			}
1249 #ifdef MAC
1250 			if (mac_check_bpfdesc_receive(d, bp->bif_ifp) == 0)
1251 #endif
1252 				catchpacket(d, pkt, pktlen, slen, bcopy, &tv);
1253 		}
1254 		BPFD_UNLOCK(d);
1255 	}
1256 	BPFIF_UNLOCK(bp);
1257 }
1258 
1259 /*
1260  * Copy data from an mbuf chain into a buffer.  This code is derived
1261  * from m_copydata in sys/uipc_mbuf.c.
1262  */
1263 static void
1264 bpf_mcopy(const void *src_arg, void *dst_arg, size_t len)
1265 {
1266 	const struct mbuf *m;
1267 	u_int count;
1268 	u_char *dst;
1269 
1270 	m = src_arg;
1271 	dst = dst_arg;
1272 	while (len > 0) {
1273 		if (m == NULL)
1274 			panic("bpf_mcopy");
1275 		count = min(m->m_len, len);
1276 		bcopy(mtod(m, void *), dst, count);
1277 		m = m->m_next;
1278 		dst += count;
1279 		len -= count;
1280 	}
1281 }
1282 
1283 /*
1284  * Incoming linkage from device drivers, when packet is in an mbuf chain.
1285  */
1286 void
1287 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
1288 {
1289 	struct bpf_d *d;
1290 	u_int pktlen, slen;
1291 	int gottime;
1292 	struct timeval tv;
1293 
1294 	gottime = 0;
1295 
1296 	pktlen = m_length(m, NULL);
1297 
1298 	BPFIF_LOCK(bp);
1299 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1300 		if (!d->bd_seesent && (m->m_pkthdr.rcvif == NULL))
1301 			continue;
1302 		BPFD_LOCK(d);
1303 		++d->bd_rcount;
1304 #ifdef BPF_JITTER
1305 		/* XXX We cannot handle multiple mbufs. */
1306 		if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL &&
1307 		    m->m_next == NULL)
1308 			slen = (*(d->bd_bfilter->func))(mtod(m, u_char *),
1309 			    pktlen, pktlen);
1310 		else
1311 #endif
1312 		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
1313 		if (slen != 0) {
1314 			d->bd_fcount++;
1315 			if (!gottime) {
1316 				microtime(&tv);
1317 				gottime = 1;
1318 			}
1319 #ifdef MAC
1320 			if (mac_check_bpfdesc_receive(d, bp->bif_ifp) == 0)
1321 #endif
1322 				catchpacket(d, (u_char *)m, pktlen, slen,
1323 				    bpf_mcopy, &tv);
1324 		}
1325 		BPFD_UNLOCK(d);
1326 	}
1327 	BPFIF_UNLOCK(bp);
1328 }
1329 
1330 /*
1331  * Incoming linkage from device drivers, when packet is in
1332  * an mbuf chain and to be prepended by a contiguous header.
1333  */
1334 void
1335 bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
1336 {
1337 	struct mbuf mb;
1338 	struct bpf_d *d;
1339 	u_int pktlen, slen;
1340 	int gottime;
1341 	struct timeval tv;
1342 
1343 	gottime = 0;
1344 
1345 	pktlen = m_length(m, NULL);
1346 	/*
1347 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
1348 	 * Note that we cut corners here; we only setup what's
1349 	 * absolutely needed--this mbuf should never go anywhere else.
1350 	 */
1351 	mb.m_next = m;
1352 	mb.m_data = data;
1353 	mb.m_len = dlen;
1354 	pktlen += dlen;
1355 
1356 	BPFIF_LOCK(bp);
1357 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1358 		if (!d->bd_seesent && (m->m_pkthdr.rcvif == NULL))
1359 			continue;
1360 		BPFD_LOCK(d);
1361 		++d->bd_rcount;
1362 		slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
1363 		if (slen != 0) {
1364 			d->bd_fcount++;
1365 			if (!gottime) {
1366 				microtime(&tv);
1367 				gottime = 1;
1368 			}
1369 #ifdef MAC
1370 			if (mac_check_bpfdesc_receive(d, bp->bif_ifp) == 0)
1371 #endif
1372 				catchpacket(d, (u_char *)&mb, pktlen, slen,
1373 				    bpf_mcopy, &tv);
1374 		}
1375 		BPFD_UNLOCK(d);
1376 	}
1377 	BPFIF_UNLOCK(bp);
1378 }
1379 
1380 /*
1381  * Move the packet data from interface memory (pkt) into the
1382  * store buffer.  "cpfn" is the routine called to do the actual data
1383  * transfer.  bcopy is passed in to copy contiguous chunks, while
1384  * bpf_mcopy is passed in to copy mbuf chains.  In the latter case,
1385  * pkt is really an mbuf.
1386  */
1387 static void
1388 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
1389     void (*cpfn)(const void *, void *, size_t), struct timeval *tv)
1390 {
1391 	struct bpf_hdr *hp;
1392 	int totlen, curlen;
1393 	int hdrlen = d->bd_bif->bif_hdrlen;
1394 	int do_wakeup = 0;
1395 
1396 	BPFD_LOCK_ASSERT(d);
1397 	/*
1398 	 * Figure out how many bytes to move.  If the packet is
1399 	 * greater or equal to the snapshot length, transfer that
1400 	 * much.  Otherwise, transfer the whole packet (unless
1401 	 * we hit the buffer size limit).
1402 	 */
1403 	totlen = hdrlen + min(snaplen, pktlen);
1404 	if (totlen > d->bd_bufsize)
1405 		totlen = d->bd_bufsize;
1406 
1407 	/*
1408 	 * Round up the end of the previous packet to the next longword.
1409 	 */
1410 	curlen = BPF_WORDALIGN(d->bd_slen);
1411 	if (curlen + totlen > d->bd_bufsize) {
1412 		/*
1413 		 * This packet will overflow the storage buffer.
1414 		 * Rotate the buffers if we can, then wakeup any
1415 		 * pending reads.
1416 		 */
1417 		if (d->bd_fbuf == NULL) {
1418 			/*
1419 			 * We haven't completed the previous read yet,
1420 			 * so drop the packet.
1421 			 */
1422 			++d->bd_dcount;
1423 			return;
1424 		}
1425 		ROTATE_BUFFERS(d);
1426 		do_wakeup = 1;
1427 		curlen = 0;
1428 	}
1429 	else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT)
1430 		/*
1431 		 * Immediate mode is set, or the read timeout has
1432 		 * already expired during a select call.  A packet
1433 		 * arrived, so the reader should be woken up.
1434 		 */
1435 		do_wakeup = 1;
1436 
1437 	/*
1438 	 * Append the bpf header.
1439 	 */
1440 	hp = (struct bpf_hdr *)(d->bd_sbuf + curlen);
1441 	hp->bh_tstamp = *tv;
1442 	hp->bh_datalen = pktlen;
1443 	hp->bh_hdrlen = hdrlen;
1444 	/*
1445 	 * Copy the packet data into the store buffer and update its length.
1446 	 */
1447 	(*cpfn)(pkt, (u_char *)hp + hdrlen, (hp->bh_caplen = totlen - hdrlen));
1448 	d->bd_slen = curlen + totlen;
1449 
1450 	if (do_wakeup)
1451 		bpf_wakeup(d);
1452 }
1453 
1454 /*
1455  * Initialize all nonzero fields of a descriptor.
1456  */
1457 static void
1458 bpf_allocbufs(struct bpf_d *d)
1459 {
1460 
1461 	KASSERT(d->bd_fbuf == NULL, ("bpf_allocbufs: bd_fbuf != NULL"));
1462 	KASSERT(d->bd_sbuf == NULL, ("bpf_allocbufs: bd_sbuf != NULL"));
1463 	KASSERT(d->bd_hbuf == NULL, ("bpf_allocbufs: bd_hbuf != NULL"));
1464 
1465 	d->bd_fbuf = (caddr_t)malloc(d->bd_bufsize, M_BPF, M_WAITOK);
1466 	d->bd_sbuf = (caddr_t)malloc(d->bd_bufsize, M_BPF, M_WAITOK);
1467 	d->bd_slen = 0;
1468 	d->bd_hlen = 0;
1469 }
1470 
1471 /*
1472  * Free buffers currently in use by a descriptor.
1473  * Called on close.
1474  */
1475 static void
1476 bpf_freed(struct bpf_d *d)
1477 {
1478 	/*
1479 	 * We don't need to lock out interrupts since this descriptor has
1480 	 * been detached from its interface and it yet hasn't been marked
1481 	 * free.
1482 	 */
1483 	if (d->bd_sbuf != NULL) {
1484 		free(d->bd_sbuf, M_BPF);
1485 		if (d->bd_hbuf != NULL)
1486 			free(d->bd_hbuf, M_BPF);
1487 		if (d->bd_fbuf != NULL)
1488 			free(d->bd_fbuf, M_BPF);
1489 	}
1490 	if (d->bd_rfilter) {
1491 		free((caddr_t)d->bd_rfilter, M_BPF);
1492 #ifdef BPF_JITTER
1493 		bpf_destroy_jit_filter(d->bd_bfilter);
1494 #endif
1495 	}
1496 	if (d->bd_wfilter)
1497 		free((caddr_t)d->bd_wfilter, M_BPF);
1498 	mtx_destroy(&d->bd_mtx);
1499 }
1500 
1501 /*
1502  * Attach an interface to bpf.  dlt is the link layer type; hdrlen is the
1503  * fixed size of the link header (variable length headers not yet supported).
1504  */
1505 void
1506 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
1507 {
1508 
1509 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
1510 }
1511 
1512 /*
1513  * Attach an interface to bpf.  ifp is a pointer to the structure
1514  * defining the interface to be attached, dlt is the link layer type,
1515  * and hdrlen is the fixed size of the link header (variable length
1516  * headers are not yet supporrted).
1517  */
1518 void
1519 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
1520 {
1521 	struct bpf_if *bp;
1522 
1523 	bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO);
1524 	if (bp == NULL)
1525 		panic("bpfattach");
1526 
1527 	LIST_INIT(&bp->bif_dlist);
1528 	bp->bif_ifp = ifp;
1529 	bp->bif_dlt = dlt;
1530 	mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF);
1531 	KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized"));
1532 	*driverp = bp;
1533 
1534 	mtx_lock(&bpf_mtx);
1535 	LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
1536 	mtx_unlock(&bpf_mtx);
1537 
1538 	/*
1539 	 * Compute the length of the bpf header.  This is not necessarily
1540 	 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
1541 	 * that the network layer header begins on a longword boundary (for
1542 	 * performance reasons and to alleviate alignment restrictions).
1543 	 */
1544 	bp->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
1545 
1546 	if (bootverbose)
1547 		if_printf(ifp, "bpf attached\n");
1548 }
1549 
1550 /*
1551  * Detach bpf from an interface.  This involves detaching each descriptor
1552  * associated with the interface, and leaving bd_bif NULL.  Notify each
1553  * descriptor as it's detached so that any sleepers wake up and get
1554  * ENXIO.
1555  */
1556 void
1557 bpfdetach(struct ifnet *ifp)
1558 {
1559 	struct bpf_if	*bp;
1560 	struct bpf_d	*d;
1561 
1562 	/* Locate BPF interface information */
1563 	mtx_lock(&bpf_mtx);
1564 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1565 		if (ifp == bp->bif_ifp)
1566 			break;
1567 	}
1568 
1569 	/* Interface wasn't attached */
1570 	if ((bp == NULL) || (bp->bif_ifp == NULL)) {
1571 		mtx_unlock(&bpf_mtx);
1572 		printf("bpfdetach: %s was not attached\n", ifp->if_xname);
1573 		return;
1574 	}
1575 
1576 	LIST_REMOVE(bp, bif_next);
1577 	mtx_unlock(&bpf_mtx);
1578 
1579 	while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) {
1580 		bpf_detachd(d);
1581 		BPFD_LOCK(d);
1582 		bpf_wakeup(d);
1583 		BPFD_UNLOCK(d);
1584 	}
1585 
1586 	mtx_destroy(&bp->bif_mtx);
1587 	free(bp, M_BPF);
1588 }
1589 
1590 /*
1591  * Get a list of available data link type of the interface.
1592  */
1593 static int
1594 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
1595 {
1596 	int n, error;
1597 	struct ifnet *ifp;
1598 	struct bpf_if *bp;
1599 
1600 	ifp = d->bd_bif->bif_ifp;
1601 	n = 0;
1602 	error = 0;
1603 	mtx_lock(&bpf_mtx);
1604 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1605 		if (bp->bif_ifp != ifp)
1606 			continue;
1607 		if (bfl->bfl_list != NULL) {
1608 			if (n >= bfl->bfl_len) {
1609 				mtx_unlock(&bpf_mtx);
1610 				return (ENOMEM);
1611 			}
1612 			error = copyout(&bp->bif_dlt,
1613 			    bfl->bfl_list + n, sizeof(u_int));
1614 		}
1615 		n++;
1616 	}
1617 	mtx_unlock(&bpf_mtx);
1618 	bfl->bfl_len = n;
1619 	return (error);
1620 }
1621 
1622 /*
1623  * Set the data link type of a BPF instance.
1624  */
1625 static int
1626 bpf_setdlt(struct bpf_d *d, u_int dlt)
1627 {
1628 	int error, opromisc;
1629 	struct ifnet *ifp;
1630 	struct bpf_if *bp;
1631 
1632 	if (d->bd_bif->bif_dlt == dlt)
1633 		return (0);
1634 	ifp = d->bd_bif->bif_ifp;
1635 	mtx_lock(&bpf_mtx);
1636 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1637 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
1638 			break;
1639 	}
1640 	mtx_unlock(&bpf_mtx);
1641 	if (bp != NULL) {
1642 		opromisc = d->bd_promisc;
1643 		bpf_detachd(d);
1644 		bpf_attachd(d, bp);
1645 		BPFD_LOCK(d);
1646 		reset_d(d);
1647 		BPFD_UNLOCK(d);
1648 		if (opromisc) {
1649 			error = ifpromisc(bp->bif_ifp, 1);
1650 			if (error)
1651 				if_printf(bp->bif_ifp,
1652 					"bpf_setdlt: ifpromisc failed (%d)\n",
1653 					error);
1654 			else
1655 				d->bd_promisc = 1;
1656 		}
1657 	}
1658 	return (bp == NULL ? EINVAL : 0);
1659 }
1660 
1661 static void
1662 bpf_clone(void *arg, struct ucred *cred, char *name, int namelen,
1663     struct cdev **dev)
1664 {
1665 	int u;
1666 
1667 	if (*dev != NULL)
1668 		return;
1669 	if (dev_stdclone(name, NULL, "bpf", &u) != 1)
1670 		return;
1671 	*dev = make_dev(&bpf_cdevsw, unit2minor(u), UID_ROOT, GID_WHEEL, 0600,
1672 	    "bpf%d", u);
1673 	dev_ref(*dev);
1674 	(*dev)->si_flags |= SI_CHEAPCLONE;
1675 	return;
1676 }
1677 
1678 static void
1679 bpf_drvinit(void *unused)
1680 {
1681 
1682 	mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF);
1683 	LIST_INIT(&bpf_iflist);
1684 	EVENTHANDLER_REGISTER(dev_clone, bpf_clone, 0, 1000);
1685 }
1686 
1687 static void
1688 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
1689 {
1690 
1691 	bzero(d, sizeof(*d));
1692 	BPFD_LOCK_ASSERT(bd);
1693 	d->bd_immediate = bd->bd_immediate;
1694 	d->bd_promisc = bd->bd_promisc;
1695 	d->bd_hdrcmplt = bd->bd_hdrcmplt;
1696 	d->bd_seesent = bd->bd_seesent;
1697 	d->bd_async = bd->bd_async;
1698 	d->bd_rcount = bd->bd_rcount;
1699 	d->bd_dcount = bd->bd_dcount;
1700 	d->bd_fcount = bd->bd_fcount;
1701 	d->bd_sig = bd->bd_sig;
1702 	d->bd_slen = bd->bd_slen;
1703 	d->bd_hlen = bd->bd_hlen;
1704 	d->bd_bufsize = bd->bd_bufsize;
1705 	d->bd_pid = bd->bd_pid;
1706 	strlcpy(d->bd_ifname,
1707 	    bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
1708 	d->bd_locked = bd->bd_locked;
1709 }
1710 
1711 static int
1712 bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
1713 {
1714 	struct xbpf_d *xbdbuf, *xbd;
1715 	int index, error;
1716 	struct bpf_if *bp;
1717 	struct bpf_d *bd;
1718 
1719 	/*
1720 	 * XXX This is not technically correct. It is possible for non
1721 	 * privileged users to open bpf devices. It would make sense
1722 	 * if the users who opened the devices were able to retrieve
1723 	 * the statistics for them, too.
1724 	 */
1725 	error = priv_check(req->td, PRIV_NET_BPF);
1726 	if (error)
1727 		return (error);
1728 	if (req->oldptr == NULL)
1729 		return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
1730 	if (bpf_bpfd_cnt == 0)
1731 		return (SYSCTL_OUT(req, 0, 0));
1732 	xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
1733 	mtx_lock(&bpf_mtx);
1734 	if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
1735 		mtx_unlock(&bpf_mtx);
1736 		free(xbdbuf, M_BPF);
1737 		return (ENOMEM);
1738 	}
1739 	index = 0;
1740 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
1741 		BPFIF_LOCK(bp);
1742 		LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
1743 			xbd = &xbdbuf[index++];
1744 			BPFD_LOCK(bd);
1745 			bpfstats_fill_xbpf(xbd, bd);
1746 			BPFD_UNLOCK(bd);
1747 		}
1748 		BPFIF_UNLOCK(bp);
1749 	}
1750 	mtx_unlock(&bpf_mtx);
1751 	error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
1752 	free(xbdbuf, M_BPF);
1753 	return (error);
1754 }
1755 
1756 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL)
1757 
1758 #else /* !DEV_BPF && !NETGRAPH_BPF */
1759 /*
1760  * NOP stubs to allow bpf-using drivers to load and function.
1761  *
1762  * A 'better' implementation would allow the core bpf functionality
1763  * to be loaded at runtime.
1764  */
1765 static struct bpf_if bp_null;
1766 
1767 void
1768 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
1769 {
1770 }
1771 
1772 void
1773 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
1774 {
1775 }
1776 
1777 void
1778 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
1779 {
1780 }
1781 
1782 void
1783 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
1784 {
1785 
1786 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
1787 }
1788 
1789 void
1790 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
1791 {
1792 
1793 	*driverp = &bp_null;
1794 }
1795 
1796 void
1797 bpfdetach(struct ifnet *ifp)
1798 {
1799 }
1800 
1801 u_int
1802 bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
1803 {
1804 	return -1;	/* "no filter" behaviour */
1805 }
1806 
1807 int
1808 bpf_validate(const struct bpf_insn *f, int len)
1809 {
1810 	return 0;		/* false */
1811 }
1812 
1813 #endif /* !DEV_BPF && !NETGRAPH_BPF */
1814