xref: /freebsd/sys/net/bpf.c (revision 952364486a4b9d135e4b28f7f88a8703a74eae6f)
1 /*-
2  * Copyright (c) 1990, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from the Stanford/CMU enet packet filter,
6  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
7  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
8  * Berkeley Laboratory.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  *    may be used to endorse or promote products derived from this software
20  *    without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  *      @(#)bpf.c	8.4 (Berkeley) 1/9/95
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include "opt_bpf.h"
41 #include "opt_compat.h"
42 #include "opt_netgraph.h"
43 
44 #include <sys/types.h>
45 #include <sys/param.h>
46 #include <sys/lock.h>
47 #include <sys/rwlock.h>
48 #include <sys/systm.h>
49 #include <sys/conf.h>
50 #include <sys/fcntl.h>
51 #include <sys/jail.h>
52 #include <sys/malloc.h>
53 #include <sys/mbuf.h>
54 #include <sys/time.h>
55 #include <sys/priv.h>
56 #include <sys/proc.h>
57 #include <sys/signalvar.h>
58 #include <sys/filio.h>
59 #include <sys/sockio.h>
60 #include <sys/ttycom.h>
61 #include <sys/uio.h>
62 
63 #include <sys/event.h>
64 #include <sys/file.h>
65 #include <sys/poll.h>
66 #include <sys/proc.h>
67 
68 #include <sys/socket.h>
69 
70 #include <net/if.h>
71 #include <net/if_var.h>
72 #include <net/bpf.h>
73 #include <net/bpf_buffer.h>
74 #ifdef BPF_JITTER
75 #include <net/bpf_jitter.h>
76 #endif
77 #include <net/bpf_zerocopy.h>
78 #include <net/bpfdesc.h>
79 #include <net/vnet.h>
80 
81 #include <netinet/in.h>
82 #include <netinet/if_ether.h>
83 #include <sys/kernel.h>
84 #include <sys/sysctl.h>
85 
86 #include <net80211/ieee80211_freebsd.h>
87 
88 #include <security/mac/mac_framework.h>
89 
90 MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
91 
92 struct bpf_if {
93 #define	bif_next	bif_ext.bif_next
94 #define	bif_dlist	bif_ext.bif_dlist
95 	struct bpf_if_ext bif_ext;	/* public members */
96 	u_int		bif_dlt;	/* link layer type */
97 	u_int		bif_hdrlen;	/* length of link header */
98 	struct ifnet	*bif_ifp;	/* corresponding interface */
99 	struct rwlock	bif_lock;	/* interface lock */
100 	LIST_HEAD(, bpf_d) bif_wlist;	/* writer-only list */
101 	int		bif_flags;	/* Interface flags */
102 };
103 
104 CTASSERT(offsetof(struct bpf_if, bif_ext) == 0);
105 
106 #if defined(DEV_BPF) || defined(NETGRAPH_BPF)
107 
108 #define PRINET  26			/* interruptible */
109 
110 #define	SIZEOF_BPF_HDR(type)	\
111     (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
112 
113 #ifdef COMPAT_FREEBSD32
114 #include <sys/mount.h>
115 #include <compat/freebsd32/freebsd32.h>
116 #define BPF_ALIGNMENT32 sizeof(int32_t)
117 #define BPF_WORDALIGN32(x) (((x)+(BPF_ALIGNMENT32-1))&~(BPF_ALIGNMENT32-1))
118 
119 #ifndef BURN_BRIDGES
120 /*
121  * 32-bit version of structure prepended to each packet.  We use this header
122  * instead of the standard one for 32-bit streams.  We mark the a stream as
123  * 32-bit the first time we see a 32-bit compat ioctl request.
124  */
125 struct bpf_hdr32 {
126 	struct timeval32 bh_tstamp;	/* time stamp */
127 	uint32_t	bh_caplen;	/* length of captured portion */
128 	uint32_t	bh_datalen;	/* original length of packet */
129 	uint16_t	bh_hdrlen;	/* length of bpf header (this struct
130 					   plus alignment padding) */
131 };
132 #endif
133 
134 struct bpf_program32 {
135 	u_int bf_len;
136 	uint32_t bf_insns;
137 };
138 
139 struct bpf_dltlist32 {
140 	u_int	bfl_len;
141 	u_int	bfl_list;
142 };
143 
144 #define	BIOCSETF32	_IOW('B', 103, struct bpf_program32)
145 #define	BIOCSRTIMEOUT32	_IOW('B', 109, struct timeval32)
146 #define	BIOCGRTIMEOUT32	_IOR('B', 110, struct timeval32)
147 #define	BIOCGDLTLIST32	_IOWR('B', 121, struct bpf_dltlist32)
148 #define	BIOCSETWF32	_IOW('B', 123, struct bpf_program32)
149 #define	BIOCSETFNR32	_IOW('B', 130, struct bpf_program32)
150 #endif
151 
152 /*
153  * bpf_iflist is a list of BPF interface structures, each corresponding to a
154  * specific DLT.  The same network interface might have several BPF interface
155  * structures registered by different layers in the stack (i.e., 802.11
156  * frames, ethernet frames, etc).
157  */
158 static LIST_HEAD(, bpf_if)	bpf_iflist, bpf_freelist;
159 static struct mtx	bpf_mtx;		/* bpf global lock */
160 static int		bpf_bpfd_cnt;
161 
162 static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
163 static void	bpf_detachd(struct bpf_d *);
164 static void	bpf_detachd_locked(struct bpf_d *);
165 static void	bpf_freed(struct bpf_d *);
166 static int	bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **,
167 		    struct sockaddr *, int *, struct bpf_insn *);
168 static int	bpf_setif(struct bpf_d *, struct ifreq *);
169 static void	bpf_timed_out(void *);
170 static __inline void
171 		bpf_wakeup(struct bpf_d *);
172 static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
173 		    void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
174 		    struct bintime *);
175 static void	reset_d(struct bpf_d *);
176 static int	bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
177 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
178 static int	bpf_setdlt(struct bpf_d *, u_int);
179 static void	filt_bpfdetach(struct knote *);
180 static int	filt_bpfread(struct knote *, long);
181 static void	bpf_drvinit(void *);
182 static int	bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
183 
184 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl");
185 int bpf_maxinsns = BPF_MAXINSNS;
186 SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
187     &bpf_maxinsns, 0, "Maximum bpf program instructions");
188 static int bpf_zerocopy_enable = 0;
189 SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
190     &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
191 static SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
192     bpf_stats_sysctl, "bpf statistics portal");
193 
194 static VNET_DEFINE(int, bpf_optimize_writers) = 0;
195 #define	V_bpf_optimize_writers VNET(bpf_optimize_writers)
196 SYSCTL_INT(_net_bpf, OID_AUTO, optimize_writers, CTLFLAG_VNET | CTLFLAG_RW,
197     &VNET_NAME(bpf_optimize_writers), 0,
198     "Do not send packets until BPF program is set");
199 
200 static	d_open_t	bpfopen;
201 static	d_read_t	bpfread;
202 static	d_write_t	bpfwrite;
203 static	d_ioctl_t	bpfioctl;
204 static	d_poll_t	bpfpoll;
205 static	d_kqfilter_t	bpfkqfilter;
206 
207 static struct cdevsw bpf_cdevsw = {
208 	.d_version =	D_VERSION,
209 	.d_open =	bpfopen,
210 	.d_read =	bpfread,
211 	.d_write =	bpfwrite,
212 	.d_ioctl =	bpfioctl,
213 	.d_poll =	bpfpoll,
214 	.d_name =	"bpf",
215 	.d_kqfilter =	bpfkqfilter,
216 };
217 
218 static struct filterops bpfread_filtops = {
219 	.f_isfd = 1,
220 	.f_detach = filt_bpfdetach,
221 	.f_event = filt_bpfread,
222 };
223 
224 eventhandler_tag	bpf_ifdetach_cookie = NULL;
225 
226 /*
227  * LOCKING MODEL USED BY BPF:
228  * Locks:
229  * 1) global lock (BPF_LOCK). Mutex, used to protect interface addition/removal,
230  * some global counters and every bpf_if reference.
231  * 2) Interface lock. Rwlock, used to protect list of BPF descriptors and their filters.
232  * 3) Descriptor lock. Mutex, used to protect BPF buffers and various structure fields
233  *   used by bpf_mtap code.
234  *
235  * Lock order:
236  *
237  * Global lock, interface lock, descriptor lock
238  *
239  * We have to acquire interface lock before descriptor main lock due to BPF_MTAP[2]
240  * working model. In many places (like bpf_detachd) we start with BPF descriptor
241  * (and we need to at least rlock it to get reliable interface pointer). This
242  * gives us potential LOR. As a result, we use global lock to protect from bpf_if
243  * change in every such place.
244  *
245  * Changing d->bd_bif is protected by 1) global lock, 2) interface lock and
246  * 3) descriptor main wlock.
247  * Reading bd_bif can be protected by any of these locks, typically global lock.
248  *
249  * Changing read/write BPF filter is protected by the same three locks,
250  * the same applies for reading.
251  *
252  * Sleeping in global lock is not allowed due to bpfdetach() using it.
253  */
254 
255 /*
256  * Wrapper functions for various buffering methods.  If the set of buffer
257  * modes expands, we will probably want to introduce a switch data structure
258  * similar to protosw, et.
259  */
260 static void
261 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
262     u_int len)
263 {
264 
265 	BPFD_LOCK_ASSERT(d);
266 
267 	switch (d->bd_bufmode) {
268 	case BPF_BUFMODE_BUFFER:
269 		return (bpf_buffer_append_bytes(d, buf, offset, src, len));
270 
271 	case BPF_BUFMODE_ZBUF:
272 		d->bd_zcopy++;
273 		return (bpf_zerocopy_append_bytes(d, buf, offset, src, len));
274 
275 	default:
276 		panic("bpf_buf_append_bytes");
277 	}
278 }
279 
280 static void
281 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
282     u_int len)
283 {
284 
285 	BPFD_LOCK_ASSERT(d);
286 
287 	switch (d->bd_bufmode) {
288 	case BPF_BUFMODE_BUFFER:
289 		return (bpf_buffer_append_mbuf(d, buf, offset, src, len));
290 
291 	case BPF_BUFMODE_ZBUF:
292 		d->bd_zcopy++;
293 		return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len));
294 
295 	default:
296 		panic("bpf_buf_append_mbuf");
297 	}
298 }
299 
300 /*
301  * This function gets called when the free buffer is re-assigned.
302  */
303 static void
304 bpf_buf_reclaimed(struct bpf_d *d)
305 {
306 
307 	BPFD_LOCK_ASSERT(d);
308 
309 	switch (d->bd_bufmode) {
310 	case BPF_BUFMODE_BUFFER:
311 		return;
312 
313 	case BPF_BUFMODE_ZBUF:
314 		bpf_zerocopy_buf_reclaimed(d);
315 		return;
316 
317 	default:
318 		panic("bpf_buf_reclaimed");
319 	}
320 }
321 
322 /*
323  * If the buffer mechanism has a way to decide that a held buffer can be made
324  * free, then it is exposed via the bpf_canfreebuf() interface.  (1) is
325  * returned if the buffer can be discarded, (0) is returned if it cannot.
326  */
327 static int
328 bpf_canfreebuf(struct bpf_d *d)
329 {
330 
331 	BPFD_LOCK_ASSERT(d);
332 
333 	switch (d->bd_bufmode) {
334 	case BPF_BUFMODE_ZBUF:
335 		return (bpf_zerocopy_canfreebuf(d));
336 	}
337 	return (0);
338 }
339 
340 /*
341  * Allow the buffer model to indicate that the current store buffer is
342  * immutable, regardless of the appearance of space.  Return (1) if the
343  * buffer is writable, and (0) if not.
344  */
345 static int
346 bpf_canwritebuf(struct bpf_d *d)
347 {
348 	BPFD_LOCK_ASSERT(d);
349 
350 	switch (d->bd_bufmode) {
351 	case BPF_BUFMODE_ZBUF:
352 		return (bpf_zerocopy_canwritebuf(d));
353 	}
354 	return (1);
355 }
356 
357 /*
358  * Notify buffer model that an attempt to write to the store buffer has
359  * resulted in a dropped packet, in which case the buffer may be considered
360  * full.
361  */
362 static void
363 bpf_buffull(struct bpf_d *d)
364 {
365 
366 	BPFD_LOCK_ASSERT(d);
367 
368 	switch (d->bd_bufmode) {
369 	case BPF_BUFMODE_ZBUF:
370 		bpf_zerocopy_buffull(d);
371 		break;
372 	}
373 }
374 
375 /*
376  * Notify the buffer model that a buffer has moved into the hold position.
377  */
378 void
379 bpf_bufheld(struct bpf_d *d)
380 {
381 
382 	BPFD_LOCK_ASSERT(d);
383 
384 	switch (d->bd_bufmode) {
385 	case BPF_BUFMODE_ZBUF:
386 		bpf_zerocopy_bufheld(d);
387 		break;
388 	}
389 }
390 
391 static void
392 bpf_free(struct bpf_d *d)
393 {
394 
395 	switch (d->bd_bufmode) {
396 	case BPF_BUFMODE_BUFFER:
397 		return (bpf_buffer_free(d));
398 
399 	case BPF_BUFMODE_ZBUF:
400 		return (bpf_zerocopy_free(d));
401 
402 	default:
403 		panic("bpf_buf_free");
404 	}
405 }
406 
407 static int
408 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio)
409 {
410 
411 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
412 		return (EOPNOTSUPP);
413 	return (bpf_buffer_uiomove(d, buf, len, uio));
414 }
415 
416 static int
417 bpf_ioctl_sblen(struct bpf_d *d, u_int *i)
418 {
419 
420 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
421 		return (EOPNOTSUPP);
422 	return (bpf_buffer_ioctl_sblen(d, i));
423 }
424 
425 static int
426 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
427 {
428 
429 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
430 		return (EOPNOTSUPP);
431 	return (bpf_zerocopy_ioctl_getzmax(td, d, i));
432 }
433 
434 static int
435 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
436 {
437 
438 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
439 		return (EOPNOTSUPP);
440 	return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz));
441 }
442 
443 static int
444 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
445 {
446 
447 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
448 		return (EOPNOTSUPP);
449 	return (bpf_zerocopy_ioctl_setzbuf(td, d, bz));
450 }
451 
452 /*
453  * General BPF functions.
454  */
455 static int
456 bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
457     struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter)
458 {
459 	const struct ieee80211_bpf_params *p;
460 	struct ether_header *eh;
461 	struct mbuf *m;
462 	int error;
463 	int len;
464 	int hlen;
465 	int slen;
466 
467 	/*
468 	 * Build a sockaddr based on the data link layer type.
469 	 * We do this at this level because the ethernet header
470 	 * is copied directly into the data field of the sockaddr.
471 	 * In the case of SLIP, there is no header and the packet
472 	 * is forwarded as is.
473 	 * Also, we are careful to leave room at the front of the mbuf
474 	 * for the link level header.
475 	 */
476 	switch (linktype) {
477 
478 	case DLT_SLIP:
479 		sockp->sa_family = AF_INET;
480 		hlen = 0;
481 		break;
482 
483 	case DLT_EN10MB:
484 		sockp->sa_family = AF_UNSPEC;
485 		/* XXX Would MAXLINKHDR be better? */
486 		hlen = ETHER_HDR_LEN;
487 		break;
488 
489 	case DLT_FDDI:
490 		sockp->sa_family = AF_IMPLINK;
491 		hlen = 0;
492 		break;
493 
494 	case DLT_RAW:
495 		sockp->sa_family = AF_UNSPEC;
496 		hlen = 0;
497 		break;
498 
499 	case DLT_NULL:
500 		/*
501 		 * null interface types require a 4 byte pseudo header which
502 		 * corresponds to the address family of the packet.
503 		 */
504 		sockp->sa_family = AF_UNSPEC;
505 		hlen = 4;
506 		break;
507 
508 	case DLT_ATM_RFC1483:
509 		/*
510 		 * en atm driver requires 4-byte atm pseudo header.
511 		 * though it isn't standard, vpi:vci needs to be
512 		 * specified anyway.
513 		 */
514 		sockp->sa_family = AF_UNSPEC;
515 		hlen = 12;	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
516 		break;
517 
518 	case DLT_PPP:
519 		sockp->sa_family = AF_UNSPEC;
520 		hlen = 4;	/* This should match PPP_HDRLEN */
521 		break;
522 
523 	case DLT_IEEE802_11:		/* IEEE 802.11 wireless */
524 		sockp->sa_family = AF_IEEE80211;
525 		hlen = 0;
526 		break;
527 
528 	case DLT_IEEE802_11_RADIO:	/* IEEE 802.11 wireless w/ phy params */
529 		sockp->sa_family = AF_IEEE80211;
530 		sockp->sa_len = 12;	/* XXX != 0 */
531 		hlen = sizeof(struct ieee80211_bpf_params);
532 		break;
533 
534 	default:
535 		return (EIO);
536 	}
537 
538 	len = uio->uio_resid;
539 	if (len < hlen || len - hlen > ifp->if_mtu)
540 		return (EMSGSIZE);
541 
542 	m = m_get2(len, M_WAITOK, MT_DATA, M_PKTHDR);
543 	if (m == NULL)
544 		return (EIO);
545 	m->m_pkthdr.len = m->m_len = len;
546 	*mp = m;
547 
548 	error = uiomove(mtod(m, u_char *), len, uio);
549 	if (error)
550 		goto bad;
551 
552 	slen = bpf_filter(wfilter, mtod(m, u_char *), len, len);
553 	if (slen == 0) {
554 		error = EPERM;
555 		goto bad;
556 	}
557 
558 	/* Check for multicast destination */
559 	switch (linktype) {
560 	case DLT_EN10MB:
561 		eh = mtod(m, struct ether_header *);
562 		if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
563 			if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost,
564 			    ETHER_ADDR_LEN) == 0)
565 				m->m_flags |= M_BCAST;
566 			else
567 				m->m_flags |= M_MCAST;
568 		}
569 		break;
570 	}
571 
572 	/*
573 	 * Make room for link header, and copy it to sockaddr
574 	 */
575 	if (hlen != 0) {
576 		if (sockp->sa_family == AF_IEEE80211) {
577 			/*
578 			 * Collect true length from the parameter header
579 			 * NB: sockp is known to be zero'd so if we do a
580 			 *     short copy unspecified parameters will be
581 			 *     zero.
582 			 * NB: packet may not be aligned after stripping
583 			 *     bpf params
584 			 * XXX check ibp_vers
585 			 */
586 			p = mtod(m, const struct ieee80211_bpf_params *);
587 			hlen = p->ibp_len;
588 			if (hlen > sizeof(sockp->sa_data)) {
589 				error = EINVAL;
590 				goto bad;
591 			}
592 		}
593 		bcopy(mtod(m, const void *), sockp->sa_data, hlen);
594 	}
595 	*hdrlen = hlen;
596 
597 	return (0);
598 bad:
599 	m_freem(m);
600 	return (error);
601 }
602 
603 /*
604  * Attach file to the bpf interface, i.e. make d listen on bp.
605  */
606 static void
607 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
608 {
609 	int op_w;
610 
611 	BPF_LOCK_ASSERT();
612 
613 	/*
614 	 * Save sysctl value to protect from sysctl change
615 	 * between reads
616 	 */
617 	op_w = V_bpf_optimize_writers || d->bd_writer;
618 
619 	if (d->bd_bif != NULL)
620 		bpf_detachd_locked(d);
621 	/*
622 	 * Point d at bp, and add d to the interface's list.
623 	 * Since there are many applicaiotns using BPF for
624 	 * sending raw packets only (dhcpd, cdpd are good examples)
625 	 * we can delay adding d to the list of active listeners until
626 	 * some filter is configured.
627 	 */
628 
629 	BPFIF_WLOCK(bp);
630 	BPFD_LOCK(d);
631 
632 	d->bd_bif = bp;
633 
634 	if (op_w != 0) {
635 		/* Add to writers-only list */
636 		LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next);
637 		/*
638 		 * We decrement bd_writer on every filter set operation.
639 		 * First BIOCSETF is done by pcap_open_live() to set up
640 		 * snap length. After that appliation usually sets its own filter
641 		 */
642 		d->bd_writer = 2;
643 	} else
644 		LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
645 
646 	BPFD_UNLOCK(d);
647 	BPFIF_WUNLOCK(bp);
648 
649 	bpf_bpfd_cnt++;
650 
651 	CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list",
652 	    __func__, d->bd_pid, d->bd_writer ? "writer" : "active");
653 
654 	if (op_w == 0)
655 		EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1);
656 }
657 
658 /*
659  * Check if we need to upgrade our descriptor @d from write-only mode.
660  */
661 static int
662 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode, int flen)
663 {
664 	int is_snap, need_upgrade;
665 
666 	/*
667 	 * Check if we've already upgraded or new filter is empty.
668 	 */
669 	if (d->bd_writer == 0 || fcode == NULL)
670 		return (0);
671 
672 	need_upgrade = 0;
673 
674 	/*
675 	 * Check if cmd looks like snaplen setting from
676 	 * pcap_bpf.c:pcap_open_live().
677 	 * Note we're not checking .k value here:
678 	 * while pcap_open_live() definitely sets to to non-zero value,
679 	 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g.
680 	 * do not consider upgrading immediately
681 	 */
682 	if (cmd == BIOCSETF && flen == 1 && fcode[0].code == (BPF_RET | BPF_K))
683 		is_snap = 1;
684 	else
685 		is_snap = 0;
686 
687 	if (is_snap == 0) {
688 		/*
689 		 * We're setting first filter and it doesn't look like
690 		 * setting snaplen.  We're probably using bpf directly.
691 		 * Upgrade immediately.
692 		 */
693 		need_upgrade = 1;
694 	} else {
695 		/*
696 		 * Do not require upgrade by first BIOCSETF
697 		 * (used to set snaplen) by pcap_open_live().
698 		 */
699 
700 		if (--d->bd_writer == 0) {
701 			/*
702 			 * First snaplen filter has already
703 			 * been set. This is probably catch-all
704 			 * filter
705 			 */
706 			need_upgrade = 1;
707 		}
708 	}
709 
710 	CTR5(KTR_NET,
711 	    "%s: filter function set by pid %d, "
712 	    "bd_writer counter %d, snap %d upgrade %d",
713 	    __func__, d->bd_pid, d->bd_writer,
714 	    is_snap, need_upgrade);
715 
716 	return (need_upgrade);
717 }
718 
719 /*
720  * Add d to the list of active bp filters.
721  * Reuqires bpf_attachd() to be called before
722  */
723 static void
724 bpf_upgraded(struct bpf_d *d)
725 {
726 	struct bpf_if *bp;
727 
728 	BPF_LOCK_ASSERT();
729 
730 	bp = d->bd_bif;
731 
732 	/*
733 	 * Filter can be set several times without specifying interface.
734 	 * Mark d as reader and exit.
735 	 */
736 	if (bp == NULL) {
737 		BPFD_LOCK(d);
738 		d->bd_writer = 0;
739 		BPFD_UNLOCK(d);
740 		return;
741 	}
742 
743 	BPFIF_WLOCK(bp);
744 	BPFD_LOCK(d);
745 
746 	/* Remove from writers-only list */
747 	LIST_REMOVE(d, bd_next);
748 	LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
749 	/* Mark d as reader */
750 	d->bd_writer = 0;
751 
752 	BPFD_UNLOCK(d);
753 	BPFIF_WUNLOCK(bp);
754 
755 	CTR2(KTR_NET, "%s: upgrade required by pid %d", __func__, d->bd_pid);
756 
757 	EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1);
758 }
759 
760 /*
761  * Detach a file from its interface.
762  */
763 static void
764 bpf_detachd(struct bpf_d *d)
765 {
766 	BPF_LOCK();
767 	bpf_detachd_locked(d);
768 	BPF_UNLOCK();
769 }
770 
771 static void
772 bpf_detachd_locked(struct bpf_d *d)
773 {
774 	int error;
775 	struct bpf_if *bp;
776 	struct ifnet *ifp;
777 
778 	CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid);
779 
780 	BPF_LOCK_ASSERT();
781 
782 	/* Check if descriptor is attached */
783 	if ((bp = d->bd_bif) == NULL)
784 		return;
785 
786 	BPFIF_WLOCK(bp);
787 	BPFD_LOCK(d);
788 
789 	/* Save bd_writer value */
790 	error = d->bd_writer;
791 
792 	/*
793 	 * Remove d from the interface's descriptor list.
794 	 */
795 	LIST_REMOVE(d, bd_next);
796 
797 	ifp = bp->bif_ifp;
798 	d->bd_bif = NULL;
799 	BPFD_UNLOCK(d);
800 	BPFIF_WUNLOCK(bp);
801 
802 	bpf_bpfd_cnt--;
803 
804 	/* Call event handler iff d is attached */
805 	if (error == 0)
806 		EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0);
807 
808 	/*
809 	 * Check if this descriptor had requested promiscuous mode.
810 	 * If so, turn it off.
811 	 */
812 	if (d->bd_promisc) {
813 		d->bd_promisc = 0;
814 		CURVNET_SET(ifp->if_vnet);
815 		error = ifpromisc(ifp, 0);
816 		CURVNET_RESTORE();
817 		if (error != 0 && error != ENXIO) {
818 			/*
819 			 * ENXIO can happen if a pccard is unplugged
820 			 * Something is really wrong if we were able to put
821 			 * the driver into promiscuous mode, but can't
822 			 * take it out.
823 			 */
824 			if_printf(bp->bif_ifp,
825 				"bpf_detach: ifpromisc failed (%d)\n", error);
826 		}
827 	}
828 }
829 
830 /*
831  * Close the descriptor by detaching it from its interface,
832  * deallocating its buffers, and marking it free.
833  */
834 static void
835 bpf_dtor(void *data)
836 {
837 	struct bpf_d *d = data;
838 
839 	BPFD_LOCK(d);
840 	if (d->bd_state == BPF_WAITING)
841 		callout_stop(&d->bd_callout);
842 	d->bd_state = BPF_IDLE;
843 	BPFD_UNLOCK(d);
844 	funsetown(&d->bd_sigio);
845 	bpf_detachd(d);
846 #ifdef MAC
847 	mac_bpfdesc_destroy(d);
848 #endif /* MAC */
849 	seldrain(&d->bd_sel);
850 	knlist_destroy(&d->bd_sel.si_note);
851 	callout_drain(&d->bd_callout);
852 	bpf_freed(d);
853 	free(d, M_BPF);
854 }
855 
856 /*
857  * Open ethernet device.  Returns ENXIO for illegal minor device number,
858  * EBUSY if file is open by another process.
859  */
860 /* ARGSUSED */
861 static	int
862 bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
863 {
864 	struct bpf_d *d;
865 	int error, size;
866 
867 	d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
868 	error = devfs_set_cdevpriv(d, bpf_dtor);
869 	if (error != 0) {
870 		free(d, M_BPF);
871 		return (error);
872 	}
873 
874 	/*
875 	 * For historical reasons, perform a one-time initialization call to
876 	 * the buffer routines, even though we're not yet committed to a
877 	 * particular buffer method.
878 	 */
879 	bpf_buffer_init(d);
880 	if ((flags & FREAD) == 0)
881 		d->bd_writer = 2;
882 	d->bd_hbuf_in_use = 0;
883 	d->bd_bufmode = BPF_BUFMODE_BUFFER;
884 	d->bd_sig = SIGIO;
885 	d->bd_direction = BPF_D_INOUT;
886 	BPF_PID_REFRESH(d, td);
887 #ifdef MAC
888 	mac_bpfdesc_init(d);
889 	mac_bpfdesc_create(td->td_ucred, d);
890 #endif
891 	mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF);
892 	callout_init_mtx(&d->bd_callout, &d->bd_lock, 0);
893 	knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock);
894 
895 	/* Allocate default buffers */
896 	size = d->bd_bufsize;
897 	bpf_buffer_ioctl_sblen(d, &size);
898 
899 	return (0);
900 }
901 
902 /*
903  *  bpfread - read next chunk of packets from buffers
904  */
905 static	int
906 bpfread(struct cdev *dev, struct uio *uio, int ioflag)
907 {
908 	struct bpf_d *d;
909 	int error;
910 	int non_block;
911 	int timed_out;
912 
913 	error = devfs_get_cdevpriv((void **)&d);
914 	if (error != 0)
915 		return (error);
916 
917 	/*
918 	 * Restrict application to use a buffer the same size as
919 	 * as kernel buffers.
920 	 */
921 	if (uio->uio_resid != d->bd_bufsize)
922 		return (EINVAL);
923 
924 	non_block = ((ioflag & O_NONBLOCK) != 0);
925 
926 	BPFD_LOCK(d);
927 	BPF_PID_REFRESH_CUR(d);
928 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER) {
929 		BPFD_UNLOCK(d);
930 		return (EOPNOTSUPP);
931 	}
932 	if (d->bd_state == BPF_WAITING)
933 		callout_stop(&d->bd_callout);
934 	timed_out = (d->bd_state == BPF_TIMED_OUT);
935 	d->bd_state = BPF_IDLE;
936 	while (d->bd_hbuf_in_use) {
937 		error = mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
938 		    PRINET|PCATCH, "bd_hbuf", 0);
939 		if (error != 0) {
940 			BPFD_UNLOCK(d);
941 			return (error);
942 		}
943 	}
944 	/*
945 	 * If the hold buffer is empty, then do a timed sleep, which
946 	 * ends when the timeout expires or when enough packets
947 	 * have arrived to fill the store buffer.
948 	 */
949 	while (d->bd_hbuf == NULL) {
950 		if (d->bd_slen != 0) {
951 			/*
952 			 * A packet(s) either arrived since the previous
953 			 * read or arrived while we were asleep.
954 			 */
955 			if (d->bd_immediate || non_block || timed_out) {
956 				/*
957 				 * Rotate the buffers and return what's here
958 				 * if we are in immediate mode, non-blocking
959 				 * flag is set, or this descriptor timed out.
960 				 */
961 				ROTATE_BUFFERS(d);
962 				break;
963 			}
964 		}
965 
966 		/*
967 		 * No data is available, check to see if the bpf device
968 		 * is still pointed at a real interface.  If not, return
969 		 * ENXIO so that the userland process knows to rebind
970 		 * it before using it again.
971 		 */
972 		if (d->bd_bif == NULL) {
973 			BPFD_UNLOCK(d);
974 			return (ENXIO);
975 		}
976 
977 		if (non_block) {
978 			BPFD_UNLOCK(d);
979 			return (EWOULDBLOCK);
980 		}
981 		error = msleep(d, &d->bd_lock, PRINET|PCATCH,
982 		     "bpf", d->bd_rtout);
983 		if (error == EINTR || error == ERESTART) {
984 			BPFD_UNLOCK(d);
985 			return (error);
986 		}
987 		if (error == EWOULDBLOCK) {
988 			/*
989 			 * On a timeout, return what's in the buffer,
990 			 * which may be nothing.  If there is something
991 			 * in the store buffer, we can rotate the buffers.
992 			 */
993 			if (d->bd_hbuf)
994 				/*
995 				 * We filled up the buffer in between
996 				 * getting the timeout and arriving
997 				 * here, so we don't need to rotate.
998 				 */
999 				break;
1000 
1001 			if (d->bd_slen == 0) {
1002 				BPFD_UNLOCK(d);
1003 				return (0);
1004 			}
1005 			ROTATE_BUFFERS(d);
1006 			break;
1007 		}
1008 	}
1009 	/*
1010 	 * At this point, we know we have something in the hold slot.
1011 	 */
1012 	d->bd_hbuf_in_use = 1;
1013 	BPFD_UNLOCK(d);
1014 
1015 	/*
1016 	 * Move data from hold buffer into user space.
1017 	 * We know the entire buffer is transferred since
1018 	 * we checked above that the read buffer is bpf_bufsize bytes.
1019   	 *
1020 	 * We do not have to worry about simultaneous reads because
1021 	 * we waited for sole access to the hold buffer above.
1022 	 */
1023 	error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio);
1024 
1025 	BPFD_LOCK(d);
1026 	KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf"));
1027 	d->bd_fbuf = d->bd_hbuf;
1028 	d->bd_hbuf = NULL;
1029 	d->bd_hlen = 0;
1030 	bpf_buf_reclaimed(d);
1031 	d->bd_hbuf_in_use = 0;
1032 	wakeup(&d->bd_hbuf_in_use);
1033 	BPFD_UNLOCK(d);
1034 
1035 	return (error);
1036 }
1037 
1038 /*
1039  * If there are processes sleeping on this descriptor, wake them up.
1040  */
1041 static __inline void
1042 bpf_wakeup(struct bpf_d *d)
1043 {
1044 
1045 	BPFD_LOCK_ASSERT(d);
1046 	if (d->bd_state == BPF_WAITING) {
1047 		callout_stop(&d->bd_callout);
1048 		d->bd_state = BPF_IDLE;
1049 	}
1050 	wakeup(d);
1051 	if (d->bd_async && d->bd_sig && d->bd_sigio)
1052 		pgsigio(&d->bd_sigio, d->bd_sig, 0);
1053 
1054 	selwakeuppri(&d->bd_sel, PRINET);
1055 	KNOTE_LOCKED(&d->bd_sel.si_note, 0);
1056 }
1057 
1058 static void
1059 bpf_timed_out(void *arg)
1060 {
1061 	struct bpf_d *d = (struct bpf_d *)arg;
1062 
1063 	BPFD_LOCK_ASSERT(d);
1064 
1065 	if (callout_pending(&d->bd_callout) || !callout_active(&d->bd_callout))
1066 		return;
1067 	if (d->bd_state == BPF_WAITING) {
1068 		d->bd_state = BPF_TIMED_OUT;
1069 		if (d->bd_slen != 0)
1070 			bpf_wakeup(d);
1071 	}
1072 }
1073 
1074 static int
1075 bpf_ready(struct bpf_d *d)
1076 {
1077 
1078 	BPFD_LOCK_ASSERT(d);
1079 
1080 	if (!bpf_canfreebuf(d) && d->bd_hlen != 0)
1081 		return (1);
1082 	if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
1083 	    d->bd_slen != 0)
1084 		return (1);
1085 	return (0);
1086 }
1087 
1088 static int
1089 bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
1090 {
1091 	struct bpf_d *d;
1092 	struct ifnet *ifp;
1093 	struct mbuf *m, *mc;
1094 	struct sockaddr dst;
1095 	int error, hlen;
1096 
1097 	error = devfs_get_cdevpriv((void **)&d);
1098 	if (error != 0)
1099 		return (error);
1100 
1101 	BPF_PID_REFRESH_CUR(d);
1102 	d->bd_wcount++;
1103 	/* XXX: locking required */
1104 	if (d->bd_bif == NULL) {
1105 		d->bd_wdcount++;
1106 		return (ENXIO);
1107 	}
1108 
1109 	ifp = d->bd_bif->bif_ifp;
1110 
1111 	if ((ifp->if_flags & IFF_UP) == 0) {
1112 		d->bd_wdcount++;
1113 		return (ENETDOWN);
1114 	}
1115 
1116 	if (uio->uio_resid == 0) {
1117 		d->bd_wdcount++;
1118 		return (0);
1119 	}
1120 
1121 	bzero(&dst, sizeof(dst));
1122 	m = NULL;
1123 	hlen = 0;
1124 	/* XXX: bpf_movein() can sleep */
1125 	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp,
1126 	    &m, &dst, &hlen, d->bd_wfilter);
1127 	if (error) {
1128 		d->bd_wdcount++;
1129 		return (error);
1130 	}
1131 	d->bd_wfcount++;
1132 	if (d->bd_hdrcmplt)
1133 		dst.sa_family = pseudo_AF_HDRCMPLT;
1134 
1135 	if (d->bd_feedback) {
1136 		mc = m_dup(m, M_NOWAIT);
1137 		if (mc != NULL)
1138 			mc->m_pkthdr.rcvif = ifp;
1139 		/* Set M_PROMISC for outgoing packets to be discarded. */
1140 		if (d->bd_direction == BPF_D_INOUT)
1141 			m->m_flags |= M_PROMISC;
1142 	} else
1143 		mc = NULL;
1144 
1145 	m->m_pkthdr.len -= hlen;
1146 	m->m_len -= hlen;
1147 	m->m_data += hlen;	/* XXX */
1148 
1149 	CURVNET_SET(ifp->if_vnet);
1150 #ifdef MAC
1151 	BPFD_LOCK(d);
1152 	mac_bpfdesc_create_mbuf(d, m);
1153 	if (mc != NULL)
1154 		mac_bpfdesc_create_mbuf(d, mc);
1155 	BPFD_UNLOCK(d);
1156 #endif
1157 
1158 	error = (*ifp->if_output)(ifp, m, &dst, NULL);
1159 	if (error)
1160 		d->bd_wdcount++;
1161 
1162 	if (mc != NULL) {
1163 		if (error == 0)
1164 			(*ifp->if_input)(ifp, mc);
1165 		else
1166 			m_freem(mc);
1167 	}
1168 	CURVNET_RESTORE();
1169 
1170 	return (error);
1171 }
1172 
1173 /*
1174  * Reset a descriptor by flushing its packet buffer and clearing the receive
1175  * and drop counts.  This is doable for kernel-only buffers, but with
1176  * zero-copy buffers, we can't write to (or rotate) buffers that are
1177  * currently owned by userspace.  It would be nice if we could encapsulate
1178  * this logic in the buffer code rather than here.
1179  */
1180 static void
1181 reset_d(struct bpf_d *d)
1182 {
1183 
1184 	BPFD_LOCK_ASSERT(d);
1185 
1186 	while (d->bd_hbuf_in_use)
1187 		mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, PRINET,
1188 		    "bd_hbuf", 0);
1189 	if ((d->bd_hbuf != NULL) &&
1190 	    (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) {
1191 		/* Free the hold buffer. */
1192 		d->bd_fbuf = d->bd_hbuf;
1193 		d->bd_hbuf = NULL;
1194 		d->bd_hlen = 0;
1195 		bpf_buf_reclaimed(d);
1196 	}
1197 	if (bpf_canwritebuf(d))
1198 		d->bd_slen = 0;
1199 	d->bd_rcount = 0;
1200 	d->bd_dcount = 0;
1201 	d->bd_fcount = 0;
1202 	d->bd_wcount = 0;
1203 	d->bd_wfcount = 0;
1204 	d->bd_wdcount = 0;
1205 	d->bd_zcopy = 0;
1206 }
1207 
1208 /*
1209  *  FIONREAD		Check for read packet available.
1210  *  BIOCGBLEN		Get buffer len [for read()].
1211  *  BIOCSETF		Set read filter.
1212  *  BIOCSETFNR		Set read filter without resetting descriptor.
1213  *  BIOCSETWF		Set write filter.
1214  *  BIOCFLUSH		Flush read packet buffer.
1215  *  BIOCPROMISC		Put interface into promiscuous mode.
1216  *  BIOCGDLT		Get link layer type.
1217  *  BIOCGETIF		Get interface name.
1218  *  BIOCSETIF		Set interface.
1219  *  BIOCSRTIMEOUT	Set read timeout.
1220  *  BIOCGRTIMEOUT	Get read timeout.
1221  *  BIOCGSTATS		Get packet stats.
1222  *  BIOCIMMEDIATE	Set immediate mode.
1223  *  BIOCVERSION		Get filter language version.
1224  *  BIOCGHDRCMPLT	Get "header already complete" flag
1225  *  BIOCSHDRCMPLT	Set "header already complete" flag
1226  *  BIOCGDIRECTION	Get packet direction flag
1227  *  BIOCSDIRECTION	Set packet direction flag
1228  *  BIOCGTSTAMP		Get time stamp format and resolution.
1229  *  BIOCSTSTAMP		Set time stamp format and resolution.
1230  *  BIOCLOCK		Set "locked" flag
1231  *  BIOCFEEDBACK	Set packet feedback mode.
1232  *  BIOCSETZBUF		Set current zero-copy buffer locations.
1233  *  BIOCGETZMAX		Get maximum zero-copy buffer size.
1234  *  BIOCROTZBUF		Force rotation of zero-copy buffer
1235  *  BIOCSETBUFMODE	Set buffer mode.
1236  *  BIOCGETBUFMODE	Get current buffer mode.
1237  */
1238 /* ARGSUSED */
1239 static	int
1240 bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
1241     struct thread *td)
1242 {
1243 	struct bpf_d *d;
1244 	int error;
1245 
1246 	error = devfs_get_cdevpriv((void **)&d);
1247 	if (error != 0)
1248 		return (error);
1249 
1250 	/*
1251 	 * Refresh PID associated with this descriptor.
1252 	 */
1253 	BPFD_LOCK(d);
1254 	BPF_PID_REFRESH(d, td);
1255 	if (d->bd_state == BPF_WAITING)
1256 		callout_stop(&d->bd_callout);
1257 	d->bd_state = BPF_IDLE;
1258 	BPFD_UNLOCK(d);
1259 
1260 	if (d->bd_locked == 1) {
1261 		switch (cmd) {
1262 		case BIOCGBLEN:
1263 		case BIOCFLUSH:
1264 		case BIOCGDLT:
1265 		case BIOCGDLTLIST:
1266 #ifdef COMPAT_FREEBSD32
1267 		case BIOCGDLTLIST32:
1268 #endif
1269 		case BIOCGETIF:
1270 		case BIOCGRTIMEOUT:
1271 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1272 		case BIOCGRTIMEOUT32:
1273 #endif
1274 		case BIOCGSTATS:
1275 		case BIOCVERSION:
1276 		case BIOCGRSIG:
1277 		case BIOCGHDRCMPLT:
1278 		case BIOCSTSTAMP:
1279 		case BIOCFEEDBACK:
1280 		case FIONREAD:
1281 		case BIOCLOCK:
1282 		case BIOCSRTIMEOUT:
1283 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1284 		case BIOCSRTIMEOUT32:
1285 #endif
1286 		case BIOCIMMEDIATE:
1287 		case TIOCGPGRP:
1288 		case BIOCROTZBUF:
1289 			break;
1290 		default:
1291 			return (EPERM);
1292 		}
1293 	}
1294 #ifdef COMPAT_FREEBSD32
1295 	/*
1296 	 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so
1297 	 * that it will get 32-bit packet headers.
1298 	 */
1299 	switch (cmd) {
1300 	case BIOCSETF32:
1301 	case BIOCSETFNR32:
1302 	case BIOCSETWF32:
1303 	case BIOCGDLTLIST32:
1304 	case BIOCGRTIMEOUT32:
1305 	case BIOCSRTIMEOUT32:
1306 		BPFD_LOCK(d);
1307 		d->bd_compat32 = 1;
1308 		BPFD_UNLOCK(d);
1309 	}
1310 #endif
1311 
1312 	CURVNET_SET(TD_TO_VNET(td));
1313 	switch (cmd) {
1314 
1315 	default:
1316 		error = EINVAL;
1317 		break;
1318 
1319 	/*
1320 	 * Check for read packet available.
1321 	 */
1322 	case FIONREAD:
1323 		{
1324 			int n;
1325 
1326 			BPFD_LOCK(d);
1327 			n = d->bd_slen;
1328 			while (d->bd_hbuf_in_use)
1329 				mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
1330 				    PRINET, "bd_hbuf", 0);
1331 			if (d->bd_hbuf)
1332 				n += d->bd_hlen;
1333 			BPFD_UNLOCK(d);
1334 
1335 			*(int *)addr = n;
1336 			break;
1337 		}
1338 
1339 	/*
1340 	 * Get buffer len [for read()].
1341 	 */
1342 	case BIOCGBLEN:
1343 		BPFD_LOCK(d);
1344 		*(u_int *)addr = d->bd_bufsize;
1345 		BPFD_UNLOCK(d);
1346 		break;
1347 
1348 	/*
1349 	 * Set buffer length.
1350 	 */
1351 	case BIOCSBLEN:
1352 		error = bpf_ioctl_sblen(d, (u_int *)addr);
1353 		break;
1354 
1355 	/*
1356 	 * Set link layer read filter.
1357 	 */
1358 	case BIOCSETF:
1359 	case BIOCSETFNR:
1360 	case BIOCSETWF:
1361 #ifdef COMPAT_FREEBSD32
1362 	case BIOCSETF32:
1363 	case BIOCSETFNR32:
1364 	case BIOCSETWF32:
1365 #endif
1366 		error = bpf_setf(d, (struct bpf_program *)addr, cmd);
1367 		break;
1368 
1369 	/*
1370 	 * Flush read packet buffer.
1371 	 */
1372 	case BIOCFLUSH:
1373 		BPFD_LOCK(d);
1374 		reset_d(d);
1375 		BPFD_UNLOCK(d);
1376 		break;
1377 
1378 	/*
1379 	 * Put interface into promiscuous mode.
1380 	 */
1381 	case BIOCPROMISC:
1382 		if (d->bd_bif == NULL) {
1383 			/*
1384 			 * No interface attached yet.
1385 			 */
1386 			error = EINVAL;
1387 			break;
1388 		}
1389 		if (d->bd_promisc == 0) {
1390 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
1391 			if (error == 0)
1392 				d->bd_promisc = 1;
1393 		}
1394 		break;
1395 
1396 	/*
1397 	 * Get current data link type.
1398 	 */
1399 	case BIOCGDLT:
1400 		BPF_LOCK();
1401 		if (d->bd_bif == NULL)
1402 			error = EINVAL;
1403 		else
1404 			*(u_int *)addr = d->bd_bif->bif_dlt;
1405 		BPF_UNLOCK();
1406 		break;
1407 
1408 	/*
1409 	 * Get a list of supported data link types.
1410 	 */
1411 #ifdef COMPAT_FREEBSD32
1412 	case BIOCGDLTLIST32:
1413 		{
1414 			struct bpf_dltlist32 *list32;
1415 			struct bpf_dltlist dltlist;
1416 
1417 			list32 = (struct bpf_dltlist32 *)addr;
1418 			dltlist.bfl_len = list32->bfl_len;
1419 			dltlist.bfl_list = PTRIN(list32->bfl_list);
1420 			BPF_LOCK();
1421 			if (d->bd_bif == NULL)
1422 				error = EINVAL;
1423 			else {
1424 				error = bpf_getdltlist(d, &dltlist);
1425 				if (error == 0)
1426 					list32->bfl_len = dltlist.bfl_len;
1427 			}
1428 			BPF_UNLOCK();
1429 			break;
1430 		}
1431 #endif
1432 
1433 	case BIOCGDLTLIST:
1434 		BPF_LOCK();
1435 		if (d->bd_bif == NULL)
1436 			error = EINVAL;
1437 		else
1438 			error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
1439 		BPF_UNLOCK();
1440 		break;
1441 
1442 	/*
1443 	 * Set data link type.
1444 	 */
1445 	case BIOCSDLT:
1446 		BPF_LOCK();
1447 		if (d->bd_bif == NULL)
1448 			error = EINVAL;
1449 		else
1450 			error = bpf_setdlt(d, *(u_int *)addr);
1451 		BPF_UNLOCK();
1452 		break;
1453 
1454 	/*
1455 	 * Get interface name.
1456 	 */
1457 	case BIOCGETIF:
1458 		BPF_LOCK();
1459 		if (d->bd_bif == NULL)
1460 			error = EINVAL;
1461 		else {
1462 			struct ifnet *const ifp = d->bd_bif->bif_ifp;
1463 			struct ifreq *const ifr = (struct ifreq *)addr;
1464 
1465 			strlcpy(ifr->ifr_name, ifp->if_xname,
1466 			    sizeof(ifr->ifr_name));
1467 		}
1468 		BPF_UNLOCK();
1469 		break;
1470 
1471 	/*
1472 	 * Set interface.
1473 	 */
1474 	case BIOCSETIF:
1475 		BPF_LOCK();
1476 		error = bpf_setif(d, (struct ifreq *)addr);
1477 		BPF_UNLOCK();
1478 		break;
1479 
1480 	/*
1481 	 * Set read timeout.
1482 	 */
1483 	case BIOCSRTIMEOUT:
1484 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1485 	case BIOCSRTIMEOUT32:
1486 #endif
1487 		{
1488 			struct timeval *tv = (struct timeval *)addr;
1489 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1490 			struct timeval32 *tv32;
1491 			struct timeval tv64;
1492 
1493 			if (cmd == BIOCSRTIMEOUT32) {
1494 				tv32 = (struct timeval32 *)addr;
1495 				tv = &tv64;
1496 				tv->tv_sec = tv32->tv_sec;
1497 				tv->tv_usec = tv32->tv_usec;
1498 			} else
1499 #endif
1500 				tv = (struct timeval *)addr;
1501 
1502 			/*
1503 			 * Subtract 1 tick from tvtohz() since this isn't
1504 			 * a one-shot timer.
1505 			 */
1506 			if ((error = itimerfix(tv)) == 0)
1507 				d->bd_rtout = tvtohz(tv) - 1;
1508 			break;
1509 		}
1510 
1511 	/*
1512 	 * Get read timeout.
1513 	 */
1514 	case BIOCGRTIMEOUT:
1515 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1516 	case BIOCGRTIMEOUT32:
1517 #endif
1518 		{
1519 			struct timeval *tv;
1520 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1521 			struct timeval32 *tv32;
1522 			struct timeval tv64;
1523 
1524 			if (cmd == BIOCGRTIMEOUT32)
1525 				tv = &tv64;
1526 			else
1527 #endif
1528 				tv = (struct timeval *)addr;
1529 
1530 			tv->tv_sec = d->bd_rtout / hz;
1531 			tv->tv_usec = (d->bd_rtout % hz) * tick;
1532 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1533 			if (cmd == BIOCGRTIMEOUT32) {
1534 				tv32 = (struct timeval32 *)addr;
1535 				tv32->tv_sec = tv->tv_sec;
1536 				tv32->tv_usec = tv->tv_usec;
1537 			}
1538 #endif
1539 
1540 			break;
1541 		}
1542 
1543 	/*
1544 	 * Get packet stats.
1545 	 */
1546 	case BIOCGSTATS:
1547 		{
1548 			struct bpf_stat *bs = (struct bpf_stat *)addr;
1549 
1550 			/* XXXCSJP overflow */
1551 			bs->bs_recv = d->bd_rcount;
1552 			bs->bs_drop = d->bd_dcount;
1553 			break;
1554 		}
1555 
1556 	/*
1557 	 * Set immediate mode.
1558 	 */
1559 	case BIOCIMMEDIATE:
1560 		BPFD_LOCK(d);
1561 		d->bd_immediate = *(u_int *)addr;
1562 		BPFD_UNLOCK(d);
1563 		break;
1564 
1565 	case BIOCVERSION:
1566 		{
1567 			struct bpf_version *bv = (struct bpf_version *)addr;
1568 
1569 			bv->bv_major = BPF_MAJOR_VERSION;
1570 			bv->bv_minor = BPF_MINOR_VERSION;
1571 			break;
1572 		}
1573 
1574 	/*
1575 	 * Get "header already complete" flag
1576 	 */
1577 	case BIOCGHDRCMPLT:
1578 		BPFD_LOCK(d);
1579 		*(u_int *)addr = d->bd_hdrcmplt;
1580 		BPFD_UNLOCK(d);
1581 		break;
1582 
1583 	/*
1584 	 * Set "header already complete" flag
1585 	 */
1586 	case BIOCSHDRCMPLT:
1587 		BPFD_LOCK(d);
1588 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1589 		BPFD_UNLOCK(d);
1590 		break;
1591 
1592 	/*
1593 	 * Get packet direction flag
1594 	 */
1595 	case BIOCGDIRECTION:
1596 		BPFD_LOCK(d);
1597 		*(u_int *)addr = d->bd_direction;
1598 		BPFD_UNLOCK(d);
1599 		break;
1600 
1601 	/*
1602 	 * Set packet direction flag
1603 	 */
1604 	case BIOCSDIRECTION:
1605 		{
1606 			u_int	direction;
1607 
1608 			direction = *(u_int *)addr;
1609 			switch (direction) {
1610 			case BPF_D_IN:
1611 			case BPF_D_INOUT:
1612 			case BPF_D_OUT:
1613 				BPFD_LOCK(d);
1614 				d->bd_direction = direction;
1615 				BPFD_UNLOCK(d);
1616 				break;
1617 			default:
1618 				error = EINVAL;
1619 			}
1620 		}
1621 		break;
1622 
1623 	/*
1624 	 * Get packet timestamp format and resolution.
1625 	 */
1626 	case BIOCGTSTAMP:
1627 		BPFD_LOCK(d);
1628 		*(u_int *)addr = d->bd_tstamp;
1629 		BPFD_UNLOCK(d);
1630 		break;
1631 
1632 	/*
1633 	 * Set packet timestamp format and resolution.
1634 	 */
1635 	case BIOCSTSTAMP:
1636 		{
1637 			u_int	func;
1638 
1639 			func = *(u_int *)addr;
1640 			if (BPF_T_VALID(func))
1641 				d->bd_tstamp = func;
1642 			else
1643 				error = EINVAL;
1644 		}
1645 		break;
1646 
1647 	case BIOCFEEDBACK:
1648 		BPFD_LOCK(d);
1649 		d->bd_feedback = *(u_int *)addr;
1650 		BPFD_UNLOCK(d);
1651 		break;
1652 
1653 	case BIOCLOCK:
1654 		BPFD_LOCK(d);
1655 		d->bd_locked = 1;
1656 		BPFD_UNLOCK(d);
1657 		break;
1658 
1659 	case FIONBIO:		/* Non-blocking I/O */
1660 		break;
1661 
1662 	case FIOASYNC:		/* Send signal on receive packets */
1663 		BPFD_LOCK(d);
1664 		d->bd_async = *(int *)addr;
1665 		BPFD_UNLOCK(d);
1666 		break;
1667 
1668 	case FIOSETOWN:
1669 		/*
1670 		 * XXX: Add some sort of locking here?
1671 		 * fsetown() can sleep.
1672 		 */
1673 		error = fsetown(*(int *)addr, &d->bd_sigio);
1674 		break;
1675 
1676 	case FIOGETOWN:
1677 		BPFD_LOCK(d);
1678 		*(int *)addr = fgetown(&d->bd_sigio);
1679 		BPFD_UNLOCK(d);
1680 		break;
1681 
1682 	/* This is deprecated, FIOSETOWN should be used instead. */
1683 	case TIOCSPGRP:
1684 		error = fsetown(-(*(int *)addr), &d->bd_sigio);
1685 		break;
1686 
1687 	/* This is deprecated, FIOGETOWN should be used instead. */
1688 	case TIOCGPGRP:
1689 		*(int *)addr = -fgetown(&d->bd_sigio);
1690 		break;
1691 
1692 	case BIOCSRSIG:		/* Set receive signal */
1693 		{
1694 			u_int sig;
1695 
1696 			sig = *(u_int *)addr;
1697 
1698 			if (sig >= NSIG)
1699 				error = EINVAL;
1700 			else {
1701 				BPFD_LOCK(d);
1702 				d->bd_sig = sig;
1703 				BPFD_UNLOCK(d);
1704 			}
1705 			break;
1706 		}
1707 	case BIOCGRSIG:
1708 		BPFD_LOCK(d);
1709 		*(u_int *)addr = d->bd_sig;
1710 		BPFD_UNLOCK(d);
1711 		break;
1712 
1713 	case BIOCGETBUFMODE:
1714 		BPFD_LOCK(d);
1715 		*(u_int *)addr = d->bd_bufmode;
1716 		BPFD_UNLOCK(d);
1717 		break;
1718 
1719 	case BIOCSETBUFMODE:
1720 		/*
1721 		 * Allow the buffering mode to be changed as long as we
1722 		 * haven't yet committed to a particular mode.  Our
1723 		 * definition of commitment, for now, is whether or not a
1724 		 * buffer has been allocated or an interface attached, since
1725 		 * that's the point where things get tricky.
1726 		 */
1727 		switch (*(u_int *)addr) {
1728 		case BPF_BUFMODE_BUFFER:
1729 			break;
1730 
1731 		case BPF_BUFMODE_ZBUF:
1732 			if (bpf_zerocopy_enable)
1733 				break;
1734 			/* FALLSTHROUGH */
1735 
1736 		default:
1737 			CURVNET_RESTORE();
1738 			return (EINVAL);
1739 		}
1740 
1741 		BPFD_LOCK(d);
1742 		if (d->bd_sbuf != NULL || d->bd_hbuf != NULL ||
1743 		    d->bd_fbuf != NULL || d->bd_bif != NULL) {
1744 			BPFD_UNLOCK(d);
1745 			CURVNET_RESTORE();
1746 			return (EBUSY);
1747 		}
1748 		d->bd_bufmode = *(u_int *)addr;
1749 		BPFD_UNLOCK(d);
1750 		break;
1751 
1752 	case BIOCGETZMAX:
1753 		error = bpf_ioctl_getzmax(td, d, (size_t *)addr);
1754 		break;
1755 
1756 	case BIOCSETZBUF:
1757 		error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr);
1758 		break;
1759 
1760 	case BIOCROTZBUF:
1761 		error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr);
1762 		break;
1763 	}
1764 	CURVNET_RESTORE();
1765 	return (error);
1766 }
1767 
1768 /*
1769  * Set d's packet filter program to fp.  If this file already has a filter,
1770  * free it and replace it.  Returns EINVAL for bogus requests.
1771  *
1772  * Note we need global lock here to serialize bpf_setf() and bpf_setif() calls
1773  * since reading d->bd_bif can't be protected by d or interface lock due to
1774  * lock order.
1775  *
1776  * Additionally, we have to acquire interface write lock due to bpf_mtap() uses
1777  * interface read lock to read all filers.
1778  *
1779  */
1780 static int
1781 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1782 {
1783 #ifdef COMPAT_FREEBSD32
1784 	struct bpf_program fp_swab;
1785 	struct bpf_program32 *fp32;
1786 #endif
1787 	struct bpf_insn *fcode, *old;
1788 #ifdef BPF_JITTER
1789 	bpf_jit_filter *jfunc, *ofunc;
1790 #endif
1791 	size_t size;
1792 	u_int flen;
1793 	int need_upgrade;
1794 
1795 #ifdef COMPAT_FREEBSD32
1796 	switch (cmd) {
1797 	case BIOCSETF32:
1798 	case BIOCSETWF32:
1799 	case BIOCSETFNR32:
1800 		fp32 = (struct bpf_program32 *)fp;
1801 		fp_swab.bf_len = fp32->bf_len;
1802 		fp_swab.bf_insns = (struct bpf_insn *)(uintptr_t)fp32->bf_insns;
1803 		fp = &fp_swab;
1804 		switch (cmd) {
1805 		case BIOCSETF32:
1806 			cmd = BIOCSETF;
1807 			break;
1808 		case BIOCSETWF32:
1809 			cmd = BIOCSETWF;
1810 			break;
1811 		}
1812 		break;
1813 	}
1814 #endif
1815 
1816 	fcode = NULL;
1817 #ifdef BPF_JITTER
1818 	jfunc = ofunc = NULL;
1819 #endif
1820 	need_upgrade = 0;
1821 
1822 	/*
1823 	 * Check new filter validness before acquiring any locks.
1824 	 * Allocate memory for new filter, if needed.
1825 	 */
1826 	flen = fp->bf_len;
1827 	if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0))
1828 		return (EINVAL);
1829 	size = flen * sizeof(*fp->bf_insns);
1830 	if (size > 0) {
1831 		/* We're setting up new filter.  Copy and check actual data. */
1832 		fcode = malloc(size, M_BPF, M_WAITOK);
1833 		if (copyin(fp->bf_insns, fcode, size) != 0 ||
1834 		    !bpf_validate(fcode, flen)) {
1835 			free(fcode, M_BPF);
1836 			return (EINVAL);
1837 		}
1838 #ifdef BPF_JITTER
1839 		/* Filter is copied inside fcode and is perfectly valid. */
1840 		jfunc = bpf_jitter(fcode, flen);
1841 #endif
1842 	}
1843 
1844 	BPF_LOCK();
1845 
1846 	/*
1847 	 * Set up new filter.
1848 	 * Protect filter change by interface lock.
1849 	 * Additionally, we are protected by global lock here.
1850 	 */
1851 	if (d->bd_bif != NULL)
1852 		BPFIF_WLOCK(d->bd_bif);
1853 	BPFD_LOCK(d);
1854 	if (cmd == BIOCSETWF) {
1855 		old = d->bd_wfilter;
1856 		d->bd_wfilter = fcode;
1857 	} else {
1858 		old = d->bd_rfilter;
1859 		d->bd_rfilter = fcode;
1860 #ifdef BPF_JITTER
1861 		ofunc = d->bd_bfilter;
1862 		d->bd_bfilter = jfunc;
1863 #endif
1864 		if (cmd == BIOCSETF)
1865 			reset_d(d);
1866 
1867 		need_upgrade = bpf_check_upgrade(cmd, d, fcode, flen);
1868 	}
1869 	BPFD_UNLOCK(d);
1870 	if (d->bd_bif != NULL)
1871 		BPFIF_WUNLOCK(d->bd_bif);
1872 	if (old != NULL)
1873 		free(old, M_BPF);
1874 #ifdef BPF_JITTER
1875 	if (ofunc != NULL)
1876 		bpf_destroy_jit_filter(ofunc);
1877 #endif
1878 
1879 	/* Move d to active readers list. */
1880 	if (need_upgrade != 0)
1881 		bpf_upgraded(d);
1882 
1883 	BPF_UNLOCK();
1884 	return (0);
1885 }
1886 
1887 /*
1888  * Detach a file from its current interface (if attached at all) and attach
1889  * to the interface indicated by the name stored in ifr.
1890  * Return an errno or 0.
1891  */
1892 static int
1893 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1894 {
1895 	struct bpf_if *bp;
1896 	struct ifnet *theywant;
1897 
1898 	BPF_LOCK_ASSERT();
1899 
1900 	theywant = ifunit(ifr->ifr_name);
1901 	if (theywant == NULL || theywant->if_bpf == NULL)
1902 		return (ENXIO);
1903 
1904 	bp = theywant->if_bpf;
1905 
1906 	/* Check if interface is not being detached from BPF */
1907 	BPFIF_RLOCK(bp);
1908 	if (bp->bif_flags & BPFIF_FLAG_DYING) {
1909 		BPFIF_RUNLOCK(bp);
1910 		return (ENXIO);
1911 	}
1912 	BPFIF_RUNLOCK(bp);
1913 
1914 	/*
1915 	 * Behavior here depends on the buffering model.  If we're using
1916 	 * kernel memory buffers, then we can allocate them here.  If we're
1917 	 * using zero-copy, then the user process must have registered
1918 	 * buffers by the time we get here.  If not, return an error.
1919 	 */
1920 	switch (d->bd_bufmode) {
1921 	case BPF_BUFMODE_BUFFER:
1922 	case BPF_BUFMODE_ZBUF:
1923 		if (d->bd_sbuf == NULL)
1924 			return (EINVAL);
1925 		break;
1926 
1927 	default:
1928 		panic("bpf_setif: bufmode %d", d->bd_bufmode);
1929 	}
1930 	if (bp != d->bd_bif)
1931 		bpf_attachd(d, bp);
1932 	BPFD_LOCK(d);
1933 	reset_d(d);
1934 	BPFD_UNLOCK(d);
1935 	return (0);
1936 }
1937 
1938 /*
1939  * Support for select() and poll() system calls
1940  *
1941  * Return true iff the specific operation will not block indefinitely.
1942  * Otherwise, return false but make a note that a selwakeup() must be done.
1943  */
1944 static int
1945 bpfpoll(struct cdev *dev, int events, struct thread *td)
1946 {
1947 	struct bpf_d *d;
1948 	int revents;
1949 
1950 	if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL)
1951 		return (events &
1952 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1953 
1954 	/*
1955 	 * Refresh PID associated with this descriptor.
1956 	 */
1957 	revents = events & (POLLOUT | POLLWRNORM);
1958 	BPFD_LOCK(d);
1959 	BPF_PID_REFRESH(d, td);
1960 	if (events & (POLLIN | POLLRDNORM)) {
1961 		if (bpf_ready(d))
1962 			revents |= events & (POLLIN | POLLRDNORM);
1963 		else {
1964 			selrecord(td, &d->bd_sel);
1965 			/* Start the read timeout if necessary. */
1966 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1967 				callout_reset(&d->bd_callout, d->bd_rtout,
1968 				    bpf_timed_out, d);
1969 				d->bd_state = BPF_WAITING;
1970 			}
1971 		}
1972 	}
1973 	BPFD_UNLOCK(d);
1974 	return (revents);
1975 }
1976 
1977 /*
1978  * Support for kevent() system call.  Register EVFILT_READ filters and
1979  * reject all others.
1980  */
1981 int
1982 bpfkqfilter(struct cdev *dev, struct knote *kn)
1983 {
1984 	struct bpf_d *d;
1985 
1986 	if (devfs_get_cdevpriv((void **)&d) != 0 ||
1987 	    kn->kn_filter != EVFILT_READ)
1988 		return (1);
1989 
1990 	/*
1991 	 * Refresh PID associated with this descriptor.
1992 	 */
1993 	BPFD_LOCK(d);
1994 	BPF_PID_REFRESH_CUR(d);
1995 	kn->kn_fop = &bpfread_filtops;
1996 	kn->kn_hook = d;
1997 	knlist_add(&d->bd_sel.si_note, kn, 1);
1998 	BPFD_UNLOCK(d);
1999 
2000 	return (0);
2001 }
2002 
2003 static void
2004 filt_bpfdetach(struct knote *kn)
2005 {
2006 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2007 
2008 	knlist_remove(&d->bd_sel.si_note, kn, 0);
2009 }
2010 
2011 static int
2012 filt_bpfread(struct knote *kn, long hint)
2013 {
2014 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2015 	int ready;
2016 
2017 	BPFD_LOCK_ASSERT(d);
2018 	ready = bpf_ready(d);
2019 	if (ready) {
2020 		kn->kn_data = d->bd_slen;
2021 		while (d->bd_hbuf_in_use)
2022 			mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
2023 			    PRINET, "bd_hbuf", 0);
2024 		if (d->bd_hbuf)
2025 			kn->kn_data += d->bd_hlen;
2026 	} else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
2027 		callout_reset(&d->bd_callout, d->bd_rtout,
2028 		    bpf_timed_out, d);
2029 		d->bd_state = BPF_WAITING;
2030 	}
2031 
2032 	return (ready);
2033 }
2034 
2035 #define	BPF_TSTAMP_NONE		0
2036 #define	BPF_TSTAMP_FAST		1
2037 #define	BPF_TSTAMP_NORMAL	2
2038 #define	BPF_TSTAMP_EXTERN	3
2039 
2040 static int
2041 bpf_ts_quality(int tstype)
2042 {
2043 
2044 	if (tstype == BPF_T_NONE)
2045 		return (BPF_TSTAMP_NONE);
2046 	if ((tstype & BPF_T_FAST) != 0)
2047 		return (BPF_TSTAMP_FAST);
2048 
2049 	return (BPF_TSTAMP_NORMAL);
2050 }
2051 
2052 static int
2053 bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
2054 {
2055 	struct m_tag *tag;
2056 	int quality;
2057 
2058 	quality = bpf_ts_quality(tstype);
2059 	if (quality == BPF_TSTAMP_NONE)
2060 		return (quality);
2061 
2062 	if (m != NULL) {
2063 		tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL);
2064 		if (tag != NULL) {
2065 			*bt = *(struct bintime *)(tag + 1);
2066 			return (BPF_TSTAMP_EXTERN);
2067 		}
2068 	}
2069 	if (quality == BPF_TSTAMP_NORMAL)
2070 		binuptime(bt);
2071 	else
2072 		getbinuptime(bt);
2073 
2074 	return (quality);
2075 }
2076 
2077 /*
2078  * Incoming linkage from device drivers.  Process the packet pkt, of length
2079  * pktlen, which is stored in a contiguous buffer.  The packet is parsed
2080  * by each process' filter, and if accepted, stashed into the corresponding
2081  * buffer.
2082  */
2083 void
2084 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2085 {
2086 	struct bintime bt;
2087 	struct bpf_d *d;
2088 #ifdef BPF_JITTER
2089 	bpf_jit_filter *bf;
2090 #endif
2091 	u_int slen;
2092 	int gottime;
2093 
2094 	gottime = BPF_TSTAMP_NONE;
2095 
2096 	BPFIF_RLOCK(bp);
2097 
2098 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2099 		/*
2100 		 * We are not using any locks for d here because:
2101 		 * 1) any filter change is protected by interface
2102 		 * write lock
2103 		 * 2) destroying/detaching d is protected by interface
2104 		 * write lock, too
2105 		 */
2106 
2107 		/* XXX: Do not protect counter for the sake of performance. */
2108 		++d->bd_rcount;
2109 		/*
2110 		 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no
2111 		 * way for the caller to indiciate to us whether this packet
2112 		 * is inbound or outbound.  In the bpf_mtap() routines, we use
2113 		 * the interface pointers on the mbuf to figure it out.
2114 		 */
2115 #ifdef BPF_JITTER
2116 		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
2117 		if (bf != NULL)
2118 			slen = (*(bf->func))(pkt, pktlen, pktlen);
2119 		else
2120 #endif
2121 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
2122 		if (slen != 0) {
2123 			/*
2124 			 * Filter matches. Let's to acquire write lock.
2125 			 */
2126 			BPFD_LOCK(d);
2127 
2128 			d->bd_fcount++;
2129 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2130 				gottime = bpf_gettime(&bt, d->bd_tstamp, NULL);
2131 #ifdef MAC
2132 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2133 #endif
2134 				catchpacket(d, pkt, pktlen, slen,
2135 				    bpf_append_bytes, &bt);
2136 			BPFD_UNLOCK(d);
2137 		}
2138 	}
2139 	BPFIF_RUNLOCK(bp);
2140 }
2141 
2142 #define	BPF_CHECK_DIRECTION(d, r, i)				\
2143 	    (((d)->bd_direction == BPF_D_IN && (r) != (i)) ||	\
2144 	    ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
2145 
2146 /*
2147  * Incoming linkage from device drivers, when packet is in an mbuf chain.
2148  * Locking model is explained in bpf_tap().
2149  */
2150 void
2151 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2152 {
2153 	struct bintime bt;
2154 	struct bpf_d *d;
2155 #ifdef BPF_JITTER
2156 	bpf_jit_filter *bf;
2157 #endif
2158 	u_int pktlen, slen;
2159 	int gottime;
2160 
2161 	/* Skip outgoing duplicate packets. */
2162 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
2163 		m->m_flags &= ~M_PROMISC;
2164 		return;
2165 	}
2166 
2167 	pktlen = m_length(m, NULL);
2168 	gottime = BPF_TSTAMP_NONE;
2169 
2170 	BPFIF_RLOCK(bp);
2171 
2172 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2173 		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
2174 			continue;
2175 		++d->bd_rcount;
2176 #ifdef BPF_JITTER
2177 		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
2178 		/* XXX We cannot handle multiple mbufs. */
2179 		if (bf != NULL && m->m_next == NULL)
2180 			slen = (*(bf->func))(mtod(m, u_char *), pktlen, pktlen);
2181 		else
2182 #endif
2183 		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
2184 		if (slen != 0) {
2185 			BPFD_LOCK(d);
2186 
2187 			d->bd_fcount++;
2188 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2189 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
2190 #ifdef MAC
2191 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2192 #endif
2193 				catchpacket(d, (u_char *)m, pktlen, slen,
2194 				    bpf_append_mbuf, &bt);
2195 			BPFD_UNLOCK(d);
2196 		}
2197 	}
2198 	BPFIF_RUNLOCK(bp);
2199 }
2200 
2201 /*
2202  * Incoming linkage from device drivers, when packet is in
2203  * an mbuf chain and to be prepended by a contiguous header.
2204  */
2205 void
2206 bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
2207 {
2208 	struct bintime bt;
2209 	struct mbuf mb;
2210 	struct bpf_d *d;
2211 	u_int pktlen, slen;
2212 	int gottime;
2213 
2214 	/* Skip outgoing duplicate packets. */
2215 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
2216 		m->m_flags &= ~M_PROMISC;
2217 		return;
2218 	}
2219 
2220 	pktlen = m_length(m, NULL);
2221 	/*
2222 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
2223 	 * Note that we cut corners here; we only setup what's
2224 	 * absolutely needed--this mbuf should never go anywhere else.
2225 	 */
2226 	mb.m_next = m;
2227 	mb.m_data = data;
2228 	mb.m_len = dlen;
2229 	pktlen += dlen;
2230 
2231 	gottime = BPF_TSTAMP_NONE;
2232 
2233 	BPFIF_RLOCK(bp);
2234 
2235 	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2236 		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
2237 			continue;
2238 		++d->bd_rcount;
2239 		slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
2240 		if (slen != 0) {
2241 			BPFD_LOCK(d);
2242 
2243 			d->bd_fcount++;
2244 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2245 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
2246 #ifdef MAC
2247 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2248 #endif
2249 				catchpacket(d, (u_char *)&mb, pktlen, slen,
2250 				    bpf_append_mbuf, &bt);
2251 			BPFD_UNLOCK(d);
2252 		}
2253 	}
2254 	BPFIF_RUNLOCK(bp);
2255 }
2256 
2257 #undef	BPF_CHECK_DIRECTION
2258 
2259 #undef	BPF_TSTAMP_NONE
2260 #undef	BPF_TSTAMP_FAST
2261 #undef	BPF_TSTAMP_NORMAL
2262 #undef	BPF_TSTAMP_EXTERN
2263 
2264 static int
2265 bpf_hdrlen(struct bpf_d *d)
2266 {
2267 	int hdrlen;
2268 
2269 	hdrlen = d->bd_bif->bif_hdrlen;
2270 #ifndef BURN_BRIDGES
2271 	if (d->bd_tstamp == BPF_T_NONE ||
2272 	    BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME)
2273 #ifdef COMPAT_FREEBSD32
2274 		if (d->bd_compat32)
2275 			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32);
2276 		else
2277 #endif
2278 			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr);
2279 	else
2280 #endif
2281 		hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr);
2282 #ifdef COMPAT_FREEBSD32
2283 	if (d->bd_compat32)
2284 		hdrlen = BPF_WORDALIGN32(hdrlen);
2285 	else
2286 #endif
2287 		hdrlen = BPF_WORDALIGN(hdrlen);
2288 
2289 	return (hdrlen - d->bd_bif->bif_hdrlen);
2290 }
2291 
2292 static void
2293 bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
2294 {
2295 	struct bintime bt2;
2296 	struct timeval tsm;
2297 	struct timespec tsn;
2298 
2299 	if ((tstype & BPF_T_MONOTONIC) == 0) {
2300 		bt2 = *bt;
2301 		bintime_add(&bt2, &boottimebin);
2302 		bt = &bt2;
2303 	}
2304 	switch (BPF_T_FORMAT(tstype)) {
2305 	case BPF_T_MICROTIME:
2306 		bintime2timeval(bt, &tsm);
2307 		ts->bt_sec = tsm.tv_sec;
2308 		ts->bt_frac = tsm.tv_usec;
2309 		break;
2310 	case BPF_T_NANOTIME:
2311 		bintime2timespec(bt, &tsn);
2312 		ts->bt_sec = tsn.tv_sec;
2313 		ts->bt_frac = tsn.tv_nsec;
2314 		break;
2315 	case BPF_T_BINTIME:
2316 		ts->bt_sec = bt->sec;
2317 		ts->bt_frac = bt->frac;
2318 		break;
2319 	}
2320 }
2321 
2322 /*
2323  * Move the packet data from interface memory (pkt) into the
2324  * store buffer.  "cpfn" is the routine called to do the actual data
2325  * transfer.  bcopy is passed in to copy contiguous chunks, while
2326  * bpf_append_mbuf is passed in to copy mbuf chains.  In the latter case,
2327  * pkt is really an mbuf.
2328  */
2329 static void
2330 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
2331     void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
2332     struct bintime *bt)
2333 {
2334 	struct bpf_xhdr hdr;
2335 #ifndef BURN_BRIDGES
2336 	struct bpf_hdr hdr_old;
2337 #ifdef COMPAT_FREEBSD32
2338 	struct bpf_hdr32 hdr32_old;
2339 #endif
2340 #endif
2341 	int caplen, curlen, hdrlen, totlen;
2342 	int do_wakeup = 0;
2343 	int do_timestamp;
2344 	int tstype;
2345 
2346 	BPFD_LOCK_ASSERT(d);
2347 
2348 	/*
2349 	 * Detect whether user space has released a buffer back to us, and if
2350 	 * so, move it from being a hold buffer to a free buffer.  This may
2351 	 * not be the best place to do it (for example, we might only want to
2352 	 * run this check if we need the space), but for now it's a reliable
2353 	 * spot to do it.
2354 	 */
2355 	if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) {
2356 		while (d->bd_hbuf_in_use)
2357 			mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
2358 			    PRINET, "bd_hbuf", 0);
2359 		d->bd_fbuf = d->bd_hbuf;
2360 		d->bd_hbuf = NULL;
2361 		d->bd_hlen = 0;
2362 		bpf_buf_reclaimed(d);
2363 	}
2364 
2365 	/*
2366 	 * Figure out how many bytes to move.  If the packet is
2367 	 * greater or equal to the snapshot length, transfer that
2368 	 * much.  Otherwise, transfer the whole packet (unless
2369 	 * we hit the buffer size limit).
2370 	 */
2371 	hdrlen = bpf_hdrlen(d);
2372 	totlen = hdrlen + min(snaplen, pktlen);
2373 	if (totlen > d->bd_bufsize)
2374 		totlen = d->bd_bufsize;
2375 
2376 	/*
2377 	 * Round up the end of the previous packet to the next longword.
2378 	 *
2379 	 * Drop the packet if there's no room and no hope of room
2380 	 * If the packet would overflow the storage buffer or the storage
2381 	 * buffer is considered immutable by the buffer model, try to rotate
2382 	 * the buffer and wakeup pending processes.
2383 	 */
2384 #ifdef COMPAT_FREEBSD32
2385 	if (d->bd_compat32)
2386 		curlen = BPF_WORDALIGN32(d->bd_slen);
2387 	else
2388 #endif
2389 		curlen = BPF_WORDALIGN(d->bd_slen);
2390 	if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) {
2391 		if (d->bd_fbuf == NULL) {
2392 			/*
2393 			 * There's no room in the store buffer, and no
2394 			 * prospect of room, so drop the packet.  Notify the
2395 			 * buffer model.
2396 			 */
2397 			bpf_buffull(d);
2398 			++d->bd_dcount;
2399 			return;
2400 		}
2401 		while (d->bd_hbuf_in_use)
2402 			mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
2403 			    PRINET, "bd_hbuf", 0);
2404 		ROTATE_BUFFERS(d);
2405 		do_wakeup = 1;
2406 		curlen = 0;
2407 	} else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT)
2408 		/*
2409 		 * Immediate mode is set, or the read timeout has already
2410 		 * expired during a select call.  A packet arrived, so the
2411 		 * reader should be woken up.
2412 		 */
2413 		do_wakeup = 1;
2414 	caplen = totlen - hdrlen;
2415 	tstype = d->bd_tstamp;
2416 	do_timestamp = tstype != BPF_T_NONE;
2417 #ifndef BURN_BRIDGES
2418 	if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) {
2419 		struct bpf_ts ts;
2420 		if (do_timestamp)
2421 			bpf_bintime2ts(bt, &ts, tstype);
2422 #ifdef COMPAT_FREEBSD32
2423 		if (d->bd_compat32) {
2424 			bzero(&hdr32_old, sizeof(hdr32_old));
2425 			if (do_timestamp) {
2426 				hdr32_old.bh_tstamp.tv_sec = ts.bt_sec;
2427 				hdr32_old.bh_tstamp.tv_usec = ts.bt_frac;
2428 			}
2429 			hdr32_old.bh_datalen = pktlen;
2430 			hdr32_old.bh_hdrlen = hdrlen;
2431 			hdr32_old.bh_caplen = caplen;
2432 			bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old,
2433 			    sizeof(hdr32_old));
2434 			goto copy;
2435 		}
2436 #endif
2437 		bzero(&hdr_old, sizeof(hdr_old));
2438 		if (do_timestamp) {
2439 			hdr_old.bh_tstamp.tv_sec = ts.bt_sec;
2440 			hdr_old.bh_tstamp.tv_usec = ts.bt_frac;
2441 		}
2442 		hdr_old.bh_datalen = pktlen;
2443 		hdr_old.bh_hdrlen = hdrlen;
2444 		hdr_old.bh_caplen = caplen;
2445 		bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old,
2446 		    sizeof(hdr_old));
2447 		goto copy;
2448 	}
2449 #endif
2450 
2451 	/*
2452 	 * Append the bpf header.  Note we append the actual header size, but
2453 	 * move forward the length of the header plus padding.
2454 	 */
2455 	bzero(&hdr, sizeof(hdr));
2456 	if (do_timestamp)
2457 		bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype);
2458 	hdr.bh_datalen = pktlen;
2459 	hdr.bh_hdrlen = hdrlen;
2460 	hdr.bh_caplen = caplen;
2461 	bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr));
2462 
2463 	/*
2464 	 * Copy the packet data into the store buffer and update its length.
2465 	 */
2466 #ifndef BURN_BRIDGES
2467 copy:
2468 #endif
2469 	(*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen);
2470 	d->bd_slen = curlen + totlen;
2471 
2472 	if (do_wakeup)
2473 		bpf_wakeup(d);
2474 }
2475 
2476 /*
2477  * Free buffers currently in use by a descriptor.
2478  * Called on close.
2479  */
2480 static void
2481 bpf_freed(struct bpf_d *d)
2482 {
2483 
2484 	/*
2485 	 * We don't need to lock out interrupts since this descriptor has
2486 	 * been detached from its interface and it yet hasn't been marked
2487 	 * free.
2488 	 */
2489 	bpf_free(d);
2490 	if (d->bd_rfilter != NULL) {
2491 		free((caddr_t)d->bd_rfilter, M_BPF);
2492 #ifdef BPF_JITTER
2493 		if (d->bd_bfilter != NULL)
2494 			bpf_destroy_jit_filter(d->bd_bfilter);
2495 #endif
2496 	}
2497 	if (d->bd_wfilter != NULL)
2498 		free((caddr_t)d->bd_wfilter, M_BPF);
2499 	mtx_destroy(&d->bd_lock);
2500 }
2501 
2502 /*
2503  * Attach an interface to bpf.  dlt is the link layer type; hdrlen is the
2504  * fixed size of the link header (variable length headers not yet supported).
2505  */
2506 void
2507 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2508 {
2509 
2510 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2511 }
2512 
2513 /*
2514  * Attach an interface to bpf.  ifp is a pointer to the structure
2515  * defining the interface to be attached, dlt is the link layer type,
2516  * and hdrlen is the fixed size of the link header (variable length
2517  * headers are not yet supporrted).
2518  */
2519 void
2520 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2521 {
2522 	struct bpf_if *bp;
2523 
2524 	bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO);
2525 	if (bp == NULL)
2526 		panic("bpfattach");
2527 
2528 	LIST_INIT(&bp->bif_dlist);
2529 	LIST_INIT(&bp->bif_wlist);
2530 	bp->bif_ifp = ifp;
2531 	bp->bif_dlt = dlt;
2532 	rw_init(&bp->bif_lock, "bpf interface lock");
2533 	KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized"));
2534 	*driverp = bp;
2535 
2536 	BPF_LOCK();
2537 	LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
2538 	BPF_UNLOCK();
2539 
2540 	bp->bif_hdrlen = hdrlen;
2541 
2542 	if (bootverbose)
2543 		if_printf(ifp, "bpf attached\n");
2544 }
2545 
2546 /*
2547  * Detach bpf from an interface. This involves detaching each descriptor
2548  * associated with the interface. Notify each descriptor as it's detached
2549  * so that any sleepers wake up and get ENXIO.
2550  */
2551 void
2552 bpfdetach(struct ifnet *ifp)
2553 {
2554 	struct bpf_if	*bp, *bp_temp;
2555 	struct bpf_d	*d;
2556 	int ndetached;
2557 
2558 	ndetached = 0;
2559 
2560 	BPF_LOCK();
2561 	/* Find all bpf_if struct's which reference ifp and detach them. */
2562 	LIST_FOREACH_SAFE(bp, &bpf_iflist, bif_next, bp_temp) {
2563 		if (ifp != bp->bif_ifp)
2564 			continue;
2565 
2566 		LIST_REMOVE(bp, bif_next);
2567 		/* Add to to-be-freed list */
2568 		LIST_INSERT_HEAD(&bpf_freelist, bp, bif_next);
2569 
2570 		ndetached++;
2571 		/*
2572 		 * Delay freeing bp till interface is detached
2573 		 * and all routes through this interface are removed.
2574 		 * Mark bp as detached to restrict new consumers.
2575 		 */
2576 		BPFIF_WLOCK(bp);
2577 		bp->bif_flags |= BPFIF_FLAG_DYING;
2578 		BPFIF_WUNLOCK(bp);
2579 
2580 		CTR4(KTR_NET, "%s: sheduling free for encap %d (%p) for if %p",
2581 		    __func__, bp->bif_dlt, bp, ifp);
2582 
2583 		/* Free common descriptors */
2584 		while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) {
2585 			bpf_detachd_locked(d);
2586 			BPFD_LOCK(d);
2587 			bpf_wakeup(d);
2588 			BPFD_UNLOCK(d);
2589 		}
2590 
2591 		/* Free writer-only descriptors */
2592 		while ((d = LIST_FIRST(&bp->bif_wlist)) != NULL) {
2593 			bpf_detachd_locked(d);
2594 			BPFD_LOCK(d);
2595 			bpf_wakeup(d);
2596 			BPFD_UNLOCK(d);
2597 		}
2598 	}
2599 	BPF_UNLOCK();
2600 
2601 #ifdef INVARIANTS
2602 	if (ndetached == 0)
2603 		printf("bpfdetach: %s was not attached\n", ifp->if_xname);
2604 #endif
2605 }
2606 
2607 /*
2608  * Interface departure handler.
2609  * Note departure event does not guarantee interface is going down.
2610  * Interface renaming is currently done via departure/arrival event set.
2611  *
2612  * Departure handled is called after all routes pointing to
2613  * given interface are removed and interface is in down state
2614  * restricting any packets to be sent/received. We assume it is now safe
2615  * to free data allocated by BPF.
2616  */
2617 static void
2618 bpf_ifdetach(void *arg __unused, struct ifnet *ifp)
2619 {
2620 	struct bpf_if *bp, *bp_temp;
2621 	int nmatched = 0;
2622 
2623 	BPF_LOCK();
2624 	/*
2625 	 * Find matching entries in free list.
2626 	 * Nothing should be found if bpfdetach() was not called.
2627 	 */
2628 	LIST_FOREACH_SAFE(bp, &bpf_freelist, bif_next, bp_temp) {
2629 		if (ifp != bp->bif_ifp)
2630 			continue;
2631 
2632 		CTR3(KTR_NET, "%s: freeing BPF instance %p for interface %p",
2633 		    __func__, bp, ifp);
2634 
2635 		LIST_REMOVE(bp, bif_next);
2636 
2637 		rw_destroy(&bp->bif_lock);
2638 		free(bp, M_BPF);
2639 
2640 		nmatched++;
2641 	}
2642 	BPF_UNLOCK();
2643 
2644 	/*
2645 	 * Note that we cannot zero other pointers to
2646 	 * custom DLTs possibly used by given interface.
2647 	 */
2648 	if (nmatched != 0)
2649 		ifp->if_bpf = NULL;
2650 }
2651 
2652 /*
2653  * Get a list of available data link type of the interface.
2654  */
2655 static int
2656 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2657 {
2658 	int n, error;
2659 	struct ifnet *ifp;
2660 	struct bpf_if *bp;
2661 
2662 	BPF_LOCK_ASSERT();
2663 
2664 	ifp = d->bd_bif->bif_ifp;
2665 	n = 0;
2666 	error = 0;
2667 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2668 		if (bp->bif_ifp != ifp)
2669 			continue;
2670 		if (bfl->bfl_list != NULL) {
2671 			if (n >= bfl->bfl_len)
2672 				return (ENOMEM);
2673 			error = copyout(&bp->bif_dlt,
2674 			    bfl->bfl_list + n, sizeof(u_int));
2675 		}
2676 		n++;
2677 	}
2678 	bfl->bfl_len = n;
2679 	return (error);
2680 }
2681 
2682 /*
2683  * Set the data link type of a BPF instance.
2684  */
2685 static int
2686 bpf_setdlt(struct bpf_d *d, u_int dlt)
2687 {
2688 	int error, opromisc;
2689 	struct ifnet *ifp;
2690 	struct bpf_if *bp;
2691 
2692 	BPF_LOCK_ASSERT();
2693 
2694 	if (d->bd_bif->bif_dlt == dlt)
2695 		return (0);
2696 	ifp = d->bd_bif->bif_ifp;
2697 
2698 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2699 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2700 			break;
2701 	}
2702 
2703 	if (bp != NULL) {
2704 		opromisc = d->bd_promisc;
2705 		bpf_attachd(d, bp);
2706 		BPFD_LOCK(d);
2707 		reset_d(d);
2708 		BPFD_UNLOCK(d);
2709 		if (opromisc) {
2710 			error = ifpromisc(bp->bif_ifp, 1);
2711 			if (error)
2712 				if_printf(bp->bif_ifp,
2713 					"bpf_setdlt: ifpromisc failed (%d)\n",
2714 					error);
2715 			else
2716 				d->bd_promisc = 1;
2717 		}
2718 	}
2719 	return (bp == NULL ? EINVAL : 0);
2720 }
2721 
2722 static void
2723 bpf_drvinit(void *unused)
2724 {
2725 	struct cdev *dev;
2726 
2727 	mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF);
2728 	LIST_INIT(&bpf_iflist);
2729 	LIST_INIT(&bpf_freelist);
2730 
2731 	dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf");
2732 	/* For compatibility */
2733 	make_dev_alias(dev, "bpf0");
2734 
2735 	/* Register interface departure handler */
2736 	bpf_ifdetach_cookie = EVENTHANDLER_REGISTER(
2737 		    ifnet_departure_event, bpf_ifdetach, NULL,
2738 		    EVENTHANDLER_PRI_ANY);
2739 }
2740 
2741 /*
2742  * Zero out the various packet counters associated with all of the bpf
2743  * descriptors.  At some point, we will probably want to get a bit more
2744  * granular and allow the user to specify descriptors to be zeroed.
2745  */
2746 static void
2747 bpf_zero_counters(void)
2748 {
2749 	struct bpf_if *bp;
2750 	struct bpf_d *bd;
2751 
2752 	BPF_LOCK();
2753 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2754 		BPFIF_RLOCK(bp);
2755 		LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2756 			BPFD_LOCK(bd);
2757 			bd->bd_rcount = 0;
2758 			bd->bd_dcount = 0;
2759 			bd->bd_fcount = 0;
2760 			bd->bd_wcount = 0;
2761 			bd->bd_wfcount = 0;
2762 			bd->bd_zcopy = 0;
2763 			BPFD_UNLOCK(bd);
2764 		}
2765 		BPFIF_RUNLOCK(bp);
2766 	}
2767 	BPF_UNLOCK();
2768 }
2769 
2770 /*
2771  * Fill filter statistics
2772  */
2773 static void
2774 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
2775 {
2776 
2777 	bzero(d, sizeof(*d));
2778 	BPFD_LOCK_ASSERT(bd);
2779 	d->bd_structsize = sizeof(*d);
2780 	/* XXX: reading should be protected by global lock */
2781 	d->bd_immediate = bd->bd_immediate;
2782 	d->bd_promisc = bd->bd_promisc;
2783 	d->bd_hdrcmplt = bd->bd_hdrcmplt;
2784 	d->bd_direction = bd->bd_direction;
2785 	d->bd_feedback = bd->bd_feedback;
2786 	d->bd_async = bd->bd_async;
2787 	d->bd_rcount = bd->bd_rcount;
2788 	d->bd_dcount = bd->bd_dcount;
2789 	d->bd_fcount = bd->bd_fcount;
2790 	d->bd_sig = bd->bd_sig;
2791 	d->bd_slen = bd->bd_slen;
2792 	d->bd_hlen = bd->bd_hlen;
2793 	d->bd_bufsize = bd->bd_bufsize;
2794 	d->bd_pid = bd->bd_pid;
2795 	strlcpy(d->bd_ifname,
2796 	    bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
2797 	d->bd_locked = bd->bd_locked;
2798 	d->bd_wcount = bd->bd_wcount;
2799 	d->bd_wdcount = bd->bd_wdcount;
2800 	d->bd_wfcount = bd->bd_wfcount;
2801 	d->bd_zcopy = bd->bd_zcopy;
2802 	d->bd_bufmode = bd->bd_bufmode;
2803 }
2804 
2805 /*
2806  * Handle `netstat -B' stats request
2807  */
2808 static int
2809 bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
2810 {
2811 	static const struct xbpf_d zerostats;
2812 	struct xbpf_d *xbdbuf, *xbd, tempstats;
2813 	int index, error;
2814 	struct bpf_if *bp;
2815 	struct bpf_d *bd;
2816 
2817 	/*
2818 	 * XXX This is not technically correct. It is possible for non
2819 	 * privileged users to open bpf devices. It would make sense
2820 	 * if the users who opened the devices were able to retrieve
2821 	 * the statistics for them, too.
2822 	 */
2823 	error = priv_check(req->td, PRIV_NET_BPF);
2824 	if (error)
2825 		return (error);
2826 	/*
2827 	 * Check to see if the user is requesting that the counters be
2828 	 * zeroed out.  Explicitly check that the supplied data is zeroed,
2829 	 * as we aren't allowing the user to set the counters currently.
2830 	 */
2831 	if (req->newptr != NULL) {
2832 		if (req->newlen != sizeof(tempstats))
2833 			return (EINVAL);
2834 		memset(&tempstats, 0, sizeof(tempstats));
2835 		error = SYSCTL_IN(req, &tempstats, sizeof(tempstats));
2836 		if (error)
2837 			return (error);
2838 		if (bcmp(&tempstats, &zerostats, sizeof(tempstats)) != 0)
2839 			return (EINVAL);
2840 		bpf_zero_counters();
2841 		return (0);
2842 	}
2843 	if (req->oldptr == NULL)
2844 		return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
2845 	if (bpf_bpfd_cnt == 0)
2846 		return (SYSCTL_OUT(req, 0, 0));
2847 	xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
2848 	BPF_LOCK();
2849 	if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
2850 		BPF_UNLOCK();
2851 		free(xbdbuf, M_BPF);
2852 		return (ENOMEM);
2853 	}
2854 	index = 0;
2855 	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2856 		BPFIF_RLOCK(bp);
2857 		/* Send writers-only first */
2858 		LIST_FOREACH(bd, &bp->bif_wlist, bd_next) {
2859 			xbd = &xbdbuf[index++];
2860 			BPFD_LOCK(bd);
2861 			bpfstats_fill_xbpf(xbd, bd);
2862 			BPFD_UNLOCK(bd);
2863 		}
2864 		LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2865 			xbd = &xbdbuf[index++];
2866 			BPFD_LOCK(bd);
2867 			bpfstats_fill_xbpf(xbd, bd);
2868 			BPFD_UNLOCK(bd);
2869 		}
2870 		BPFIF_RUNLOCK(bp);
2871 	}
2872 	BPF_UNLOCK();
2873 	error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
2874 	free(xbdbuf, M_BPF);
2875 	return (error);
2876 }
2877 
2878 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL);
2879 
2880 #else /* !DEV_BPF && !NETGRAPH_BPF */
2881 /*
2882  * NOP stubs to allow bpf-using drivers to load and function.
2883  *
2884  * A 'better' implementation would allow the core bpf functionality
2885  * to be loaded at runtime.
2886  */
2887 static struct bpf_if bp_null;
2888 
2889 void
2890 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2891 {
2892 }
2893 
2894 void
2895 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2896 {
2897 }
2898 
2899 void
2900 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
2901 {
2902 }
2903 
2904 void
2905 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2906 {
2907 
2908 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2909 }
2910 
2911 void
2912 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2913 {
2914 
2915 	*driverp = &bp_null;
2916 }
2917 
2918 void
2919 bpfdetach(struct ifnet *ifp)
2920 {
2921 }
2922 
2923 u_int
2924 bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
2925 {
2926 	return -1;	/* "no filter" behaviour */
2927 }
2928 
2929 int
2930 bpf_validate(const struct bpf_insn *f, int len)
2931 {
2932 	return 0;		/* false */
2933 }
2934 
2935 #endif /* !DEV_BPF && !NETGRAPH_BPF */
2936