xref: /freebsd/sys/net/bpf.c (revision 6aae3517ed2500fb963ba0a4264b4756088dd0f4)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1990, 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * Copyright (c) 2019 Andrey V. Elsukov <ae@FreeBSD.org>
7  *
8  * This code is derived from the Stanford/CMU enet packet filter,
9  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
10  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
11  * Berkeley Laboratory.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  *
37  *      @(#)bpf.c	8.4 (Berkeley) 1/9/95
38  */
39 
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
42 
43 #include "opt_bpf.h"
44 #include "opt_ddb.h"
45 #include "opt_netgraph.h"
46 
47 #include <sys/param.h>
48 #include <sys/conf.h>
49 #include <sys/eventhandler.h>
50 #include <sys/fcntl.h>
51 #include <sys/jail.h>
52 #include <sys/ktr.h>
53 #include <sys/lock.h>
54 #include <sys/malloc.h>
55 #include <sys/mbuf.h>
56 #include <sys/mutex.h>
57 #include <sys/time.h>
58 #include <sys/priv.h>
59 #include <sys/proc.h>
60 #include <sys/signalvar.h>
61 #include <sys/filio.h>
62 #include <sys/sockio.h>
63 #include <sys/ttycom.h>
64 #include <sys/uio.h>
65 #include <sys/sysent.h>
66 #include <sys/systm.h>
67 
68 #include <sys/event.h>
69 #include <sys/file.h>
70 #include <sys/poll.h>
71 #include <sys/proc.h>
72 
73 #include <sys/socket.h>
74 
75 #ifdef DDB
76 #include <ddb/ddb.h>
77 #endif
78 
79 #include <net/if.h>
80 #include <net/if_var.h>
81 #include <net/if_vlan_var.h>
82 #include <net/if_dl.h>
83 #include <net/bpf.h>
84 #include <net/bpf_buffer.h>
85 #ifdef BPF_JITTER
86 #include <net/bpf_jitter.h>
87 #endif
88 #include <net/bpf_zerocopy.h>
89 #include <net/bpfdesc.h>
90 #include <net/route.h>
91 #include <net/vnet.h>
92 
93 #include <netinet/in.h>
94 #include <netinet/if_ether.h>
95 #include <sys/kernel.h>
96 #include <sys/sysctl.h>
97 
98 #include <net80211/ieee80211_freebsd.h>
99 
100 #include <security/mac/mac_framework.h>
101 
102 MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
103 
104 static struct bpf_if_ext dead_bpf_if = {
105 	.bif_dlist = CK_LIST_HEAD_INITIALIZER()
106 };
107 
108 struct bpf_if {
109 #define	bif_next	bif_ext.bif_next
110 #define	bif_dlist	bif_ext.bif_dlist
111 	struct bpf_if_ext bif_ext;	/* public members */
112 	u_int		bif_dlt;	/* link layer type */
113 	u_int		bif_hdrlen;	/* length of link header */
114 	struct bpfd_list bif_wlist;	/* writer-only list */
115 	struct ifnet	*bif_ifp;	/* corresponding interface */
116 	struct bpf_if	**bif_bpf;	/* Pointer to pointer to us */
117 	volatile u_int	bif_refcnt;
118 	struct epoch_context epoch_ctx;
119 };
120 
121 CTASSERT(offsetof(struct bpf_if, bif_ext) == 0);
122 
123 struct bpf_program_buffer {
124 	struct epoch_context	epoch_ctx;
125 #ifdef BPF_JITTER
126 	bpf_jit_filter		*func;
127 #endif
128 	void			*buffer[0];
129 };
130 
131 #if defined(DEV_BPF) || defined(NETGRAPH_BPF)
132 
133 #define PRINET  26			/* interruptible */
134 #define BPF_PRIO_MAX	7
135 
136 #define	SIZEOF_BPF_HDR(type)	\
137     (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
138 
139 #ifdef COMPAT_FREEBSD32
140 #include <sys/mount.h>
141 #include <compat/freebsd32/freebsd32.h>
142 #define BPF_ALIGNMENT32 sizeof(int32_t)
143 #define	BPF_WORDALIGN32(x) roundup2(x, BPF_ALIGNMENT32)
144 
145 #ifndef BURN_BRIDGES
146 /*
147  * 32-bit version of structure prepended to each packet.  We use this header
148  * instead of the standard one for 32-bit streams.  We mark the a stream as
149  * 32-bit the first time we see a 32-bit compat ioctl request.
150  */
151 struct bpf_hdr32 {
152 	struct timeval32 bh_tstamp;	/* time stamp */
153 	uint32_t	bh_caplen;	/* length of captured portion */
154 	uint32_t	bh_datalen;	/* original length of packet */
155 	uint16_t	bh_hdrlen;	/* length of bpf header (this struct
156 					   plus alignment padding) */
157 };
158 #endif
159 
160 struct bpf_program32 {
161 	u_int bf_len;
162 	uint32_t bf_insns;
163 };
164 
165 struct bpf_dltlist32 {
166 	u_int	bfl_len;
167 	u_int	bfl_list;
168 };
169 
170 #define	BIOCSETF32	_IOW('B', 103, struct bpf_program32)
171 #define	BIOCSRTIMEOUT32	_IOW('B', 109, struct timeval32)
172 #define	BIOCGRTIMEOUT32	_IOR('B', 110, struct timeval32)
173 #define	BIOCGDLTLIST32	_IOWR('B', 121, struct bpf_dltlist32)
174 #define	BIOCSETWF32	_IOW('B', 123, struct bpf_program32)
175 #define	BIOCSETFNR32	_IOW('B', 130, struct bpf_program32)
176 #endif
177 
178 #define BPF_LOCK()	   sx_xlock(&bpf_sx)
179 #define BPF_UNLOCK()		sx_xunlock(&bpf_sx)
180 #define BPF_LOCK_ASSERT()	sx_assert(&bpf_sx, SA_XLOCKED)
181 /*
182  * bpf_iflist is a list of BPF interface structures, each corresponding to a
183  * specific DLT. The same network interface might have several BPF interface
184  * structures registered by different layers in the stack (i.e., 802.11
185  * frames, ethernet frames, etc).
186  */
187 CK_LIST_HEAD(bpf_iflist, bpf_if);
188 static struct bpf_iflist bpf_iflist;
189 static struct sx	bpf_sx;		/* bpf global lock */
190 static int		bpf_bpfd_cnt;
191 
192 static void	bpfif_ref(struct bpf_if *);
193 static void	bpfif_rele(struct bpf_if *);
194 
195 static void	bpfd_ref(struct bpf_d *);
196 static void	bpfd_rele(struct bpf_d *);
197 static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
198 static void	bpf_detachd(struct bpf_d *);
199 static void	bpf_detachd_locked(struct bpf_d *, bool);
200 static void	bpfd_free(epoch_context_t);
201 static int	bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **,
202 		    struct sockaddr *, int *, struct bpf_d *);
203 static int	bpf_setif(struct bpf_d *, struct ifreq *);
204 static void	bpf_timed_out(void *);
205 static __inline void
206 		bpf_wakeup(struct bpf_d *);
207 static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
208 		    void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
209 		    struct bintime *);
210 static void	reset_d(struct bpf_d *);
211 static int	bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
212 static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
213 static int	bpf_setdlt(struct bpf_d *, u_int);
214 static void	filt_bpfdetach(struct knote *);
215 static int	filt_bpfread(struct knote *, long);
216 static int	filt_bpfwrite(struct knote *, long);
217 static void	bpf_drvinit(void *);
218 static int	bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
219 
220 SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
221     "bpf sysctl");
222 int bpf_maxinsns = BPF_MAXINSNS;
223 SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
224     &bpf_maxinsns, 0, "Maximum bpf program instructions");
225 static int bpf_zerocopy_enable = 0;
226 SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
227     &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
228 static SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
229     bpf_stats_sysctl, "bpf statistics portal");
230 
231 VNET_DEFINE_STATIC(int, bpf_optimize_writers) = 0;
232 #define	V_bpf_optimize_writers VNET(bpf_optimize_writers)
233 SYSCTL_INT(_net_bpf, OID_AUTO, optimize_writers, CTLFLAG_VNET | CTLFLAG_RWTUN,
234     &VNET_NAME(bpf_optimize_writers), 0,
235     "Do not send packets until BPF program is set");
236 
237 static	d_open_t	bpfopen;
238 static	d_read_t	bpfread;
239 static	d_write_t	bpfwrite;
240 static	d_ioctl_t	bpfioctl;
241 static	d_poll_t	bpfpoll;
242 static	d_kqfilter_t	bpfkqfilter;
243 
244 static struct cdevsw bpf_cdevsw = {
245 	.d_version =	D_VERSION,
246 	.d_open =	bpfopen,
247 	.d_read =	bpfread,
248 	.d_write =	bpfwrite,
249 	.d_ioctl =	bpfioctl,
250 	.d_poll =	bpfpoll,
251 	.d_name =	"bpf",
252 	.d_kqfilter =	bpfkqfilter,
253 };
254 
255 static struct filterops bpfread_filtops = {
256 	.f_isfd = 1,
257 	.f_detach = filt_bpfdetach,
258 	.f_event = filt_bpfread,
259 };
260 
261 static struct filterops bpfwrite_filtops = {
262 	.f_isfd = 1,
263 	.f_detach = filt_bpfdetach,
264 	.f_event = filt_bpfwrite,
265 };
266 
267 /*
268  * LOCKING MODEL USED BY BPF
269  *
270  * Locks:
271  * 1) global lock (BPF_LOCK). Sx, used to protect some global counters,
272  * every bpf_iflist changes, serializes ioctl access to bpf descriptors.
273  * 2) Descriptor lock. Mutex, used to protect BPF buffers and various
274  * structure fields used by bpf_*tap* code.
275  *
276  * Lock order: global lock, then descriptor lock.
277  *
278  * There are several possible consumers:
279  *
280  * 1. The kernel registers interface pointer with bpfattach().
281  * Each call allocates new bpf_if structure, references ifnet pointer
282  * and links bpf_if into bpf_iflist chain. This is protected with global
283  * lock.
284  *
285  * 2. An userland application uses ioctl() call to bpf_d descriptor.
286  * All such call are serialized with global lock. BPF filters can be
287  * changed, but pointer to old filter will be freed using NET_EPOCH_CALL().
288  * Thus it should be safe for bpf_tap/bpf_mtap* code to do access to
289  * filter pointers, even if change will happen during bpf_tap execution.
290  * Destroying of bpf_d descriptor also is doing using NET_EPOCH_CALL().
291  *
292  * 3. An userland application can write packets into bpf_d descriptor.
293  * There we need to be sure, that ifnet won't disappear during bpfwrite().
294  *
295  * 4. The kernel invokes bpf_tap/bpf_mtap* functions. The access to
296  * bif_dlist is protected with net_epoch_preempt section. So, it should
297  * be safe to make access to bpf_d descriptor inside the section.
298  *
299  * 5. The kernel invokes bpfdetach() on interface destroying. All lists
300  * are modified with global lock held and actual free() is done using
301  * NET_EPOCH_CALL().
302  */
303 
304 static void
305 bpfif_free(epoch_context_t ctx)
306 {
307 	struct bpf_if *bp;
308 
309 	bp = __containerof(ctx, struct bpf_if, epoch_ctx);
310 	if_rele(bp->bif_ifp);
311 	free(bp, M_BPF);
312 }
313 
314 static void
315 bpfif_ref(struct bpf_if *bp)
316 {
317 
318 	refcount_acquire(&bp->bif_refcnt);
319 }
320 
321 static void
322 bpfif_rele(struct bpf_if *bp)
323 {
324 
325 	if (!refcount_release(&bp->bif_refcnt))
326 		return;
327 	NET_EPOCH_CALL(bpfif_free, &bp->epoch_ctx);
328 }
329 
330 static void
331 bpfd_ref(struct bpf_d *d)
332 {
333 
334 	refcount_acquire(&d->bd_refcnt);
335 }
336 
337 static void
338 bpfd_rele(struct bpf_d *d)
339 {
340 
341 	if (!refcount_release(&d->bd_refcnt))
342 		return;
343 	NET_EPOCH_CALL(bpfd_free, &d->epoch_ctx);
344 }
345 
346 static struct bpf_program_buffer*
347 bpf_program_buffer_alloc(size_t size, int flags)
348 {
349 
350 	return (malloc(sizeof(struct bpf_program_buffer) + size,
351 	    M_BPF, flags));
352 }
353 
354 static void
355 bpf_program_buffer_free(epoch_context_t ctx)
356 {
357 	struct bpf_program_buffer *ptr;
358 
359 	ptr = __containerof(ctx, struct bpf_program_buffer, epoch_ctx);
360 #ifdef BPF_JITTER
361 	if (ptr->func != NULL)
362 		bpf_destroy_jit_filter(ptr->func);
363 #endif
364 	free(ptr, M_BPF);
365 }
366 
367 /*
368  * Wrapper functions for various buffering methods.  If the set of buffer
369  * modes expands, we will probably want to introduce a switch data structure
370  * similar to protosw, et.
371  */
372 static void
373 bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
374     u_int len)
375 {
376 
377 	BPFD_LOCK_ASSERT(d);
378 
379 	switch (d->bd_bufmode) {
380 	case BPF_BUFMODE_BUFFER:
381 		return (bpf_buffer_append_bytes(d, buf, offset, src, len));
382 
383 	case BPF_BUFMODE_ZBUF:
384 		counter_u64_add(d->bd_zcopy, 1);
385 		return (bpf_zerocopy_append_bytes(d, buf, offset, src, len));
386 
387 	default:
388 		panic("bpf_buf_append_bytes");
389 	}
390 }
391 
392 static void
393 bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
394     u_int len)
395 {
396 
397 	BPFD_LOCK_ASSERT(d);
398 
399 	switch (d->bd_bufmode) {
400 	case BPF_BUFMODE_BUFFER:
401 		return (bpf_buffer_append_mbuf(d, buf, offset, src, len));
402 
403 	case BPF_BUFMODE_ZBUF:
404 		counter_u64_add(d->bd_zcopy, 1);
405 		return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len));
406 
407 	default:
408 		panic("bpf_buf_append_mbuf");
409 	}
410 }
411 
412 /*
413  * This function gets called when the free buffer is re-assigned.
414  */
415 static void
416 bpf_buf_reclaimed(struct bpf_d *d)
417 {
418 
419 	BPFD_LOCK_ASSERT(d);
420 
421 	switch (d->bd_bufmode) {
422 	case BPF_BUFMODE_BUFFER:
423 		return;
424 
425 	case BPF_BUFMODE_ZBUF:
426 		bpf_zerocopy_buf_reclaimed(d);
427 		return;
428 
429 	default:
430 		panic("bpf_buf_reclaimed");
431 	}
432 }
433 
434 /*
435  * If the buffer mechanism has a way to decide that a held buffer can be made
436  * free, then it is exposed via the bpf_canfreebuf() interface.  (1) is
437  * returned if the buffer can be discarded, (0) is returned if it cannot.
438  */
439 static int
440 bpf_canfreebuf(struct bpf_d *d)
441 {
442 
443 	BPFD_LOCK_ASSERT(d);
444 
445 	switch (d->bd_bufmode) {
446 	case BPF_BUFMODE_ZBUF:
447 		return (bpf_zerocopy_canfreebuf(d));
448 	}
449 	return (0);
450 }
451 
452 /*
453  * Allow the buffer model to indicate that the current store buffer is
454  * immutable, regardless of the appearance of space.  Return (1) if the
455  * buffer is writable, and (0) if not.
456  */
457 static int
458 bpf_canwritebuf(struct bpf_d *d)
459 {
460 	BPFD_LOCK_ASSERT(d);
461 
462 	switch (d->bd_bufmode) {
463 	case BPF_BUFMODE_ZBUF:
464 		return (bpf_zerocopy_canwritebuf(d));
465 	}
466 	return (1);
467 }
468 
469 /*
470  * Notify buffer model that an attempt to write to the store buffer has
471  * resulted in a dropped packet, in which case the buffer may be considered
472  * full.
473  */
474 static void
475 bpf_buffull(struct bpf_d *d)
476 {
477 
478 	BPFD_LOCK_ASSERT(d);
479 
480 	switch (d->bd_bufmode) {
481 	case BPF_BUFMODE_ZBUF:
482 		bpf_zerocopy_buffull(d);
483 		break;
484 	}
485 }
486 
487 /*
488  * Notify the buffer model that a buffer has moved into the hold position.
489  */
490 void
491 bpf_bufheld(struct bpf_d *d)
492 {
493 
494 	BPFD_LOCK_ASSERT(d);
495 
496 	switch (d->bd_bufmode) {
497 	case BPF_BUFMODE_ZBUF:
498 		bpf_zerocopy_bufheld(d);
499 		break;
500 	}
501 }
502 
503 static void
504 bpf_free(struct bpf_d *d)
505 {
506 
507 	switch (d->bd_bufmode) {
508 	case BPF_BUFMODE_BUFFER:
509 		return (bpf_buffer_free(d));
510 
511 	case BPF_BUFMODE_ZBUF:
512 		return (bpf_zerocopy_free(d));
513 
514 	default:
515 		panic("bpf_buf_free");
516 	}
517 }
518 
519 static int
520 bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio)
521 {
522 
523 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
524 		return (EOPNOTSUPP);
525 	return (bpf_buffer_uiomove(d, buf, len, uio));
526 }
527 
528 static int
529 bpf_ioctl_sblen(struct bpf_d *d, u_int *i)
530 {
531 
532 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
533 		return (EOPNOTSUPP);
534 	return (bpf_buffer_ioctl_sblen(d, i));
535 }
536 
537 static int
538 bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
539 {
540 
541 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
542 		return (EOPNOTSUPP);
543 	return (bpf_zerocopy_ioctl_getzmax(td, d, i));
544 }
545 
546 static int
547 bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
548 {
549 
550 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
551 		return (EOPNOTSUPP);
552 	return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz));
553 }
554 
555 static int
556 bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
557 {
558 
559 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
560 		return (EOPNOTSUPP);
561 	return (bpf_zerocopy_ioctl_setzbuf(td, d, bz));
562 }
563 
564 /*
565  * General BPF functions.
566  */
567 static int
568 bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
569     struct sockaddr *sockp, int *hdrlen, struct bpf_d *d)
570 {
571 	const struct ieee80211_bpf_params *p;
572 	struct ether_header *eh;
573 	struct mbuf *m;
574 	int error;
575 	int len;
576 	int hlen;
577 	int slen;
578 
579 	/*
580 	 * Build a sockaddr based on the data link layer type.
581 	 * We do this at this level because the ethernet header
582 	 * is copied directly into the data field of the sockaddr.
583 	 * In the case of SLIP, there is no header and the packet
584 	 * is forwarded as is.
585 	 * Also, we are careful to leave room at the front of the mbuf
586 	 * for the link level header.
587 	 */
588 	switch (linktype) {
589 	case DLT_SLIP:
590 		sockp->sa_family = AF_INET;
591 		hlen = 0;
592 		break;
593 
594 	case DLT_EN10MB:
595 		sockp->sa_family = AF_UNSPEC;
596 		/* XXX Would MAXLINKHDR be better? */
597 		hlen = ETHER_HDR_LEN;
598 		break;
599 
600 	case DLT_FDDI:
601 		sockp->sa_family = AF_IMPLINK;
602 		hlen = 0;
603 		break;
604 
605 	case DLT_RAW:
606 		sockp->sa_family = AF_UNSPEC;
607 		hlen = 0;
608 		break;
609 
610 	case DLT_NULL:
611 		/*
612 		 * null interface types require a 4 byte pseudo header which
613 		 * corresponds to the address family of the packet.
614 		 */
615 		sockp->sa_family = AF_UNSPEC;
616 		hlen = 4;
617 		break;
618 
619 	case DLT_ATM_RFC1483:
620 		/*
621 		 * en atm driver requires 4-byte atm pseudo header.
622 		 * though it isn't standard, vpi:vci needs to be
623 		 * specified anyway.
624 		 */
625 		sockp->sa_family = AF_UNSPEC;
626 		hlen = 12;	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
627 		break;
628 
629 	case DLT_PPP:
630 		sockp->sa_family = AF_UNSPEC;
631 		hlen = 4;	/* This should match PPP_HDRLEN */
632 		break;
633 
634 	case DLT_IEEE802_11:		/* IEEE 802.11 wireless */
635 		sockp->sa_family = AF_IEEE80211;
636 		hlen = 0;
637 		break;
638 
639 	case DLT_IEEE802_11_RADIO:	/* IEEE 802.11 wireless w/ phy params */
640 		sockp->sa_family = AF_IEEE80211;
641 		sockp->sa_len = 12;	/* XXX != 0 */
642 		hlen = sizeof(struct ieee80211_bpf_params);
643 		break;
644 
645 	default:
646 		return (EIO);
647 	}
648 
649 	len = uio->uio_resid;
650 	if (len < hlen || len - hlen > ifp->if_mtu)
651 		return (EMSGSIZE);
652 
653 	/* Allocate a mbuf for our write, since m_get2 fails if len >= to MJUMPAGESIZE, use m_getjcl for bigger buffers */
654 	m = m_get3(len, M_WAITOK, MT_DATA, M_PKTHDR);
655 	if (m == NULL)
656 		return (EIO);
657 	m->m_pkthdr.len = m->m_len = len;
658 	*mp = m;
659 
660 	error = uiomove(mtod(m, u_char *), len, uio);
661 	if (error)
662 		goto bad;
663 
664 	slen = bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len);
665 	if (slen == 0) {
666 		error = EPERM;
667 		goto bad;
668 	}
669 
670 	/* Check for multicast destination */
671 	switch (linktype) {
672 	case DLT_EN10MB:
673 		eh = mtod(m, struct ether_header *);
674 		if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
675 			if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost,
676 			    ETHER_ADDR_LEN) == 0)
677 				m->m_flags |= M_BCAST;
678 			else
679 				m->m_flags |= M_MCAST;
680 		}
681 		if (d->bd_hdrcmplt == 0) {
682 			memcpy(eh->ether_shost, IF_LLADDR(ifp),
683 			    sizeof(eh->ether_shost));
684 		}
685 		break;
686 	}
687 
688 	/*
689 	 * Make room for link header, and copy it to sockaddr
690 	 */
691 	if (hlen != 0) {
692 		if (sockp->sa_family == AF_IEEE80211) {
693 			/*
694 			 * Collect true length from the parameter header
695 			 * NB: sockp is known to be zero'd so if we do a
696 			 *     short copy unspecified parameters will be
697 			 *     zero.
698 			 * NB: packet may not be aligned after stripping
699 			 *     bpf params
700 			 * XXX check ibp_vers
701 			 */
702 			p = mtod(m, const struct ieee80211_bpf_params *);
703 			hlen = p->ibp_len;
704 			if (hlen > sizeof(sockp->sa_data)) {
705 				error = EINVAL;
706 				goto bad;
707 			}
708 		}
709 		bcopy(mtod(m, const void *), sockp->sa_data, hlen);
710 	}
711 	*hdrlen = hlen;
712 
713 	return (0);
714 bad:
715 	m_freem(m);
716 	return (error);
717 }
718 
719 /*
720  * Attach descriptor to the bpf interface, i.e. make d listen on bp,
721  * then reset its buffers and counters with reset_d().
722  */
723 static void
724 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
725 {
726 	int op_w;
727 
728 	BPF_LOCK_ASSERT();
729 
730 	/*
731 	 * Save sysctl value to protect from sysctl change
732 	 * between reads
733 	 */
734 	op_w = V_bpf_optimize_writers || d->bd_writer;
735 
736 	if (d->bd_bif != NULL)
737 		bpf_detachd_locked(d, false);
738 	/*
739 	 * Point d at bp, and add d to the interface's list.
740 	 * Since there are many applications using BPF for
741 	 * sending raw packets only (dhcpd, cdpd are good examples)
742 	 * we can delay adding d to the list of active listeners until
743 	 * some filter is configured.
744 	 */
745 
746 	BPFD_LOCK(d);
747 	/*
748 	 * Hold reference to bpif while descriptor uses this interface.
749 	 */
750 	bpfif_ref(bp);
751 	d->bd_bif = bp;
752 	if (op_w != 0) {
753 		/* Add to writers-only list */
754 		CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next);
755 		/*
756 		 * We decrement bd_writer on every filter set operation.
757 		 * First BIOCSETF is done by pcap_open_live() to set up
758 		 * snap length. After that appliation usually sets its own
759 		 * filter.
760 		 */
761 		d->bd_writer = 2;
762 	} else
763 		CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
764 
765 	reset_d(d);
766 	BPFD_UNLOCK(d);
767 	bpf_bpfd_cnt++;
768 
769 	CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list",
770 	    __func__, d->bd_pid, d->bd_writer ? "writer" : "active");
771 
772 	if (op_w == 0)
773 		EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1);
774 }
775 
776 /*
777  * Check if we need to upgrade our descriptor @d from write-only mode.
778  */
779 static int
780 bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode,
781     int flen)
782 {
783 	int is_snap, need_upgrade;
784 
785 	/*
786 	 * Check if we've already upgraded or new filter is empty.
787 	 */
788 	if (d->bd_writer == 0 || fcode == NULL)
789 		return (0);
790 
791 	need_upgrade = 0;
792 
793 	/*
794 	 * Check if cmd looks like snaplen setting from
795 	 * pcap_bpf.c:pcap_open_live().
796 	 * Note we're not checking .k value here:
797 	 * while pcap_open_live() definitely sets to non-zero value,
798 	 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g.
799 	 * do not consider upgrading immediately
800 	 */
801 	if (cmd == BIOCSETF && flen == 1 &&
802 	    fcode[0].code == (BPF_RET | BPF_K))
803 		is_snap = 1;
804 	else
805 		is_snap = 0;
806 
807 	if (is_snap == 0) {
808 		/*
809 		 * We're setting first filter and it doesn't look like
810 		 * setting snaplen.  We're probably using bpf directly.
811 		 * Upgrade immediately.
812 		 */
813 		need_upgrade = 1;
814 	} else {
815 		/*
816 		 * Do not require upgrade by first BIOCSETF
817 		 * (used to set snaplen) by pcap_open_live().
818 		 */
819 
820 		if (--d->bd_writer == 0) {
821 			/*
822 			 * First snaplen filter has already
823 			 * been set. This is probably catch-all
824 			 * filter
825 			 */
826 			need_upgrade = 1;
827 		}
828 	}
829 
830 	CTR5(KTR_NET,
831 	    "%s: filter function set by pid %d, "
832 	    "bd_writer counter %d, snap %d upgrade %d",
833 	    __func__, d->bd_pid, d->bd_writer,
834 	    is_snap, need_upgrade);
835 
836 	return (need_upgrade);
837 }
838 
839 /*
840  * Detach a file from its interface.
841  */
842 static void
843 bpf_detachd(struct bpf_d *d)
844 {
845 	BPF_LOCK();
846 	bpf_detachd_locked(d, false);
847 	BPF_UNLOCK();
848 }
849 
850 static void
851 bpf_detachd_locked(struct bpf_d *d, bool detached_ifp)
852 {
853 	struct bpf_if *bp;
854 	struct ifnet *ifp;
855 	int error;
856 
857 	BPF_LOCK_ASSERT();
858 	CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid);
859 
860 	/* Check if descriptor is attached */
861 	if ((bp = d->bd_bif) == NULL)
862 		return;
863 
864 	BPFD_LOCK(d);
865 	/* Remove d from the interface's descriptor list. */
866 	CK_LIST_REMOVE(d, bd_next);
867 	/* Save bd_writer value */
868 	error = d->bd_writer;
869 	ifp = bp->bif_ifp;
870 	d->bd_bif = NULL;
871 	if (detached_ifp) {
872 		/*
873 		 * Notify descriptor as it's detached, so that any
874 		 * sleepers wake up and get ENXIO.
875 		 */
876 		bpf_wakeup(d);
877 	}
878 	BPFD_UNLOCK(d);
879 	bpf_bpfd_cnt--;
880 
881 	/* Call event handler iff d is attached */
882 	if (error == 0)
883 		EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0);
884 
885 	/*
886 	 * Check if this descriptor had requested promiscuous mode.
887 	 * If so and ifnet is not detached, turn it off.
888 	 */
889 	if (d->bd_promisc && !detached_ifp) {
890 		d->bd_promisc = 0;
891 		CURVNET_SET(ifp->if_vnet);
892 		error = ifpromisc(ifp, 0);
893 		CURVNET_RESTORE();
894 		if (error != 0 && error != ENXIO) {
895 			/*
896 			 * ENXIO can happen if a pccard is unplugged
897 			 * Something is really wrong if we were able to put
898 			 * the driver into promiscuous mode, but can't
899 			 * take it out.
900 			 */
901 			if_printf(bp->bif_ifp,
902 				"bpf_detach: ifpromisc failed (%d)\n", error);
903 		}
904 	}
905 	bpfif_rele(bp);
906 }
907 
908 /*
909  * Close the descriptor by detaching it from its interface,
910  * deallocating its buffers, and marking it free.
911  */
912 static void
913 bpf_dtor(void *data)
914 {
915 	struct bpf_d *d = data;
916 
917 	BPFD_LOCK(d);
918 	if (d->bd_state == BPF_WAITING)
919 		callout_stop(&d->bd_callout);
920 	d->bd_state = BPF_IDLE;
921 	BPFD_UNLOCK(d);
922 	funsetown(&d->bd_sigio);
923 	bpf_detachd(d);
924 #ifdef MAC
925 	mac_bpfdesc_destroy(d);
926 #endif /* MAC */
927 	seldrain(&d->bd_sel);
928 	knlist_destroy(&d->bd_sel.si_note);
929 	callout_drain(&d->bd_callout);
930 	bpfd_rele(d);
931 }
932 
933 /*
934  * Open ethernet device.  Returns ENXIO for illegal minor device number,
935  * EBUSY if file is open by another process.
936  */
937 /* ARGSUSED */
938 static	int
939 bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
940 {
941 	struct bpf_d *d;
942 	int error;
943 
944 	d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
945 	error = devfs_set_cdevpriv(d, bpf_dtor);
946 	if (error != 0) {
947 		free(d, M_BPF);
948 		return (error);
949 	}
950 
951 	/* Setup counters */
952 	d->bd_rcount = counter_u64_alloc(M_WAITOK);
953 	d->bd_dcount = counter_u64_alloc(M_WAITOK);
954 	d->bd_fcount = counter_u64_alloc(M_WAITOK);
955 	d->bd_wcount = counter_u64_alloc(M_WAITOK);
956 	d->bd_wfcount = counter_u64_alloc(M_WAITOK);
957 	d->bd_wdcount = counter_u64_alloc(M_WAITOK);
958 	d->bd_zcopy = counter_u64_alloc(M_WAITOK);
959 
960 	/*
961 	 * For historical reasons, perform a one-time initialization call to
962 	 * the buffer routines, even though we're not yet committed to a
963 	 * particular buffer method.
964 	 */
965 	bpf_buffer_init(d);
966 	if ((flags & FREAD) == 0)
967 		d->bd_writer = 2;
968 	d->bd_hbuf_in_use = 0;
969 	d->bd_bufmode = BPF_BUFMODE_BUFFER;
970 	d->bd_sig = SIGIO;
971 	d->bd_direction = BPF_D_INOUT;
972 	d->bd_refcnt = 1;
973 	BPF_PID_REFRESH(d, td);
974 #ifdef MAC
975 	mac_bpfdesc_init(d);
976 	mac_bpfdesc_create(td->td_ucred, d);
977 #endif
978 	mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF);
979 	callout_init_mtx(&d->bd_callout, &d->bd_lock, 0);
980 	knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock);
981 
982 	/* Disable VLAN pcp tagging. */
983 	d->bd_pcp = 0;
984 
985 	return (0);
986 }
987 
988 /*
989  *  bpfread - read next chunk of packets from buffers
990  */
991 static	int
992 bpfread(struct cdev *dev, struct uio *uio, int ioflag)
993 {
994 	struct bpf_d *d;
995 	int error;
996 	int non_block;
997 	int timed_out;
998 
999 	error = devfs_get_cdevpriv((void **)&d);
1000 	if (error != 0)
1001 		return (error);
1002 
1003 	/*
1004 	 * Restrict application to use a buffer the same size as
1005 	 * as kernel buffers.
1006 	 */
1007 	if (uio->uio_resid != d->bd_bufsize)
1008 		return (EINVAL);
1009 
1010 	non_block = ((ioflag & O_NONBLOCK) != 0);
1011 
1012 	BPFD_LOCK(d);
1013 	BPF_PID_REFRESH_CUR(d);
1014 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER) {
1015 		BPFD_UNLOCK(d);
1016 		return (EOPNOTSUPP);
1017 	}
1018 	if (d->bd_state == BPF_WAITING)
1019 		callout_stop(&d->bd_callout);
1020 	timed_out = (d->bd_state == BPF_TIMED_OUT);
1021 	d->bd_state = BPF_IDLE;
1022 	while (d->bd_hbuf_in_use) {
1023 		error = mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
1024 		    PRINET|PCATCH, "bd_hbuf", 0);
1025 		if (error != 0) {
1026 			BPFD_UNLOCK(d);
1027 			return (error);
1028 		}
1029 	}
1030 	/*
1031 	 * If the hold buffer is empty, then do a timed sleep, which
1032 	 * ends when the timeout expires or when enough packets
1033 	 * have arrived to fill the store buffer.
1034 	 */
1035 	while (d->bd_hbuf == NULL) {
1036 		if (d->bd_slen != 0) {
1037 			/*
1038 			 * A packet(s) either arrived since the previous
1039 			 * read or arrived while we were asleep.
1040 			 */
1041 			if (d->bd_immediate || non_block || timed_out) {
1042 				/*
1043 				 * Rotate the buffers and return what's here
1044 				 * if we are in immediate mode, non-blocking
1045 				 * flag is set, or this descriptor timed out.
1046 				 */
1047 				ROTATE_BUFFERS(d);
1048 				break;
1049 			}
1050 		}
1051 
1052 		/*
1053 		 * No data is available, check to see if the bpf device
1054 		 * is still pointed at a real interface.  If not, return
1055 		 * ENXIO so that the userland process knows to rebind
1056 		 * it before using it again.
1057 		 */
1058 		if (d->bd_bif == NULL) {
1059 			BPFD_UNLOCK(d);
1060 			return (ENXIO);
1061 		}
1062 
1063 		if (non_block) {
1064 			BPFD_UNLOCK(d);
1065 			return (EWOULDBLOCK);
1066 		}
1067 		error = msleep(d, &d->bd_lock, PRINET|PCATCH,
1068 		     "bpf", d->bd_rtout);
1069 		if (error == EINTR || error == ERESTART) {
1070 			BPFD_UNLOCK(d);
1071 			return (error);
1072 		}
1073 		if (error == EWOULDBLOCK) {
1074 			/*
1075 			 * On a timeout, return what's in the buffer,
1076 			 * which may be nothing.  If there is something
1077 			 * in the store buffer, we can rotate the buffers.
1078 			 */
1079 			if (d->bd_hbuf)
1080 				/*
1081 				 * We filled up the buffer in between
1082 				 * getting the timeout and arriving
1083 				 * here, so we don't need to rotate.
1084 				 */
1085 				break;
1086 
1087 			if (d->bd_slen == 0) {
1088 				BPFD_UNLOCK(d);
1089 				return (0);
1090 			}
1091 			ROTATE_BUFFERS(d);
1092 			break;
1093 		}
1094 	}
1095 	/*
1096 	 * At this point, we know we have something in the hold slot.
1097 	 */
1098 	d->bd_hbuf_in_use = 1;
1099 	BPFD_UNLOCK(d);
1100 
1101 	/*
1102 	 * Move data from hold buffer into user space.
1103 	 * We know the entire buffer is transferred since
1104 	 * we checked above that the read buffer is bpf_bufsize bytes.
1105   	 *
1106 	 * We do not have to worry about simultaneous reads because
1107 	 * we waited for sole access to the hold buffer above.
1108 	 */
1109 	error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio);
1110 
1111 	BPFD_LOCK(d);
1112 	KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf"));
1113 	d->bd_fbuf = d->bd_hbuf;
1114 	d->bd_hbuf = NULL;
1115 	d->bd_hlen = 0;
1116 	bpf_buf_reclaimed(d);
1117 	d->bd_hbuf_in_use = 0;
1118 	wakeup(&d->bd_hbuf_in_use);
1119 	BPFD_UNLOCK(d);
1120 
1121 	return (error);
1122 }
1123 
1124 /*
1125  * If there are processes sleeping on this descriptor, wake them up.
1126  */
1127 static __inline void
1128 bpf_wakeup(struct bpf_d *d)
1129 {
1130 
1131 	BPFD_LOCK_ASSERT(d);
1132 	if (d->bd_state == BPF_WAITING) {
1133 		callout_stop(&d->bd_callout);
1134 		d->bd_state = BPF_IDLE;
1135 	}
1136 	wakeup(d);
1137 	if (d->bd_async && d->bd_sig && d->bd_sigio)
1138 		pgsigio(&d->bd_sigio, d->bd_sig, 0);
1139 
1140 	selwakeuppri(&d->bd_sel, PRINET);
1141 	KNOTE_LOCKED(&d->bd_sel.si_note, 0);
1142 }
1143 
1144 static void
1145 bpf_timed_out(void *arg)
1146 {
1147 	struct bpf_d *d = (struct bpf_d *)arg;
1148 
1149 	BPFD_LOCK_ASSERT(d);
1150 
1151 	if (callout_pending(&d->bd_callout) ||
1152 	    !callout_active(&d->bd_callout))
1153 		return;
1154 	if (d->bd_state == BPF_WAITING) {
1155 		d->bd_state = BPF_TIMED_OUT;
1156 		if (d->bd_slen != 0)
1157 			bpf_wakeup(d);
1158 	}
1159 }
1160 
1161 static int
1162 bpf_ready(struct bpf_d *d)
1163 {
1164 
1165 	BPFD_LOCK_ASSERT(d);
1166 
1167 	if (!bpf_canfreebuf(d) && d->bd_hlen != 0)
1168 		return (1);
1169 	if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
1170 	    d->bd_slen != 0)
1171 		return (1);
1172 	return (0);
1173 }
1174 
1175 static int
1176 bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
1177 {
1178 	struct route ro;
1179 	struct sockaddr dst;
1180 	struct epoch_tracker et;
1181 	struct bpf_if *bp;
1182 	struct bpf_d *d;
1183 	struct ifnet *ifp;
1184 	struct mbuf *m, *mc;
1185 	int error, hlen;
1186 
1187 	error = devfs_get_cdevpriv((void **)&d);
1188 	if (error != 0)
1189 		return (error);
1190 
1191 	NET_EPOCH_ENTER(et);
1192 	BPFD_LOCK(d);
1193 	BPF_PID_REFRESH_CUR(d);
1194 	counter_u64_add(d->bd_wcount, 1);
1195 	if ((bp = d->bd_bif) == NULL) {
1196 		error = ENXIO;
1197 		goto out_locked;
1198 	}
1199 
1200 	ifp = bp->bif_ifp;
1201 	if ((ifp->if_flags & IFF_UP) == 0) {
1202 		error = ENETDOWN;
1203 		goto out_locked;
1204 	}
1205 
1206 	if (uio->uio_resid == 0)
1207 		goto out_locked;
1208 
1209 	bzero(&dst, sizeof(dst));
1210 	m = NULL;
1211 	hlen = 0;
1212 
1213 	/*
1214 	 * Take extra reference, unlock d and exit from epoch section,
1215 	 * since bpf_movein() can sleep.
1216 	 */
1217 	bpfd_ref(d);
1218 	NET_EPOCH_EXIT(et);
1219 	BPFD_UNLOCK(d);
1220 
1221 	error = bpf_movein(uio, (int)bp->bif_dlt, ifp,
1222 	    &m, &dst, &hlen, d);
1223 
1224 	if (error != 0) {
1225 		counter_u64_add(d->bd_wdcount, 1);
1226 		bpfd_rele(d);
1227 		return (error);
1228 	}
1229 
1230 	BPFD_LOCK(d);
1231 	/*
1232 	 * Check that descriptor is still attached to the interface.
1233 	 * This can happen on bpfdetach(). To avoid access to detached
1234 	 * ifnet, free mbuf and return ENXIO.
1235 	 */
1236 	if (d->bd_bif == NULL) {
1237 		counter_u64_add(d->bd_wdcount, 1);
1238 		BPFD_UNLOCK(d);
1239 		bpfd_rele(d);
1240 		m_freem(m);
1241 		return (ENXIO);
1242 	}
1243 	counter_u64_add(d->bd_wfcount, 1);
1244 	if (d->bd_hdrcmplt)
1245 		dst.sa_family = pseudo_AF_HDRCMPLT;
1246 
1247 	if (d->bd_feedback) {
1248 		mc = m_dup(m, M_NOWAIT);
1249 		if (mc != NULL)
1250 			mc->m_pkthdr.rcvif = ifp;
1251 		/* Set M_PROMISC for outgoing packets to be discarded. */
1252 		if (d->bd_direction == BPF_D_INOUT)
1253 			m->m_flags |= M_PROMISC;
1254 	} else
1255 		mc = NULL;
1256 
1257 	m->m_pkthdr.len -= hlen;
1258 	m->m_len -= hlen;
1259 	m->m_data += hlen;	/* XXX */
1260 
1261 	CURVNET_SET(ifp->if_vnet);
1262 #ifdef MAC
1263 	mac_bpfdesc_create_mbuf(d, m);
1264 	if (mc != NULL)
1265 		mac_bpfdesc_create_mbuf(d, mc);
1266 #endif
1267 
1268 	bzero(&ro, sizeof(ro));
1269 	if (hlen != 0) {
1270 		ro.ro_prepend = (u_char *)&dst.sa_data;
1271 		ro.ro_plen = hlen;
1272 		ro.ro_flags = RT_HAS_HEADER;
1273 	}
1274 
1275 	if (d->bd_pcp != 0)
1276 		vlan_set_pcp(m, d->bd_pcp);
1277 
1278 	/* Avoid possible recursion on BPFD_LOCK(). */
1279 	NET_EPOCH_ENTER(et);
1280 	BPFD_UNLOCK(d);
1281 	error = (*ifp->if_output)(ifp, m, &dst, &ro);
1282 	if (error)
1283 		counter_u64_add(d->bd_wdcount, 1);
1284 
1285 	if (mc != NULL) {
1286 		if (error == 0)
1287 			(*ifp->if_input)(ifp, mc);
1288 		else
1289 			m_freem(mc);
1290 	}
1291 	NET_EPOCH_EXIT(et);
1292 	CURVNET_RESTORE();
1293 	bpfd_rele(d);
1294 	return (error);
1295 
1296 out_locked:
1297 	counter_u64_add(d->bd_wdcount, 1);
1298 	NET_EPOCH_EXIT(et);
1299 	BPFD_UNLOCK(d);
1300 	return (error);
1301 }
1302 
1303 /*
1304  * Reset a descriptor by flushing its packet buffer and clearing the receive
1305  * and drop counts.  This is doable for kernel-only buffers, but with
1306  * zero-copy buffers, we can't write to (or rotate) buffers that are
1307  * currently owned by userspace.  It would be nice if we could encapsulate
1308  * this logic in the buffer code rather than here.
1309  */
1310 static void
1311 reset_d(struct bpf_d *d)
1312 {
1313 
1314 	BPFD_LOCK_ASSERT(d);
1315 
1316 	while (d->bd_hbuf_in_use)
1317 		mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, PRINET,
1318 		    "bd_hbuf", 0);
1319 	if ((d->bd_hbuf != NULL) &&
1320 	    (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) {
1321 		/* Free the hold buffer. */
1322 		d->bd_fbuf = d->bd_hbuf;
1323 		d->bd_hbuf = NULL;
1324 		d->bd_hlen = 0;
1325 		bpf_buf_reclaimed(d);
1326 	}
1327 	if (bpf_canwritebuf(d))
1328 		d->bd_slen = 0;
1329 	counter_u64_zero(d->bd_rcount);
1330 	counter_u64_zero(d->bd_dcount);
1331 	counter_u64_zero(d->bd_fcount);
1332 	counter_u64_zero(d->bd_wcount);
1333 	counter_u64_zero(d->bd_wfcount);
1334 	counter_u64_zero(d->bd_wdcount);
1335 	counter_u64_zero(d->bd_zcopy);
1336 }
1337 
1338 /*
1339  *  FIONREAD		Check for read packet available.
1340  *  BIOCGBLEN		Get buffer len [for read()].
1341  *  BIOCSETF		Set read filter.
1342  *  BIOCSETFNR		Set read filter without resetting descriptor.
1343  *  BIOCSETWF		Set write filter.
1344  *  BIOCFLUSH		Flush read packet buffer.
1345  *  BIOCPROMISC		Put interface into promiscuous mode.
1346  *  BIOCGDLT		Get link layer type.
1347  *  BIOCGETIF		Get interface name.
1348  *  BIOCSETIF		Set interface.
1349  *  BIOCSRTIMEOUT	Set read timeout.
1350  *  BIOCGRTIMEOUT	Get read timeout.
1351  *  BIOCGSTATS		Get packet stats.
1352  *  BIOCIMMEDIATE	Set immediate mode.
1353  *  BIOCVERSION		Get filter language version.
1354  *  BIOCGHDRCMPLT	Get "header already complete" flag
1355  *  BIOCSHDRCMPLT	Set "header already complete" flag
1356  *  BIOCGDIRECTION	Get packet direction flag
1357  *  BIOCSDIRECTION	Set packet direction flag
1358  *  BIOCGTSTAMP		Get time stamp format and resolution.
1359  *  BIOCSTSTAMP		Set time stamp format and resolution.
1360  *  BIOCLOCK		Set "locked" flag
1361  *  BIOCFEEDBACK	Set packet feedback mode.
1362  *  BIOCSETZBUF		Set current zero-copy buffer locations.
1363  *  BIOCGETZMAX		Get maximum zero-copy buffer size.
1364  *  BIOCROTZBUF		Force rotation of zero-copy buffer
1365  *  BIOCSETBUFMODE	Set buffer mode.
1366  *  BIOCGETBUFMODE	Get current buffer mode.
1367  *  BIOCSETVLANPCP	Set VLAN PCP tag.
1368  */
1369 /* ARGSUSED */
1370 static	int
1371 bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
1372     struct thread *td)
1373 {
1374 	struct bpf_d *d;
1375 	int error;
1376 
1377 	error = devfs_get_cdevpriv((void **)&d);
1378 	if (error != 0)
1379 		return (error);
1380 
1381 	/*
1382 	 * Refresh PID associated with this descriptor.
1383 	 */
1384 	BPFD_LOCK(d);
1385 	BPF_PID_REFRESH(d, td);
1386 	if (d->bd_state == BPF_WAITING)
1387 		callout_stop(&d->bd_callout);
1388 	d->bd_state = BPF_IDLE;
1389 	BPFD_UNLOCK(d);
1390 
1391 	if (d->bd_locked == 1) {
1392 		switch (cmd) {
1393 		case BIOCGBLEN:
1394 		case BIOCFLUSH:
1395 		case BIOCGDLT:
1396 		case BIOCGDLTLIST:
1397 #ifdef COMPAT_FREEBSD32
1398 		case BIOCGDLTLIST32:
1399 #endif
1400 		case BIOCGETIF:
1401 		case BIOCGRTIMEOUT:
1402 #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1403 		case BIOCGRTIMEOUT32:
1404 #endif
1405 		case BIOCGSTATS:
1406 		case BIOCVERSION:
1407 		case BIOCGRSIG:
1408 		case BIOCGHDRCMPLT:
1409 		case BIOCSTSTAMP:
1410 		case BIOCFEEDBACK:
1411 		case FIONREAD:
1412 		case BIOCLOCK:
1413 		case BIOCSRTIMEOUT:
1414 #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1415 		case BIOCSRTIMEOUT32:
1416 #endif
1417 		case BIOCIMMEDIATE:
1418 		case TIOCGPGRP:
1419 		case BIOCROTZBUF:
1420 			break;
1421 		default:
1422 			return (EPERM);
1423 		}
1424 	}
1425 #ifdef COMPAT_FREEBSD32
1426 	/*
1427 	 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so
1428 	 * that it will get 32-bit packet headers.
1429 	 */
1430 	switch (cmd) {
1431 	case BIOCSETF32:
1432 	case BIOCSETFNR32:
1433 	case BIOCSETWF32:
1434 	case BIOCGDLTLIST32:
1435 	case BIOCGRTIMEOUT32:
1436 	case BIOCSRTIMEOUT32:
1437 		if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
1438 			BPFD_LOCK(d);
1439 			d->bd_compat32 = 1;
1440 			BPFD_UNLOCK(d);
1441 		}
1442 	}
1443 #endif
1444 
1445 	CURVNET_SET(TD_TO_VNET(td));
1446 	switch (cmd) {
1447 	default:
1448 		error = EINVAL;
1449 		break;
1450 
1451 	/*
1452 	 * Check for read packet available.
1453 	 */
1454 	case FIONREAD:
1455 		{
1456 			int n;
1457 
1458 			BPFD_LOCK(d);
1459 			n = d->bd_slen;
1460 			while (d->bd_hbuf_in_use)
1461 				mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
1462 				    PRINET, "bd_hbuf", 0);
1463 			if (d->bd_hbuf)
1464 				n += d->bd_hlen;
1465 			BPFD_UNLOCK(d);
1466 
1467 			*(int *)addr = n;
1468 			break;
1469 		}
1470 
1471 	/*
1472 	 * Get buffer len [for read()].
1473 	 */
1474 	case BIOCGBLEN:
1475 		BPFD_LOCK(d);
1476 		*(u_int *)addr = d->bd_bufsize;
1477 		BPFD_UNLOCK(d);
1478 		break;
1479 
1480 	/*
1481 	 * Set buffer length.
1482 	 */
1483 	case BIOCSBLEN:
1484 		error = bpf_ioctl_sblen(d, (u_int *)addr);
1485 		break;
1486 
1487 	/*
1488 	 * Set link layer read filter.
1489 	 */
1490 	case BIOCSETF:
1491 	case BIOCSETFNR:
1492 	case BIOCSETWF:
1493 #ifdef COMPAT_FREEBSD32
1494 	case BIOCSETF32:
1495 	case BIOCSETFNR32:
1496 	case BIOCSETWF32:
1497 #endif
1498 		error = bpf_setf(d, (struct bpf_program *)addr, cmd);
1499 		break;
1500 
1501 	/*
1502 	 * Flush read packet buffer.
1503 	 */
1504 	case BIOCFLUSH:
1505 		BPFD_LOCK(d);
1506 		reset_d(d);
1507 		BPFD_UNLOCK(d);
1508 		break;
1509 
1510 	/*
1511 	 * Put interface into promiscuous mode.
1512 	 */
1513 	case BIOCPROMISC:
1514 		if (d->bd_bif == NULL) {
1515 			/*
1516 			 * No interface attached yet.
1517 			 */
1518 			error = EINVAL;
1519 			break;
1520 		}
1521 		if (d->bd_promisc == 0) {
1522 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
1523 			if (error == 0)
1524 				d->bd_promisc = 1;
1525 		}
1526 		break;
1527 
1528 	/*
1529 	 * Get current data link type.
1530 	 */
1531 	case BIOCGDLT:
1532 		BPF_LOCK();
1533 		if (d->bd_bif == NULL)
1534 			error = EINVAL;
1535 		else
1536 			*(u_int *)addr = d->bd_bif->bif_dlt;
1537 		BPF_UNLOCK();
1538 		break;
1539 
1540 	/*
1541 	 * Get a list of supported data link types.
1542 	 */
1543 #ifdef COMPAT_FREEBSD32
1544 	case BIOCGDLTLIST32:
1545 		{
1546 			struct bpf_dltlist32 *list32;
1547 			struct bpf_dltlist dltlist;
1548 
1549 			list32 = (struct bpf_dltlist32 *)addr;
1550 			dltlist.bfl_len = list32->bfl_len;
1551 			dltlist.bfl_list = PTRIN(list32->bfl_list);
1552 			BPF_LOCK();
1553 			if (d->bd_bif == NULL)
1554 				error = EINVAL;
1555 			else {
1556 				error = bpf_getdltlist(d, &dltlist);
1557 				if (error == 0)
1558 					list32->bfl_len = dltlist.bfl_len;
1559 			}
1560 			BPF_UNLOCK();
1561 			break;
1562 		}
1563 #endif
1564 
1565 	case BIOCGDLTLIST:
1566 		BPF_LOCK();
1567 		if (d->bd_bif == NULL)
1568 			error = EINVAL;
1569 		else
1570 			error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
1571 		BPF_UNLOCK();
1572 		break;
1573 
1574 	/*
1575 	 * Set data link type.
1576 	 */
1577 	case BIOCSDLT:
1578 		BPF_LOCK();
1579 		if (d->bd_bif == NULL)
1580 			error = EINVAL;
1581 		else
1582 			error = bpf_setdlt(d, *(u_int *)addr);
1583 		BPF_UNLOCK();
1584 		break;
1585 
1586 	/*
1587 	 * Get interface name.
1588 	 */
1589 	case BIOCGETIF:
1590 		BPF_LOCK();
1591 		if (d->bd_bif == NULL)
1592 			error = EINVAL;
1593 		else {
1594 			struct ifnet *const ifp = d->bd_bif->bif_ifp;
1595 			struct ifreq *const ifr = (struct ifreq *)addr;
1596 
1597 			strlcpy(ifr->ifr_name, ifp->if_xname,
1598 			    sizeof(ifr->ifr_name));
1599 		}
1600 		BPF_UNLOCK();
1601 		break;
1602 
1603 	/*
1604 	 * Set interface.
1605 	 */
1606 	case BIOCSETIF:
1607 		{
1608 			int alloc_buf, size;
1609 
1610 			/*
1611 			 * Behavior here depends on the buffering model.  If
1612 			 * we're using kernel memory buffers, then we can
1613 			 * allocate them here.  If we're using zero-copy,
1614 			 * then the user process must have registered buffers
1615 			 * by the time we get here.
1616 			 */
1617 			alloc_buf = 0;
1618 			BPFD_LOCK(d);
1619 			if (d->bd_bufmode == BPF_BUFMODE_BUFFER &&
1620 			    d->bd_sbuf == NULL)
1621 				alloc_buf = 1;
1622 			BPFD_UNLOCK(d);
1623 			if (alloc_buf) {
1624 				size = d->bd_bufsize;
1625 				error = bpf_buffer_ioctl_sblen(d, &size);
1626 				if (error != 0)
1627 					break;
1628 			}
1629 			BPF_LOCK();
1630 			error = bpf_setif(d, (struct ifreq *)addr);
1631 			BPF_UNLOCK();
1632 			break;
1633 		}
1634 
1635 	/*
1636 	 * Set read timeout.
1637 	 */
1638 	case BIOCSRTIMEOUT:
1639 #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1640 	case BIOCSRTIMEOUT32:
1641 #endif
1642 		{
1643 			struct timeval *tv = (struct timeval *)addr;
1644 #if defined(COMPAT_FREEBSD32) && !defined(__mips__)
1645 			struct timeval32 *tv32;
1646 			struct timeval tv64;
1647 
1648 			if (cmd == BIOCSRTIMEOUT32) {
1649 				tv32 = (struct timeval32 *)addr;
1650 				tv = &tv64;
1651 				tv->tv_sec = tv32->tv_sec;
1652 				tv->tv_usec = tv32->tv_usec;
1653 			} else
1654 #endif
1655 				tv = (struct timeval *)addr;
1656 
1657 			/*
1658 			 * Subtract 1 tick from tvtohz() since this isn't
1659 			 * a one-shot timer.
1660 			 */
1661 			if ((error = itimerfix(tv)) == 0)
1662 				d->bd_rtout = tvtohz(tv) - 1;
1663 			break;
1664 		}
1665 
1666 	/*
1667 	 * Get read timeout.
1668 	 */
1669 	case BIOCGRTIMEOUT:
1670 #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1671 	case BIOCGRTIMEOUT32:
1672 #endif
1673 		{
1674 			struct timeval *tv;
1675 #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1676 			struct timeval32 *tv32;
1677 			struct timeval tv64;
1678 
1679 			if (cmd == BIOCGRTIMEOUT32)
1680 				tv = &tv64;
1681 			else
1682 #endif
1683 				tv = (struct timeval *)addr;
1684 
1685 			tv->tv_sec = d->bd_rtout / hz;
1686 			tv->tv_usec = (d->bd_rtout % hz) * tick;
1687 #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1688 			if (cmd == BIOCGRTIMEOUT32) {
1689 				tv32 = (struct timeval32 *)addr;
1690 				tv32->tv_sec = tv->tv_sec;
1691 				tv32->tv_usec = tv->tv_usec;
1692 			}
1693 #endif
1694 
1695 			break;
1696 		}
1697 
1698 	/*
1699 	 * Get packet stats.
1700 	 */
1701 	case BIOCGSTATS:
1702 		{
1703 			struct bpf_stat *bs = (struct bpf_stat *)addr;
1704 
1705 			/* XXXCSJP overflow */
1706 			bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount);
1707 			bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount);
1708 			break;
1709 		}
1710 
1711 	/*
1712 	 * Set immediate mode.
1713 	 */
1714 	case BIOCIMMEDIATE:
1715 		BPFD_LOCK(d);
1716 		d->bd_immediate = *(u_int *)addr;
1717 		BPFD_UNLOCK(d);
1718 		break;
1719 
1720 	case BIOCVERSION:
1721 		{
1722 			struct bpf_version *bv = (struct bpf_version *)addr;
1723 
1724 			bv->bv_major = BPF_MAJOR_VERSION;
1725 			bv->bv_minor = BPF_MINOR_VERSION;
1726 			break;
1727 		}
1728 
1729 	/*
1730 	 * Get "header already complete" flag
1731 	 */
1732 	case BIOCGHDRCMPLT:
1733 		BPFD_LOCK(d);
1734 		*(u_int *)addr = d->bd_hdrcmplt;
1735 		BPFD_UNLOCK(d);
1736 		break;
1737 
1738 	/*
1739 	 * Set "header already complete" flag
1740 	 */
1741 	case BIOCSHDRCMPLT:
1742 		BPFD_LOCK(d);
1743 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1744 		BPFD_UNLOCK(d);
1745 		break;
1746 
1747 	/*
1748 	 * Get packet direction flag
1749 	 */
1750 	case BIOCGDIRECTION:
1751 		BPFD_LOCK(d);
1752 		*(u_int *)addr = d->bd_direction;
1753 		BPFD_UNLOCK(d);
1754 		break;
1755 
1756 	/*
1757 	 * Set packet direction flag
1758 	 */
1759 	case BIOCSDIRECTION:
1760 		{
1761 			u_int	direction;
1762 
1763 			direction = *(u_int *)addr;
1764 			switch (direction) {
1765 			case BPF_D_IN:
1766 			case BPF_D_INOUT:
1767 			case BPF_D_OUT:
1768 				BPFD_LOCK(d);
1769 				d->bd_direction = direction;
1770 				BPFD_UNLOCK(d);
1771 				break;
1772 			default:
1773 				error = EINVAL;
1774 			}
1775 		}
1776 		break;
1777 
1778 	/*
1779 	 * Get packet timestamp format and resolution.
1780 	 */
1781 	case BIOCGTSTAMP:
1782 		BPFD_LOCK(d);
1783 		*(u_int *)addr = d->bd_tstamp;
1784 		BPFD_UNLOCK(d);
1785 		break;
1786 
1787 	/*
1788 	 * Set packet timestamp format and resolution.
1789 	 */
1790 	case BIOCSTSTAMP:
1791 		{
1792 			u_int	func;
1793 
1794 			func = *(u_int *)addr;
1795 			if (BPF_T_VALID(func))
1796 				d->bd_tstamp = func;
1797 			else
1798 				error = EINVAL;
1799 		}
1800 		break;
1801 
1802 	case BIOCFEEDBACK:
1803 		BPFD_LOCK(d);
1804 		d->bd_feedback = *(u_int *)addr;
1805 		BPFD_UNLOCK(d);
1806 		break;
1807 
1808 	case BIOCLOCK:
1809 		BPFD_LOCK(d);
1810 		d->bd_locked = 1;
1811 		BPFD_UNLOCK(d);
1812 		break;
1813 
1814 	case FIONBIO:		/* Non-blocking I/O */
1815 		break;
1816 
1817 	case FIOASYNC:		/* Send signal on receive packets */
1818 		BPFD_LOCK(d);
1819 		d->bd_async = *(int *)addr;
1820 		BPFD_UNLOCK(d);
1821 		break;
1822 
1823 	case FIOSETOWN:
1824 		/*
1825 		 * XXX: Add some sort of locking here?
1826 		 * fsetown() can sleep.
1827 		 */
1828 		error = fsetown(*(int *)addr, &d->bd_sigio);
1829 		break;
1830 
1831 	case FIOGETOWN:
1832 		BPFD_LOCK(d);
1833 		*(int *)addr = fgetown(&d->bd_sigio);
1834 		BPFD_UNLOCK(d);
1835 		break;
1836 
1837 	/* This is deprecated, FIOSETOWN should be used instead. */
1838 	case TIOCSPGRP:
1839 		error = fsetown(-(*(int *)addr), &d->bd_sigio);
1840 		break;
1841 
1842 	/* This is deprecated, FIOGETOWN should be used instead. */
1843 	case TIOCGPGRP:
1844 		*(int *)addr = -fgetown(&d->bd_sigio);
1845 		break;
1846 
1847 	case BIOCSRSIG:		/* Set receive signal */
1848 		{
1849 			u_int sig;
1850 
1851 			sig = *(u_int *)addr;
1852 
1853 			if (sig >= NSIG)
1854 				error = EINVAL;
1855 			else {
1856 				BPFD_LOCK(d);
1857 				d->bd_sig = sig;
1858 				BPFD_UNLOCK(d);
1859 			}
1860 			break;
1861 		}
1862 	case BIOCGRSIG:
1863 		BPFD_LOCK(d);
1864 		*(u_int *)addr = d->bd_sig;
1865 		BPFD_UNLOCK(d);
1866 		break;
1867 
1868 	case BIOCGETBUFMODE:
1869 		BPFD_LOCK(d);
1870 		*(u_int *)addr = d->bd_bufmode;
1871 		BPFD_UNLOCK(d);
1872 		break;
1873 
1874 	case BIOCSETBUFMODE:
1875 		/*
1876 		 * Allow the buffering mode to be changed as long as we
1877 		 * haven't yet committed to a particular mode.  Our
1878 		 * definition of commitment, for now, is whether or not a
1879 		 * buffer has been allocated or an interface attached, since
1880 		 * that's the point where things get tricky.
1881 		 */
1882 		switch (*(u_int *)addr) {
1883 		case BPF_BUFMODE_BUFFER:
1884 			break;
1885 
1886 		case BPF_BUFMODE_ZBUF:
1887 			if (bpf_zerocopy_enable)
1888 				break;
1889 			/* FALLSTHROUGH */
1890 
1891 		default:
1892 			CURVNET_RESTORE();
1893 			return (EINVAL);
1894 		}
1895 
1896 		BPFD_LOCK(d);
1897 		if (d->bd_sbuf != NULL || d->bd_hbuf != NULL ||
1898 		    d->bd_fbuf != NULL || d->bd_bif != NULL) {
1899 			BPFD_UNLOCK(d);
1900 			CURVNET_RESTORE();
1901 			return (EBUSY);
1902 		}
1903 		d->bd_bufmode = *(u_int *)addr;
1904 		BPFD_UNLOCK(d);
1905 		break;
1906 
1907 	case BIOCGETZMAX:
1908 		error = bpf_ioctl_getzmax(td, d, (size_t *)addr);
1909 		break;
1910 
1911 	case BIOCSETZBUF:
1912 		error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr);
1913 		break;
1914 
1915 	case BIOCROTZBUF:
1916 		error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr);
1917 		break;
1918 
1919 	case BIOCSETVLANPCP:
1920 		{
1921 			u_int pcp;
1922 
1923 			pcp = *(u_int *)addr;
1924 			if (pcp > BPF_PRIO_MAX || pcp < 0) {
1925 				error = EINVAL;
1926 				break;
1927 			}
1928 			d->bd_pcp = pcp;
1929 			break;
1930 		}
1931 	}
1932 	CURVNET_RESTORE();
1933 	return (error);
1934 }
1935 
1936 /*
1937  * Set d's packet filter program to fp. If this file already has a filter,
1938  * free it and replace it. Returns EINVAL for bogus requests.
1939  *
1940  * Note we use global lock here to serialize bpf_setf() and bpf_setif()
1941  * calls.
1942  */
1943 static int
1944 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1945 {
1946 #ifdef COMPAT_FREEBSD32
1947 	struct bpf_program fp_swab;
1948 	struct bpf_program32 *fp32;
1949 #endif
1950 	struct bpf_program_buffer *fcode;
1951 	struct bpf_insn *filter;
1952 #ifdef BPF_JITTER
1953 	bpf_jit_filter *jfunc;
1954 #endif
1955 	size_t size;
1956 	u_int flen;
1957 	bool track_event;
1958 
1959 #ifdef COMPAT_FREEBSD32
1960 	switch (cmd) {
1961 	case BIOCSETF32:
1962 	case BIOCSETWF32:
1963 	case BIOCSETFNR32:
1964 		fp32 = (struct bpf_program32 *)fp;
1965 		fp_swab.bf_len = fp32->bf_len;
1966 		fp_swab.bf_insns =
1967 		    (struct bpf_insn *)(uintptr_t)fp32->bf_insns;
1968 		fp = &fp_swab;
1969 		switch (cmd) {
1970 		case BIOCSETF32:
1971 			cmd = BIOCSETF;
1972 			break;
1973 		case BIOCSETWF32:
1974 			cmd = BIOCSETWF;
1975 			break;
1976 		}
1977 		break;
1978 	}
1979 #endif
1980 
1981 	filter = NULL;
1982 #ifdef BPF_JITTER
1983 	jfunc = NULL;
1984 #endif
1985 	/*
1986 	 * Check new filter validness before acquiring any locks.
1987 	 * Allocate memory for new filter, if needed.
1988 	 */
1989 	flen = fp->bf_len;
1990 	if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0))
1991 		return (EINVAL);
1992 	size = flen * sizeof(*fp->bf_insns);
1993 	if (size > 0) {
1994 		/* We're setting up new filter. Copy and check actual data. */
1995 		fcode = bpf_program_buffer_alloc(size, M_WAITOK);
1996 		filter = (struct bpf_insn *)fcode->buffer;
1997 		if (copyin(fp->bf_insns, filter, size) != 0 ||
1998 		    !bpf_validate(filter, flen)) {
1999 			free(fcode, M_BPF);
2000 			return (EINVAL);
2001 		}
2002 #ifdef BPF_JITTER
2003 		if (cmd != BIOCSETWF) {
2004 			/*
2005 			 * Filter is copied inside fcode and is
2006 			 * perfectly valid.
2007 			 */
2008 			jfunc = bpf_jitter(filter, flen);
2009 		}
2010 #endif
2011 	}
2012 
2013 	track_event = false;
2014 	fcode = NULL;
2015 
2016 	BPF_LOCK();
2017 	BPFD_LOCK(d);
2018 	/* Set up new filter. */
2019 	if (cmd == BIOCSETWF) {
2020 		if (d->bd_wfilter != NULL) {
2021 			fcode = __containerof((void *)d->bd_wfilter,
2022 			    struct bpf_program_buffer, buffer);
2023 #ifdef BPF_JITTER
2024 			fcode->func = NULL;
2025 #endif
2026 		}
2027 		d->bd_wfilter = filter;
2028 	} else {
2029 		if (d->bd_rfilter != NULL) {
2030 			fcode = __containerof((void *)d->bd_rfilter,
2031 			    struct bpf_program_buffer, buffer);
2032 #ifdef BPF_JITTER
2033 			fcode->func = d->bd_bfilter;
2034 #endif
2035 		}
2036 		d->bd_rfilter = filter;
2037 #ifdef BPF_JITTER
2038 		d->bd_bfilter = jfunc;
2039 #endif
2040 		if (cmd == BIOCSETF)
2041 			reset_d(d);
2042 
2043 		if (bpf_check_upgrade(cmd, d, filter, flen) != 0) {
2044 			/*
2045 			 * Filter can be set several times without
2046 			 * specifying interface. In this case just mark d
2047 			 * as reader.
2048 			 */
2049 			d->bd_writer = 0;
2050 			if (d->bd_bif != NULL) {
2051 				/*
2052 				 * Remove descriptor from writers-only list
2053 				 * and add it to active readers list.
2054 				 */
2055 				CK_LIST_REMOVE(d, bd_next);
2056 				CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist,
2057 				    d, bd_next);
2058 				CTR2(KTR_NET,
2059 				    "%s: upgrade required by pid %d",
2060 				    __func__, d->bd_pid);
2061 				track_event = true;
2062 			}
2063 		}
2064 	}
2065 	BPFD_UNLOCK(d);
2066 
2067 	if (fcode != NULL)
2068 		NET_EPOCH_CALL(bpf_program_buffer_free, &fcode->epoch_ctx);
2069 
2070 	if (track_event)
2071 		EVENTHANDLER_INVOKE(bpf_track,
2072 		    d->bd_bif->bif_ifp, d->bd_bif->bif_dlt, 1);
2073 
2074 	BPF_UNLOCK();
2075 	return (0);
2076 }
2077 
2078 /*
2079  * Detach a file from its current interface (if attached at all) and attach
2080  * to the interface indicated by the name stored in ifr.
2081  * Return an errno or 0.
2082  */
2083 static int
2084 bpf_setif(struct bpf_d *d, struct ifreq *ifr)
2085 {
2086 	struct bpf_if *bp;
2087 	struct ifnet *theywant;
2088 
2089 	BPF_LOCK_ASSERT();
2090 
2091 	theywant = ifunit(ifr->ifr_name);
2092 	if (theywant == NULL || theywant->if_bpf == NULL)
2093 		return (ENXIO);
2094 
2095 	bp = theywant->if_bpf;
2096 	/*
2097 	 * At this point, we expect the buffer is already allocated.  If not,
2098 	 * return an error.
2099 	 */
2100 	switch (d->bd_bufmode) {
2101 	case BPF_BUFMODE_BUFFER:
2102 	case BPF_BUFMODE_ZBUF:
2103 		if (d->bd_sbuf == NULL)
2104 			return (EINVAL);
2105 		break;
2106 
2107 	default:
2108 		panic("bpf_setif: bufmode %d", d->bd_bufmode);
2109 	}
2110 	if (bp != d->bd_bif)
2111 		bpf_attachd(d, bp);
2112 	else {
2113 		BPFD_LOCK(d);
2114 		reset_d(d);
2115 		BPFD_UNLOCK(d);
2116 	}
2117 	return (0);
2118 }
2119 
2120 /*
2121  * Support for select() and poll() system calls
2122  *
2123  * Return true iff the specific operation will not block indefinitely.
2124  * Otherwise, return false but make a note that a selwakeup() must be done.
2125  */
2126 static int
2127 bpfpoll(struct cdev *dev, int events, struct thread *td)
2128 {
2129 	struct bpf_d *d;
2130 	int revents;
2131 
2132 	if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL)
2133 		return (events &
2134 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
2135 
2136 	/*
2137 	 * Refresh PID associated with this descriptor.
2138 	 */
2139 	revents = events & (POLLOUT | POLLWRNORM);
2140 	BPFD_LOCK(d);
2141 	BPF_PID_REFRESH(d, td);
2142 	if (events & (POLLIN | POLLRDNORM)) {
2143 		if (bpf_ready(d))
2144 			revents |= events & (POLLIN | POLLRDNORM);
2145 		else {
2146 			selrecord(td, &d->bd_sel);
2147 			/* Start the read timeout if necessary. */
2148 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
2149 				callout_reset(&d->bd_callout, d->bd_rtout,
2150 				    bpf_timed_out, d);
2151 				d->bd_state = BPF_WAITING;
2152 			}
2153 		}
2154 	}
2155 	BPFD_UNLOCK(d);
2156 	return (revents);
2157 }
2158 
2159 /*
2160  * Support for kevent() system call.  Register EVFILT_READ filters and
2161  * reject all others.
2162  */
2163 int
2164 bpfkqfilter(struct cdev *dev, struct knote *kn)
2165 {
2166 	struct bpf_d *d;
2167 
2168 	if (devfs_get_cdevpriv((void **)&d) != 0)
2169 		return (1);
2170 
2171 	switch (kn->kn_filter) {
2172 	case EVFILT_READ:
2173 		kn->kn_fop = &bpfread_filtops;
2174 		break;
2175 
2176 	case EVFILT_WRITE:
2177 		kn->kn_fop = &bpfwrite_filtops;
2178 		break;
2179 
2180 	default:
2181 		return (1);
2182 	}
2183 
2184 	/*
2185 	 * Refresh PID associated with this descriptor.
2186 	 */
2187 	BPFD_LOCK(d);
2188 	BPF_PID_REFRESH_CUR(d);
2189 	kn->kn_hook = d;
2190 	knlist_add(&d->bd_sel.si_note, kn, 1);
2191 	BPFD_UNLOCK(d);
2192 
2193 	return (0);
2194 }
2195 
2196 static void
2197 filt_bpfdetach(struct knote *kn)
2198 {
2199 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2200 
2201 	knlist_remove(&d->bd_sel.si_note, kn, 0);
2202 }
2203 
2204 static int
2205 filt_bpfread(struct knote *kn, long hint)
2206 {
2207 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2208 	int ready;
2209 
2210 	BPFD_LOCK_ASSERT(d);
2211 	ready = bpf_ready(d);
2212 	if (ready) {
2213 		kn->kn_data = d->bd_slen;
2214 		/*
2215 		 * Ignore the hold buffer if it is being copied to user space.
2216 		 */
2217 		if (!d->bd_hbuf_in_use && d->bd_hbuf)
2218 			kn->kn_data += d->bd_hlen;
2219 	} else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
2220 		callout_reset(&d->bd_callout, d->bd_rtout,
2221 		    bpf_timed_out, d);
2222 		d->bd_state = BPF_WAITING;
2223 	}
2224 
2225 	return (ready);
2226 }
2227 
2228 static int
2229 filt_bpfwrite(struct knote *kn, long hint)
2230 {
2231 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2232 	BPFD_LOCK_ASSERT(d);
2233 
2234 	kn->kn_data = d->bd_bif->bif_ifp->if_mtu;
2235 
2236 	return (1);
2237 }
2238 
2239 #define	BPF_TSTAMP_NONE		0
2240 #define	BPF_TSTAMP_FAST		1
2241 #define	BPF_TSTAMP_NORMAL	2
2242 #define	BPF_TSTAMP_EXTERN	3
2243 
2244 static int
2245 bpf_ts_quality(int tstype)
2246 {
2247 
2248 	if (tstype == BPF_T_NONE)
2249 		return (BPF_TSTAMP_NONE);
2250 	if ((tstype & BPF_T_FAST) != 0)
2251 		return (BPF_TSTAMP_FAST);
2252 
2253 	return (BPF_TSTAMP_NORMAL);
2254 }
2255 
2256 static int
2257 bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
2258 {
2259 	struct m_tag *tag;
2260 	int quality;
2261 
2262 	quality = bpf_ts_quality(tstype);
2263 	if (quality == BPF_TSTAMP_NONE)
2264 		return (quality);
2265 
2266 	if (m != NULL) {
2267 		tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL);
2268 		if (tag != NULL) {
2269 			*bt = *(struct bintime *)(tag + 1);
2270 			return (BPF_TSTAMP_EXTERN);
2271 		}
2272 	}
2273 	if (quality == BPF_TSTAMP_NORMAL)
2274 		binuptime(bt);
2275 	else
2276 		getbinuptime(bt);
2277 
2278 	return (quality);
2279 }
2280 
2281 /*
2282  * Incoming linkage from device drivers.  Process the packet pkt, of length
2283  * pktlen, which is stored in a contiguous buffer.  The packet is parsed
2284  * by each process' filter, and if accepted, stashed into the corresponding
2285  * buffer.
2286  */
2287 void
2288 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2289 {
2290 	struct epoch_tracker et;
2291 	struct bintime bt;
2292 	struct bpf_d *d;
2293 #ifdef BPF_JITTER
2294 	bpf_jit_filter *bf;
2295 #endif
2296 	u_int slen;
2297 	int gottime;
2298 
2299 	gottime = BPF_TSTAMP_NONE;
2300 	NET_EPOCH_ENTER(et);
2301 	CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2302 		counter_u64_add(d->bd_rcount, 1);
2303 		/*
2304 		 * NB: We dont call BPF_CHECK_DIRECTION() here since there
2305 		 * is no way for the caller to indiciate to us whether this
2306 		 * packet is inbound or outbound. In the bpf_mtap() routines,
2307 		 * we use the interface pointers on the mbuf to figure it out.
2308 		 */
2309 #ifdef BPF_JITTER
2310 		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
2311 		if (bf != NULL)
2312 			slen = (*(bf->func))(pkt, pktlen, pktlen);
2313 		else
2314 #endif
2315 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
2316 		if (slen != 0) {
2317 			/*
2318 			 * Filter matches. Let's to acquire write lock.
2319 			 */
2320 			BPFD_LOCK(d);
2321 			counter_u64_add(d->bd_fcount, 1);
2322 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2323 				gottime = bpf_gettime(&bt, d->bd_tstamp,
2324 				    NULL);
2325 #ifdef MAC
2326 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2327 #endif
2328 				catchpacket(d, pkt, pktlen, slen,
2329 				    bpf_append_bytes, &bt);
2330 			BPFD_UNLOCK(d);
2331 		}
2332 	}
2333 	NET_EPOCH_EXIT(et);
2334 }
2335 
2336 #define	BPF_CHECK_DIRECTION(d, r, i)				\
2337 	    (((d)->bd_direction == BPF_D_IN && (r) != (i)) ||	\
2338 	    ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
2339 
2340 /*
2341  * Incoming linkage from device drivers, when packet is in an mbuf chain.
2342  * Locking model is explained in bpf_tap().
2343  */
2344 void
2345 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2346 {
2347 	struct epoch_tracker et;
2348 	struct bintime bt;
2349 	struct bpf_d *d;
2350 #ifdef BPF_JITTER
2351 	bpf_jit_filter *bf;
2352 #endif
2353 	u_int pktlen, slen;
2354 	int gottime;
2355 
2356 	/* Skip outgoing duplicate packets. */
2357 	if ((m->m_flags & M_PROMISC) != 0 && m_rcvif(m) == NULL) {
2358 		m->m_flags &= ~M_PROMISC;
2359 		return;
2360 	}
2361 
2362 	pktlen = m_length(m, NULL);
2363 	gottime = BPF_TSTAMP_NONE;
2364 
2365 	NET_EPOCH_ENTER(et);
2366 	CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2367 		if (BPF_CHECK_DIRECTION(d, m_rcvif(m), bp->bif_ifp))
2368 			continue;
2369 		counter_u64_add(d->bd_rcount, 1);
2370 #ifdef BPF_JITTER
2371 		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
2372 		/* XXX We cannot handle multiple mbufs. */
2373 		if (bf != NULL && m->m_next == NULL)
2374 			slen = (*(bf->func))(mtod(m, u_char *), pktlen,
2375 			    pktlen);
2376 		else
2377 #endif
2378 		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
2379 		if (slen != 0) {
2380 			BPFD_LOCK(d);
2381 
2382 			counter_u64_add(d->bd_fcount, 1);
2383 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2384 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
2385 #ifdef MAC
2386 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2387 #endif
2388 				catchpacket(d, (u_char *)m, pktlen, slen,
2389 				    bpf_append_mbuf, &bt);
2390 			BPFD_UNLOCK(d);
2391 		}
2392 	}
2393 	NET_EPOCH_EXIT(et);
2394 }
2395 
2396 /*
2397  * Incoming linkage from device drivers, when packet is in
2398  * an mbuf chain and to be prepended by a contiguous header.
2399  */
2400 void
2401 bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
2402 {
2403 	struct epoch_tracker et;
2404 	struct bintime bt;
2405 	struct mbuf mb;
2406 	struct bpf_d *d;
2407 	u_int pktlen, slen;
2408 	int gottime;
2409 
2410 	/* Skip outgoing duplicate packets. */
2411 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
2412 		m->m_flags &= ~M_PROMISC;
2413 		return;
2414 	}
2415 
2416 	pktlen = m_length(m, NULL);
2417 	/*
2418 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
2419 	 * Note that we cut corners here; we only setup what's
2420 	 * absolutely needed--this mbuf should never go anywhere else.
2421 	 */
2422 	mb.m_flags = 0;
2423 	mb.m_next = m;
2424 	mb.m_data = data;
2425 	mb.m_len = dlen;
2426 	pktlen += dlen;
2427 
2428 	gottime = BPF_TSTAMP_NONE;
2429 
2430 	NET_EPOCH_ENTER(et);
2431 	CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2432 		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
2433 			continue;
2434 		counter_u64_add(d->bd_rcount, 1);
2435 		slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
2436 		if (slen != 0) {
2437 			BPFD_LOCK(d);
2438 
2439 			counter_u64_add(d->bd_fcount, 1);
2440 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2441 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
2442 #ifdef MAC
2443 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2444 #endif
2445 				catchpacket(d, (u_char *)&mb, pktlen, slen,
2446 				    bpf_append_mbuf, &bt);
2447 			BPFD_UNLOCK(d);
2448 		}
2449 	}
2450 	NET_EPOCH_EXIT(et);
2451 }
2452 
2453 #undef	BPF_CHECK_DIRECTION
2454 #undef	BPF_TSTAMP_NONE
2455 #undef	BPF_TSTAMP_FAST
2456 #undef	BPF_TSTAMP_NORMAL
2457 #undef	BPF_TSTAMP_EXTERN
2458 
2459 static int
2460 bpf_hdrlen(struct bpf_d *d)
2461 {
2462 	int hdrlen;
2463 
2464 	hdrlen = d->bd_bif->bif_hdrlen;
2465 #ifndef BURN_BRIDGES
2466 	if (d->bd_tstamp == BPF_T_NONE ||
2467 	    BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME)
2468 #ifdef COMPAT_FREEBSD32
2469 		if (d->bd_compat32)
2470 			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32);
2471 		else
2472 #endif
2473 			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr);
2474 	else
2475 #endif
2476 		hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr);
2477 #ifdef COMPAT_FREEBSD32
2478 	if (d->bd_compat32)
2479 		hdrlen = BPF_WORDALIGN32(hdrlen);
2480 	else
2481 #endif
2482 		hdrlen = BPF_WORDALIGN(hdrlen);
2483 
2484 	return (hdrlen - d->bd_bif->bif_hdrlen);
2485 }
2486 
2487 static void
2488 bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
2489 {
2490 	struct bintime bt2, boottimebin;
2491 	struct timeval tsm;
2492 	struct timespec tsn;
2493 
2494 	if ((tstype & BPF_T_MONOTONIC) == 0) {
2495 		bt2 = *bt;
2496 		getboottimebin(&boottimebin);
2497 		bintime_add(&bt2, &boottimebin);
2498 		bt = &bt2;
2499 	}
2500 	switch (BPF_T_FORMAT(tstype)) {
2501 	case BPF_T_MICROTIME:
2502 		bintime2timeval(bt, &tsm);
2503 		ts->bt_sec = tsm.tv_sec;
2504 		ts->bt_frac = tsm.tv_usec;
2505 		break;
2506 	case BPF_T_NANOTIME:
2507 		bintime2timespec(bt, &tsn);
2508 		ts->bt_sec = tsn.tv_sec;
2509 		ts->bt_frac = tsn.tv_nsec;
2510 		break;
2511 	case BPF_T_BINTIME:
2512 		ts->bt_sec = bt->sec;
2513 		ts->bt_frac = bt->frac;
2514 		break;
2515 	}
2516 }
2517 
2518 /*
2519  * Move the packet data from interface memory (pkt) into the
2520  * store buffer.  "cpfn" is the routine called to do the actual data
2521  * transfer.  bcopy is passed in to copy contiguous chunks, while
2522  * bpf_append_mbuf is passed in to copy mbuf chains.  In the latter case,
2523  * pkt is really an mbuf.
2524  */
2525 static void
2526 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
2527     void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
2528     struct bintime *bt)
2529 {
2530 	struct bpf_xhdr hdr;
2531 #ifndef BURN_BRIDGES
2532 	struct bpf_hdr hdr_old;
2533 #ifdef COMPAT_FREEBSD32
2534 	struct bpf_hdr32 hdr32_old;
2535 #endif
2536 #endif
2537 	int caplen, curlen, hdrlen, totlen;
2538 	int do_wakeup = 0;
2539 	int do_timestamp;
2540 	int tstype;
2541 
2542 	BPFD_LOCK_ASSERT(d);
2543 	if (d->bd_bif == NULL) {
2544 		/* Descriptor was detached in concurrent thread */
2545 		counter_u64_add(d->bd_dcount, 1);
2546 		return;
2547 	}
2548 
2549 	/*
2550 	 * Detect whether user space has released a buffer back to us, and if
2551 	 * so, move it from being a hold buffer to a free buffer.  This may
2552 	 * not be the best place to do it (for example, we might only want to
2553 	 * run this check if we need the space), but for now it's a reliable
2554 	 * spot to do it.
2555 	 */
2556 	if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) {
2557 		d->bd_fbuf = d->bd_hbuf;
2558 		d->bd_hbuf = NULL;
2559 		d->bd_hlen = 0;
2560 		bpf_buf_reclaimed(d);
2561 	}
2562 
2563 	/*
2564 	 * Figure out how many bytes to move.  If the packet is
2565 	 * greater or equal to the snapshot length, transfer that
2566 	 * much.  Otherwise, transfer the whole packet (unless
2567 	 * we hit the buffer size limit).
2568 	 */
2569 	hdrlen = bpf_hdrlen(d);
2570 	totlen = hdrlen + min(snaplen, pktlen);
2571 	if (totlen > d->bd_bufsize)
2572 		totlen = d->bd_bufsize;
2573 
2574 	/*
2575 	 * Round up the end of the previous packet to the next longword.
2576 	 *
2577 	 * Drop the packet if there's no room and no hope of room
2578 	 * If the packet would overflow the storage buffer or the storage
2579 	 * buffer is considered immutable by the buffer model, try to rotate
2580 	 * the buffer and wakeup pending processes.
2581 	 */
2582 #ifdef COMPAT_FREEBSD32
2583 	if (d->bd_compat32)
2584 		curlen = BPF_WORDALIGN32(d->bd_slen);
2585 	else
2586 #endif
2587 		curlen = BPF_WORDALIGN(d->bd_slen);
2588 	if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) {
2589 		if (d->bd_fbuf == NULL) {
2590 			/*
2591 			 * There's no room in the store buffer, and no
2592 			 * prospect of room, so drop the packet.  Notify the
2593 			 * buffer model.
2594 			 */
2595 			bpf_buffull(d);
2596 			counter_u64_add(d->bd_dcount, 1);
2597 			return;
2598 		}
2599 		KASSERT(!d->bd_hbuf_in_use, ("hold buffer is in use"));
2600 		ROTATE_BUFFERS(d);
2601 		do_wakeup = 1;
2602 		curlen = 0;
2603 	} else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT)
2604 		/*
2605 		 * Immediate mode is set, or the read timeout has already
2606 		 * expired during a select call.  A packet arrived, so the
2607 		 * reader should be woken up.
2608 		 */
2609 		do_wakeup = 1;
2610 	caplen = totlen - hdrlen;
2611 	tstype = d->bd_tstamp;
2612 	do_timestamp = tstype != BPF_T_NONE;
2613 #ifndef BURN_BRIDGES
2614 	if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) {
2615 		struct bpf_ts ts;
2616 		if (do_timestamp)
2617 			bpf_bintime2ts(bt, &ts, tstype);
2618 #ifdef COMPAT_FREEBSD32
2619 		if (d->bd_compat32) {
2620 			bzero(&hdr32_old, sizeof(hdr32_old));
2621 			if (do_timestamp) {
2622 				hdr32_old.bh_tstamp.tv_sec = ts.bt_sec;
2623 				hdr32_old.bh_tstamp.tv_usec = ts.bt_frac;
2624 			}
2625 			hdr32_old.bh_datalen = pktlen;
2626 			hdr32_old.bh_hdrlen = hdrlen;
2627 			hdr32_old.bh_caplen = caplen;
2628 			bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old,
2629 			    sizeof(hdr32_old));
2630 			goto copy;
2631 		}
2632 #endif
2633 		bzero(&hdr_old, sizeof(hdr_old));
2634 		if (do_timestamp) {
2635 			hdr_old.bh_tstamp.tv_sec = ts.bt_sec;
2636 			hdr_old.bh_tstamp.tv_usec = ts.bt_frac;
2637 		}
2638 		hdr_old.bh_datalen = pktlen;
2639 		hdr_old.bh_hdrlen = hdrlen;
2640 		hdr_old.bh_caplen = caplen;
2641 		bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old,
2642 		    sizeof(hdr_old));
2643 		goto copy;
2644 	}
2645 #endif
2646 
2647 	/*
2648 	 * Append the bpf header.  Note we append the actual header size, but
2649 	 * move forward the length of the header plus padding.
2650 	 */
2651 	bzero(&hdr, sizeof(hdr));
2652 	if (do_timestamp)
2653 		bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype);
2654 	hdr.bh_datalen = pktlen;
2655 	hdr.bh_hdrlen = hdrlen;
2656 	hdr.bh_caplen = caplen;
2657 	bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr));
2658 
2659 	/*
2660 	 * Copy the packet data into the store buffer and update its length.
2661 	 */
2662 #ifndef BURN_BRIDGES
2663 copy:
2664 #endif
2665 	(*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen);
2666 	d->bd_slen = curlen + totlen;
2667 
2668 	if (do_wakeup)
2669 		bpf_wakeup(d);
2670 }
2671 
2672 /*
2673  * Free buffers currently in use by a descriptor.
2674  * Called on close.
2675  */
2676 static void
2677 bpfd_free(epoch_context_t ctx)
2678 {
2679 	struct bpf_d *d;
2680 	struct bpf_program_buffer *p;
2681 
2682 	/*
2683 	 * We don't need to lock out interrupts since this descriptor has
2684 	 * been detached from its interface and it yet hasn't been marked
2685 	 * free.
2686 	 */
2687 	d = __containerof(ctx, struct bpf_d, epoch_ctx);
2688 	bpf_free(d);
2689 	if (d->bd_rfilter != NULL) {
2690 		p = __containerof((void *)d->bd_rfilter,
2691 		    struct bpf_program_buffer, buffer);
2692 #ifdef BPF_JITTER
2693 		p->func = d->bd_bfilter;
2694 #endif
2695 		bpf_program_buffer_free(&p->epoch_ctx);
2696 	}
2697 	if (d->bd_wfilter != NULL) {
2698 		p = __containerof((void *)d->bd_wfilter,
2699 		    struct bpf_program_buffer, buffer);
2700 #ifdef BPF_JITTER
2701 		p->func = NULL;
2702 #endif
2703 		bpf_program_buffer_free(&p->epoch_ctx);
2704 	}
2705 
2706 	mtx_destroy(&d->bd_lock);
2707 	counter_u64_free(d->bd_rcount);
2708 	counter_u64_free(d->bd_dcount);
2709 	counter_u64_free(d->bd_fcount);
2710 	counter_u64_free(d->bd_wcount);
2711 	counter_u64_free(d->bd_wfcount);
2712 	counter_u64_free(d->bd_wdcount);
2713 	counter_u64_free(d->bd_zcopy);
2714 	free(d, M_BPF);
2715 }
2716 
2717 /*
2718  * Attach an interface to bpf.  dlt is the link layer type; hdrlen is the
2719  * fixed size of the link header (variable length headers not yet supported).
2720  */
2721 void
2722 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2723 {
2724 
2725 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2726 }
2727 
2728 /*
2729  * Attach an interface to bpf.  ifp is a pointer to the structure
2730  * defining the interface to be attached, dlt is the link layer type,
2731  * and hdrlen is the fixed size of the link header (variable length
2732  * headers are not yet supporrted).
2733  */
2734 void
2735 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen,
2736     struct bpf_if **driverp)
2737 {
2738 	struct bpf_if *bp;
2739 
2740 	KASSERT(*driverp == NULL,
2741 	    ("bpfattach2: driverp already initialized"));
2742 
2743 	bp = malloc(sizeof(*bp), M_BPF, M_WAITOK | M_ZERO);
2744 
2745 	CK_LIST_INIT(&bp->bif_dlist);
2746 	CK_LIST_INIT(&bp->bif_wlist);
2747 	bp->bif_ifp = ifp;
2748 	bp->bif_dlt = dlt;
2749 	bp->bif_hdrlen = hdrlen;
2750 	bp->bif_bpf = driverp;
2751 	bp->bif_refcnt = 1;
2752 	*driverp = bp;
2753 	/*
2754 	 * Reference ifnet pointer, so it won't freed until
2755 	 * we release it.
2756 	 */
2757 	if_ref(ifp);
2758 	BPF_LOCK();
2759 	CK_LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
2760 	BPF_UNLOCK();
2761 
2762 	if (bootverbose && IS_DEFAULT_VNET(curvnet))
2763 		if_printf(ifp, "bpf attached\n");
2764 }
2765 
2766 #ifdef VIMAGE
2767 /*
2768  * When moving interfaces between vnet instances we need a way to
2769  * query the dlt and hdrlen before detach so we can re-attch the if_bpf
2770  * after the vmove.  We unfortunately have no device driver infrastructure
2771  * to query the interface for these values after creation/attach, thus
2772  * add this as a workaround.
2773  */
2774 int
2775 bpf_get_bp_params(struct bpf_if *bp, u_int *bif_dlt, u_int *bif_hdrlen)
2776 {
2777 
2778 	if (bp == NULL)
2779 		return (ENXIO);
2780 	if (bif_dlt == NULL && bif_hdrlen == NULL)
2781 		return (0);
2782 
2783 	if (bif_dlt != NULL)
2784 		*bif_dlt = bp->bif_dlt;
2785 	if (bif_hdrlen != NULL)
2786 		*bif_hdrlen = bp->bif_hdrlen;
2787 
2788 	return (0);
2789 }
2790 #endif
2791 
2792 /*
2793  * Detach bpf from an interface. This involves detaching each descriptor
2794  * associated with the interface. Notify each descriptor as it's detached
2795  * so that any sleepers wake up and get ENXIO.
2796  */
2797 void
2798 bpfdetach(struct ifnet *ifp)
2799 {
2800 	struct bpf_if *bp, *bp_temp;
2801 	struct bpf_d *d;
2802 
2803 	BPF_LOCK();
2804 	/* Find all bpf_if struct's which reference ifp and detach them. */
2805 	CK_LIST_FOREACH_SAFE(bp, &bpf_iflist, bif_next, bp_temp) {
2806 		if (ifp != bp->bif_ifp)
2807 			continue;
2808 
2809 		CK_LIST_REMOVE(bp, bif_next);
2810 		*bp->bif_bpf = (struct bpf_if *)&dead_bpf_if;
2811 
2812 		CTR4(KTR_NET,
2813 		    "%s: sheduling free for encap %d (%p) for if %p",
2814 		    __func__, bp->bif_dlt, bp, ifp);
2815 
2816 		/* Detach common descriptors */
2817 		while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) {
2818 			bpf_detachd_locked(d, true);
2819 		}
2820 
2821 		/* Detach writer-only descriptors */
2822 		while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) {
2823 			bpf_detachd_locked(d, true);
2824 		}
2825 		bpfif_rele(bp);
2826 	}
2827 	BPF_UNLOCK();
2828 }
2829 
2830 /*
2831  * Get a list of available data link type of the interface.
2832  */
2833 static int
2834 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2835 {
2836 	struct ifnet *ifp;
2837 	struct bpf_if *bp;
2838 	u_int *lst;
2839 	int error, n, n1;
2840 
2841 	BPF_LOCK_ASSERT();
2842 
2843 	ifp = d->bd_bif->bif_ifp;
2844 	n1 = 0;
2845 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2846 		if (bp->bif_ifp == ifp)
2847 			n1++;
2848 	}
2849 	if (bfl->bfl_list == NULL) {
2850 		bfl->bfl_len = n1;
2851 		return (0);
2852 	}
2853 	if (n1 > bfl->bfl_len)
2854 		return (ENOMEM);
2855 
2856 	lst = malloc(n1 * sizeof(u_int), M_TEMP, M_WAITOK);
2857 	n = 0;
2858 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2859 		if (bp->bif_ifp != ifp)
2860 			continue;
2861 		lst[n++] = bp->bif_dlt;
2862 	}
2863 	error = copyout(lst, bfl->bfl_list, sizeof(u_int) * n);
2864 	free(lst, M_TEMP);
2865 	bfl->bfl_len = n;
2866 	return (error);
2867 }
2868 
2869 /*
2870  * Set the data link type of a BPF instance.
2871  */
2872 static int
2873 bpf_setdlt(struct bpf_d *d, u_int dlt)
2874 {
2875 	int error, opromisc;
2876 	struct ifnet *ifp;
2877 	struct bpf_if *bp;
2878 
2879 	BPF_LOCK_ASSERT();
2880 	MPASS(d->bd_bif != NULL);
2881 
2882 	/*
2883 	 * It is safe to check bd_bif without BPFD_LOCK, it can not be
2884 	 * changed while we hold global lock.
2885 	 */
2886 	if (d->bd_bif->bif_dlt == dlt)
2887 		return (0);
2888 
2889 	ifp = d->bd_bif->bif_ifp;
2890 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2891 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2892 			break;
2893 	}
2894 	if (bp == NULL)
2895 		return (EINVAL);
2896 
2897 	opromisc = d->bd_promisc;
2898 	bpf_attachd(d, bp);
2899 	if (opromisc) {
2900 		error = ifpromisc(bp->bif_ifp, 1);
2901 		if (error)
2902 			if_printf(bp->bif_ifp, "%s: ifpromisc failed (%d)\n",
2903 			    __func__, error);
2904 		else
2905 			d->bd_promisc = 1;
2906 	}
2907 	return (0);
2908 }
2909 
2910 static void
2911 bpf_drvinit(void *unused)
2912 {
2913 	struct cdev *dev;
2914 
2915 	sx_init(&bpf_sx, "bpf global lock");
2916 	CK_LIST_INIT(&bpf_iflist);
2917 
2918 	dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf");
2919 	/* For compatibility */
2920 	make_dev_alias(dev, "bpf0");
2921 }
2922 
2923 /*
2924  * Zero out the various packet counters associated with all of the bpf
2925  * descriptors.  At some point, we will probably want to get a bit more
2926  * granular and allow the user to specify descriptors to be zeroed.
2927  */
2928 static void
2929 bpf_zero_counters(void)
2930 {
2931 	struct bpf_if *bp;
2932 	struct bpf_d *bd;
2933 
2934 	BPF_LOCK();
2935 	/*
2936 	 * We are protected by global lock here, interfaces and
2937 	 * descriptors can not be deleted while we hold it.
2938 	 */
2939 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2940 		CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2941 			counter_u64_zero(bd->bd_rcount);
2942 			counter_u64_zero(bd->bd_dcount);
2943 			counter_u64_zero(bd->bd_fcount);
2944 			counter_u64_zero(bd->bd_wcount);
2945 			counter_u64_zero(bd->bd_wfcount);
2946 			counter_u64_zero(bd->bd_zcopy);
2947 		}
2948 	}
2949 	BPF_UNLOCK();
2950 }
2951 
2952 /*
2953  * Fill filter statistics
2954  */
2955 static void
2956 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
2957 {
2958 
2959 	BPF_LOCK_ASSERT();
2960 	bzero(d, sizeof(*d));
2961 	d->bd_structsize = sizeof(*d);
2962 	d->bd_immediate = bd->bd_immediate;
2963 	d->bd_promisc = bd->bd_promisc;
2964 	d->bd_hdrcmplt = bd->bd_hdrcmplt;
2965 	d->bd_direction = bd->bd_direction;
2966 	d->bd_feedback = bd->bd_feedback;
2967 	d->bd_async = bd->bd_async;
2968 	d->bd_rcount = counter_u64_fetch(bd->bd_rcount);
2969 	d->bd_dcount = counter_u64_fetch(bd->bd_dcount);
2970 	d->bd_fcount = counter_u64_fetch(bd->bd_fcount);
2971 	d->bd_sig = bd->bd_sig;
2972 	d->bd_slen = bd->bd_slen;
2973 	d->bd_hlen = bd->bd_hlen;
2974 	d->bd_bufsize = bd->bd_bufsize;
2975 	d->bd_pid = bd->bd_pid;
2976 	strlcpy(d->bd_ifname,
2977 	    bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
2978 	d->bd_locked = bd->bd_locked;
2979 	d->bd_wcount = counter_u64_fetch(bd->bd_wcount);
2980 	d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount);
2981 	d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount);
2982 	d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy);
2983 	d->bd_bufmode = bd->bd_bufmode;
2984 }
2985 
2986 /*
2987  * Handle `netstat -B' stats request
2988  */
2989 static int
2990 bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
2991 {
2992 	static const struct xbpf_d zerostats;
2993 	struct xbpf_d *xbdbuf, *xbd, tempstats;
2994 	int index, error;
2995 	struct bpf_if *bp;
2996 	struct bpf_d *bd;
2997 
2998 	/*
2999 	 * XXX This is not technically correct. It is possible for non
3000 	 * privileged users to open bpf devices. It would make sense
3001 	 * if the users who opened the devices were able to retrieve
3002 	 * the statistics for them, too.
3003 	 */
3004 	error = priv_check(req->td, PRIV_NET_BPF);
3005 	if (error)
3006 		return (error);
3007 	/*
3008 	 * Check to see if the user is requesting that the counters be
3009 	 * zeroed out.  Explicitly check that the supplied data is zeroed,
3010 	 * as we aren't allowing the user to set the counters currently.
3011 	 */
3012 	if (req->newptr != NULL) {
3013 		if (req->newlen != sizeof(tempstats))
3014 			return (EINVAL);
3015 		memset(&tempstats, 0, sizeof(tempstats));
3016 		error = SYSCTL_IN(req, &tempstats, sizeof(tempstats));
3017 		if (error)
3018 			return (error);
3019 		if (bcmp(&tempstats, &zerostats, sizeof(tempstats)) != 0)
3020 			return (EINVAL);
3021 		bpf_zero_counters();
3022 		return (0);
3023 	}
3024 	if (req->oldptr == NULL)
3025 		return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
3026 	if (bpf_bpfd_cnt == 0)
3027 		return (SYSCTL_OUT(req, 0, 0));
3028 	xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
3029 	BPF_LOCK();
3030 	if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
3031 		BPF_UNLOCK();
3032 		free(xbdbuf, M_BPF);
3033 		return (ENOMEM);
3034 	}
3035 	index = 0;
3036 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
3037 		/* Send writers-only first */
3038 		CK_LIST_FOREACH(bd, &bp->bif_wlist, bd_next) {
3039 			xbd = &xbdbuf[index++];
3040 			bpfstats_fill_xbpf(xbd, bd);
3041 		}
3042 		CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
3043 			xbd = &xbdbuf[index++];
3044 			bpfstats_fill_xbpf(xbd, bd);
3045 		}
3046 	}
3047 	BPF_UNLOCK();
3048 	error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
3049 	free(xbdbuf, M_BPF);
3050 	return (error);
3051 }
3052 
3053 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL);
3054 
3055 #else /* !DEV_BPF && !NETGRAPH_BPF */
3056 
3057 /*
3058  * NOP stubs to allow bpf-using drivers to load and function.
3059  *
3060  * A 'better' implementation would allow the core bpf functionality
3061  * to be loaded at runtime.
3062  */
3063 
3064 void
3065 bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
3066 {
3067 }
3068 
3069 void
3070 bpf_mtap(struct bpf_if *bp, struct mbuf *m)
3071 {
3072 }
3073 
3074 void
3075 bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
3076 {
3077 }
3078 
3079 void
3080 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
3081 {
3082 
3083 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
3084 }
3085 
3086 void
3087 bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
3088 {
3089 
3090 	*driverp = (struct bpf_if *)&dead_bpf_if;
3091 }
3092 
3093 void
3094 bpfdetach(struct ifnet *ifp)
3095 {
3096 }
3097 
3098 u_int
3099 bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
3100 {
3101 	return -1;	/* "no filter" behaviour */
3102 }
3103 
3104 int
3105 bpf_validate(const struct bpf_insn *f, int len)
3106 {
3107 	return 0;		/* false */
3108 }
3109 
3110 #endif /* !DEV_BPF && !NETGRAPH_BPF */
3111 
3112 #ifdef DDB
3113 static void
3114 bpf_show_bpf_if(struct bpf_if *bpf_if)
3115 {
3116 
3117 	if (bpf_if == NULL)
3118 		return;
3119 	db_printf("%p:\n", bpf_if);
3120 #define	BPF_DB_PRINTF(f, e)	db_printf("   %s = " f "\n", #e, bpf_if->e);
3121 	/* bif_ext.bif_next */
3122 	/* bif_ext.bif_dlist */
3123 	BPF_DB_PRINTF("%#x", bif_dlt);
3124 	BPF_DB_PRINTF("%u", bif_hdrlen);
3125 	/* bif_wlist */
3126 	BPF_DB_PRINTF("%p", bif_ifp);
3127 	BPF_DB_PRINTF("%p", bif_bpf);
3128 	BPF_DB_PRINTF("%u", bif_refcnt);
3129 }
3130 
3131 DB_SHOW_COMMAND(bpf_if, db_show_bpf_if)
3132 {
3133 
3134 	if (!have_addr) {
3135 		db_printf("usage: show bpf_if <struct bpf_if *>\n");
3136 		return;
3137 	}
3138 
3139 	bpf_show_bpf_if((struct bpf_if *)addr);
3140 }
3141 #endif
3142