xref: /freebsd/sys/net/bpf.c (revision 1baf6164e4d671ca1547a3096d8ce8fe69ed4e5c)
1c398230bSWarner Losh /*-
251369649SPedro F. Giffuni  * SPDX-License-Identifier: BSD-3-Clause
351369649SPedro F. Giffuni  *
4df8bae1dSRodney W. Grimes  * Copyright (c) 1990, 1991, 1993
5253a3814SLawrence Stewart  *	The Regents of the University of California.  All rights reserved.
6699281b5SAndrey V. Elsukov  * Copyright (c) 2019 Andrey V. Elsukov <ae@FreeBSD.org>
7df8bae1dSRodney W. Grimes  *
8df8bae1dSRodney W. Grimes  * This code is derived from the Stanford/CMU enet packet filter,
9df8bae1dSRodney W. Grimes  * (net/enet.c) distributed as part of 4.3BSD, and code contributed
10df8bae1dSRodney W. Grimes  * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
11df8bae1dSRodney W. Grimes  * Berkeley Laboratory.
12df8bae1dSRodney W. Grimes  *
13df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
14df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
15df8bae1dSRodney W. Grimes  * are met:
16df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
17df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
18df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
19df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
20df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
21fbbd9655SWarner Losh  * 3. Neither the name of the University nor the names of its contributors
22df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
23df8bae1dSRodney W. Grimes  *    without specific prior written permission.
24df8bae1dSRodney W. Grimes  *
25df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
36df8bae1dSRodney W. Grimes  */
37df8bae1dSRodney W. Grimes 
38c7866007SRobert Watson #include <sys/cdefs.h>
395bb5f2c9SPeter Wemm #include "opt_bpf.h"
4005fc4164SBjoern A. Zeeb #include "opt_ddb.h"
415bb5f2c9SPeter Wemm #include "opt_netgraph.h"
42df8bae1dSRodney W. Grimes 
43df8bae1dSRodney W. Grimes #include <sys/param.h>
44ce7609a4SBruce Evans #include <sys/conf.h>
45e2e050c8SConrad Meyer #include <sys/eventhandler.h>
46e76eee55SPoul-Henning Kamp #include <sys/fcntl.h>
47ebd8672cSBjoern A. Zeeb #include <sys/jail.h>
48e2e050c8SConrad Meyer #include <sys/ktr.h>
49e2e050c8SConrad Meyer #include <sys/lock.h>
504d1d4912SBruce Evans #include <sys/malloc.h>
51df8bae1dSRodney W. Grimes #include <sys/mbuf.h>
52e2e050c8SConrad Meyer #include <sys/mutex.h>
53df8bae1dSRodney W. Grimes #include <sys/time.h>
54acd3428bSRobert Watson #include <sys/priv.h>
55df8bae1dSRodney W. Grimes #include <sys/proc.h>
560310c19fSBruce Evans #include <sys/signalvar.h>
57528f627fSBruce Evans #include <sys/filio.h>
58528f627fSBruce Evans #include <sys/sockio.h>
59528f627fSBruce Evans #include <sys/ttycom.h>
60e76eee55SPoul-Henning Kamp #include <sys/uio.h>
6174549d4bSWojciech Macek #include <sys/sysent.h>
62e2e050c8SConrad Meyer #include <sys/systm.h>
63df8bae1dSRodney W. Grimes 
6495aab9ccSJohn-Mark Gurney #include <sys/event.h>
6595aab9ccSJohn-Mark Gurney #include <sys/file.h>
66243ac7d8SPeter Wemm #include <sys/poll.h>
6795aab9ccSJohn-Mark Gurney #include <sys/proc.h>
68df8bae1dSRodney W. Grimes 
69df8bae1dSRodney W. Grimes #include <sys/socket.h>
70df8bae1dSRodney W. Grimes 
7105fc4164SBjoern A. Zeeb #ifdef DDB
7205fc4164SBjoern A. Zeeb #include <ddb/ddb.h>
7305fc4164SBjoern A. Zeeb #endif
7405fc4164SBjoern A. Zeeb 
75fba9235dSBruce Evans #include <net/if.h>
7676039bc8SGleb Smirnoff #include <net/if_var.h>
772c2b37adSJustin Hibbits #include <net/if_private.h>
781e7fe2fbSLuiz Otavio O Souza #include <net/if_vlan_var.h>
794fb3a820SAlexander V. Chernikov #include <net/if_dl.h>
80df8bae1dSRodney W. Grimes #include <net/bpf.h>
814d621040SChristian S.J. Peron #include <net/bpf_buffer.h>
82ae275efcSJung-uk Kim #ifdef BPF_JITTER
83ae275efcSJung-uk Kim #include <net/bpf_jitter.h>
84ae275efcSJung-uk Kim #endif
854d621040SChristian S.J. Peron #include <net/bpf_zerocopy.h>
86df8bae1dSRodney W. Grimes #include <net/bpfdesc.h>
874fb3a820SAlexander V. Chernikov #include <net/route.h>
88530c0060SRobert Watson #include <net/vnet.h>
89df8bae1dSRodney W. Grimes 
90df8bae1dSRodney W. Grimes #include <netinet/in.h>
91df8bae1dSRodney W. Grimes #include <netinet/if_ether.h>
92df8bae1dSRodney W. Grimes #include <sys/kernel.h>
93f708ef1bSPoul-Henning Kamp #include <sys/sysctl.h>
947b778b5eSEivind Eklund 
95246b5467SSam Leffler #include <net80211/ieee80211_freebsd.h>
96246b5467SSam Leffler 
97aed55708SRobert Watson #include <security/mac/mac_framework.h>
98aed55708SRobert Watson 
994d621040SChristian S.J. Peron MALLOC_DEFINE(M_BPF, "BPF", "BPF data");
10087f6c662SJulian Elischer 
1017a974a64SZhenlei Huang static const struct bpf_if_ext dead_bpf_if = {
102699281b5SAndrey V. Elsukov 	.bif_dlist = CK_LIST_HEAD_INITIALIZER()
1032b9600b4SAndrey V. Elsukov };
1042b9600b4SAndrey V. Elsukov 
105b23cbbe6SMark Johnston struct bpf_if {
106b23cbbe6SMark Johnston #define	bif_next	bif_ext.bif_next
107b23cbbe6SMark Johnston #define	bif_dlist	bif_ext.bif_dlist
108b23cbbe6SMark Johnston 	struct bpf_if_ext bif_ext;	/* public members */
109b23cbbe6SMark Johnston 	u_int		bif_dlt;	/* link layer type */
110b23cbbe6SMark Johnston 	u_int		bif_hdrlen;	/* length of link header */
111699281b5SAndrey V. Elsukov 	struct bpfd_list bif_wlist;	/* writer-only list */
112b23cbbe6SMark Johnston 	struct ifnet	*bif_ifp;	/* corresponding interface */
1139ce40d32SKristof Provost 	struct bpf_if	**bif_bpf;	/* Pointer to pointer to us */
114699281b5SAndrey V. Elsukov 	volatile u_int	bif_refcnt;
115699281b5SAndrey V. Elsukov 	struct epoch_context epoch_ctx;
116b23cbbe6SMark Johnston };
117b23cbbe6SMark Johnston 
118b23cbbe6SMark Johnston CTASSERT(offsetof(struct bpf_if, bif_ext) == 0);
119b23cbbe6SMark Johnston 
120699281b5SAndrey V. Elsukov struct bpf_program_buffer {
121699281b5SAndrey V. Elsukov 	struct epoch_context	epoch_ctx;
122699281b5SAndrey V. Elsukov #ifdef BPF_JITTER
123699281b5SAndrey V. Elsukov 	bpf_jit_filter		*func;
124699281b5SAndrey V. Elsukov #endif
125699281b5SAndrey V. Elsukov 	void			*buffer[0];
126699281b5SAndrey V. Elsukov };
127ca3cd72bSAndrey V. Elsukov 
1285bb5f2c9SPeter Wemm #if defined(DEV_BPF) || defined(NETGRAPH_BPF)
12953ac6efbSJulian Elischer 
130df8bae1dSRodney W. Grimes #define PRINET  26			/* interruptible */
1311e7fe2fbSLuiz Otavio O Souza #define BPF_PRIO_MAX	7
132df8bae1dSRodney W. Grimes 
133547d94bdSJung-uk Kim #define	SIZEOF_BPF_HDR(type)	\
134547d94bdSJung-uk Kim     (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
135547d94bdSJung-uk Kim 
136fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
137fc0a61a4SKonstantin Belousov #include <sys/mount.h>
138fc0a61a4SKonstantin Belousov #include <compat/freebsd32/freebsd32.h>
139fc0a61a4SKonstantin Belousov #define BPF_ALIGNMENT32 sizeof(int32_t)
140d9c9c81cSPedro F. Giffuni #define	BPF_WORDALIGN32(x) roundup2(x, BPF_ALIGNMENT32)
141fc0a61a4SKonstantin Belousov 
142547d94bdSJung-uk Kim #ifndef BURN_BRIDGES
143fc0a61a4SKonstantin Belousov /*
144fc0a61a4SKonstantin Belousov  * 32-bit version of structure prepended to each packet.  We use this header
145fc0a61a4SKonstantin Belousov  * instead of the standard one for 32-bit streams.  We mark the a stream as
146fc0a61a4SKonstantin Belousov  * 32-bit the first time we see a 32-bit compat ioctl request.
147fc0a61a4SKonstantin Belousov  */
148fc0a61a4SKonstantin Belousov struct bpf_hdr32 {
149fc0a61a4SKonstantin Belousov 	struct timeval32 bh_tstamp;	/* time stamp */
150fc0a61a4SKonstantin Belousov 	uint32_t	bh_caplen;	/* length of captured portion */
151fc0a61a4SKonstantin Belousov 	uint32_t	bh_datalen;	/* original length of packet */
152fc0a61a4SKonstantin Belousov 	uint16_t	bh_hdrlen;	/* length of bpf header (this struct
153fc0a61a4SKonstantin Belousov 					   plus alignment padding) */
154fc0a61a4SKonstantin Belousov };
155253a3814SLawrence Stewart #endif
156fc0a61a4SKonstantin Belousov 
157fc0a61a4SKonstantin Belousov struct bpf_program32 {
158fc0a61a4SKonstantin Belousov 	u_int bf_len;
159fc0a61a4SKonstantin Belousov 	uint32_t bf_insns;
160fc0a61a4SKonstantin Belousov };
161fc0a61a4SKonstantin Belousov 
162fc0a61a4SKonstantin Belousov struct bpf_dltlist32 {
163fc0a61a4SKonstantin Belousov 	u_int	bfl_len;
164fc0a61a4SKonstantin Belousov 	u_int	bfl_list;
165fc0a61a4SKonstantin Belousov };
166fc0a61a4SKonstantin Belousov 
167fc0a61a4SKonstantin Belousov #define	BIOCSETF32	_IOW('B', 103, struct bpf_program32)
168fc0a61a4SKonstantin Belousov #define	BIOCSRTIMEOUT32	_IOW('B', 109, struct timeval32)
169fc0a61a4SKonstantin Belousov #define	BIOCGRTIMEOUT32	_IOR('B', 110, struct timeval32)
170fc0a61a4SKonstantin Belousov #define	BIOCGDLTLIST32	_IOWR('B', 121, struct bpf_dltlist32)
171fc0a61a4SKonstantin Belousov #define	BIOCSETWF32	_IOW('B', 123, struct bpf_program32)
172fc0a61a4SKonstantin Belousov #define	BIOCSETFNR32	_IOW('B', 130, struct bpf_program32)
173253a3814SLawrence Stewart #endif
174fc0a61a4SKonstantin Belousov 
175f422673eSStephen Hurd #define BPF_LOCK()		sx_xlock(&bpf_sx)
176f422673eSStephen Hurd #define BPF_UNLOCK()		sx_xunlock(&bpf_sx)
177f422673eSStephen Hurd #define BPF_LOCK_ASSERT()	sx_assert(&bpf_sx, SA_XLOCKED)
178df8bae1dSRodney W. Grimes /*
179d1a67300SRobert Watson  * bpf_iflist is a list of BPF interface structures, each corresponding to a
180d1a67300SRobert Watson  * specific DLT. The same network interface might have several BPF interface
181d1a67300SRobert Watson  * structures registered by different layers in the stack (i.e., 802.11
182d1a67300SRobert Watson  * frames, ethernet frames, etc).
183df8bae1dSRodney W. Grimes  */
184699281b5SAndrey V. Elsukov CK_LIST_HEAD(bpf_iflist, bpf_if);
185699281b5SAndrey V. Elsukov static struct bpf_iflist bpf_iflist;
186f422673eSStephen Hurd static struct sx	bpf_sx;		/* bpf global lock */
18769f7644bSChristian S.J. Peron static int		bpf_bpfd_cnt;
188df8bae1dSRodney W. Grimes 
189699281b5SAndrey V. Elsukov static void	bpfif_ref(struct bpf_if *);
190699281b5SAndrey V. Elsukov static void	bpfif_rele(struct bpf_if *);
191699281b5SAndrey V. Elsukov 
192699281b5SAndrey V. Elsukov static void	bpfd_ref(struct bpf_d *);
193699281b5SAndrey V. Elsukov static void	bpfd_rele(struct bpf_d *);
19419ba8395SChristian S.J. Peron static void	bpf_attachd(struct bpf_d *, struct bpf_if *);
19519ba8395SChristian S.J. Peron static void	bpf_detachd(struct bpf_d *);
196699281b5SAndrey V. Elsukov static void	bpf_detachd_locked(struct bpf_d *, bool);
197699281b5SAndrey V. Elsukov static void	bpfd_free(epoch_context_t);
198cb44b6dfSAndrew Thompson static int	bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **,
1994fb3a820SAlexander V. Chernikov 		    struct sockaddr *, int *, struct bpf_d *);
200929ddbbbSAlfred Perlstein static int	bpf_setif(struct bpf_d *, struct ifreq *);
201929ddbbbSAlfred Perlstein static void	bpf_timed_out(void *);
202e7bb21b3SJonathan Lemon static __inline void
203929ddbbbSAlfred Perlstein 		bpf_wakeup(struct bpf_d *);
2044d621040SChristian S.J. Peron static void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
2054d621040SChristian S.J. Peron 		    void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
206547d94bdSJung-uk Kim 		    struct bintime *);
207929ddbbbSAlfred Perlstein static void	reset_d(struct bpf_d *);
20893e39f0bSChristian S.J. Peron static int	bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
2098eab61f3SSam Leffler static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
2108eab61f3SSam Leffler static int	bpf_setdlt(struct bpf_d *, u_int);
21195aab9ccSJohn-Mark Gurney static void	filt_bpfdetach(struct knote *);
21295aab9ccSJohn-Mark Gurney static int	filt_bpfread(struct knote *, long);
213ded77e02SHartmut Brandt static int	filt_bpfwrite(struct knote *, long);
214a3272e3cSChristian S.J. Peron static void	bpf_drvinit(void *);
21569f7644bSChristian S.J. Peron static int	bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
21669f7644bSChristian S.J. Peron 
2177029da5cSPawel Biernacki SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
2187029da5cSPawel Biernacki     "bpf sysctl");
21912dc9582SJung-uk Kim int bpf_maxinsns = BPF_MAXINSNS;
22069f7644bSChristian S.J. Peron SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
22169f7644bSChristian S.J. Peron     &bpf_maxinsns, 0, "Maximum bpf program instructions");
222ffeeb924SChristian S.J. Peron static int bpf_zerocopy_enable = 0;
2234d621040SChristian S.J. Peron SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
2244d621040SChristian S.J. Peron     &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
2256472ac3dSEd Schouten static SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
22669f7644bSChristian S.J. Peron     bpf_stats_sysctl, "bpf statistics portal");
227df8bae1dSRodney W. Grimes 
2285f901c92SAndrew Turner VNET_DEFINE_STATIC(int, bpf_optimize_writers) = 0;
22951ec1eb7SAlexander V. Chernikov #define	V_bpf_optimize_writers VNET(bpf_optimize_writers)
230c774294cSTycho Nightingale SYSCTL_INT(_net_bpf, OID_AUTO, optimize_writers, CTLFLAG_VNET | CTLFLAG_RWTUN,
2316df8a710SGleb Smirnoff     &VNET_NAME(bpf_optimize_writers), 0,
23251ec1eb7SAlexander V. Chernikov     "Do not send packets until BPF program is set");
23351ec1eb7SAlexander V. Chernikov 
23487f6c662SJulian Elischer static	d_open_t	bpfopen;
23587f6c662SJulian Elischer static	d_read_t	bpfread;
23687f6c662SJulian Elischer static	d_write_t	bpfwrite;
23787f6c662SJulian Elischer static	d_ioctl_t	bpfioctl;
238243ac7d8SPeter Wemm static	d_poll_t	bpfpoll;
23995aab9ccSJohn-Mark Gurney static	d_kqfilter_t	bpfkqfilter;
24087f6c662SJulian Elischer 
2414e2f199eSPoul-Henning Kamp static struct cdevsw bpf_cdevsw = {
242dc08ffecSPoul-Henning Kamp 	.d_version =	D_VERSION,
2437ac40f5fSPoul-Henning Kamp 	.d_open =	bpfopen,
2447ac40f5fSPoul-Henning Kamp 	.d_read =	bpfread,
2457ac40f5fSPoul-Henning Kamp 	.d_write =	bpfwrite,
2467ac40f5fSPoul-Henning Kamp 	.d_ioctl =	bpfioctl,
2477ac40f5fSPoul-Henning Kamp 	.d_poll =	bpfpoll,
2487ac40f5fSPoul-Henning Kamp 	.d_name =	"bpf",
24995aab9ccSJohn-Mark Gurney 	.d_kqfilter =	bpfkqfilter,
2504e2f199eSPoul-Henning Kamp };
25187f6c662SJulian Elischer 
252e76d823bSRobert Watson static struct filterops bpfread_filtops = {
253e76d823bSRobert Watson 	.f_isfd = 1,
254e76d823bSRobert Watson 	.f_detach = filt_bpfdetach,
255e76d823bSRobert Watson 	.f_event = filt_bpfread,
256e76d823bSRobert Watson };
25787f6c662SJulian Elischer 
258ded77e02SHartmut Brandt static struct filterops bpfwrite_filtops = {
259ded77e02SHartmut Brandt 	.f_isfd = 1,
260ded77e02SHartmut Brandt 	.f_detach = filt_bpfdetach,
261ded77e02SHartmut Brandt 	.f_event = filt_bpfwrite,
262ded77e02SHartmut Brandt };
263ded77e02SHartmut Brandt 
2644d621040SChristian S.J. Peron /*
265699281b5SAndrey V. Elsukov  * LOCKING MODEL USED BY BPF
266699281b5SAndrey V. Elsukov  *
2676c74ff0eSAlexander V. Chernikov  * Locks:
268699281b5SAndrey V. Elsukov  * 1) global lock (BPF_LOCK). Sx, used to protect some global counters,
269699281b5SAndrey V. Elsukov  * every bpf_iflist changes, serializes ioctl access to bpf descriptors.
270699281b5SAndrey V. Elsukov  * 2) Descriptor lock. Mutex, used to protect BPF buffers and various
271699281b5SAndrey V. Elsukov  * structure fields used by bpf_*tap* code.
2726c74ff0eSAlexander V. Chernikov  *
273699281b5SAndrey V. Elsukov  * Lock order: global lock, then descriptor lock.
2746c74ff0eSAlexander V. Chernikov  *
275699281b5SAndrey V. Elsukov  * There are several possible consumers:
2766c74ff0eSAlexander V. Chernikov  *
277699281b5SAndrey V. Elsukov  * 1. The kernel registers interface pointer with bpfattach().
278699281b5SAndrey V. Elsukov  * Each call allocates new bpf_if structure, references ifnet pointer
279699281b5SAndrey V. Elsukov  * and links bpf_if into bpf_iflist chain. This is protected with global
280699281b5SAndrey V. Elsukov  * lock.
2816c74ff0eSAlexander V. Chernikov  *
282699281b5SAndrey V. Elsukov  * 2. An userland application uses ioctl() call to bpf_d descriptor.
283699281b5SAndrey V. Elsukov  * All such call are serialized with global lock. BPF filters can be
2842a4bd982SGleb Smirnoff  * changed, but pointer to old filter will be freed using NET_EPOCH_CALL().
285699281b5SAndrey V. Elsukov  * Thus it should be safe for bpf_tap/bpf_mtap* code to do access to
286699281b5SAndrey V. Elsukov  * filter pointers, even if change will happen during bpf_tap execution.
2872a4bd982SGleb Smirnoff  * Destroying of bpf_d descriptor also is doing using NET_EPOCH_CALL().
2886c74ff0eSAlexander V. Chernikov  *
289699281b5SAndrey V. Elsukov  * 3. An userland application can write packets into bpf_d descriptor.
290699281b5SAndrey V. Elsukov  * There we need to be sure, that ifnet won't disappear during bpfwrite().
2916c74ff0eSAlexander V. Chernikov  *
292699281b5SAndrey V. Elsukov  * 4. The kernel invokes bpf_tap/bpf_mtap* functions. The access to
293699281b5SAndrey V. Elsukov  * bif_dlist is protected with net_epoch_preempt section. So, it should
294699281b5SAndrey V. Elsukov  * be safe to make access to bpf_d descriptor inside the section.
295699281b5SAndrey V. Elsukov  *
296699281b5SAndrey V. Elsukov  * 5. The kernel invokes bpfdetach() on interface destroying. All lists
297699281b5SAndrey V. Elsukov  * are modified with global lock held and actual free() is done using
2982a4bd982SGleb Smirnoff  * NET_EPOCH_CALL().
2996c74ff0eSAlexander V. Chernikov  */
3006c74ff0eSAlexander V. Chernikov 
301699281b5SAndrey V. Elsukov static void
302699281b5SAndrey V. Elsukov bpfif_free(epoch_context_t ctx)
303699281b5SAndrey V. Elsukov {
304699281b5SAndrey V. Elsukov 	struct bpf_if *bp;
305699281b5SAndrey V. Elsukov 
306699281b5SAndrey V. Elsukov 	bp = __containerof(ctx, struct bpf_if, epoch_ctx);
307699281b5SAndrey V. Elsukov 	if_rele(bp->bif_ifp);
308699281b5SAndrey V. Elsukov 	free(bp, M_BPF);
309699281b5SAndrey V. Elsukov }
310699281b5SAndrey V. Elsukov 
311699281b5SAndrey V. Elsukov static void
312699281b5SAndrey V. Elsukov bpfif_ref(struct bpf_if *bp)
313699281b5SAndrey V. Elsukov {
314699281b5SAndrey V. Elsukov 
315699281b5SAndrey V. Elsukov 	refcount_acquire(&bp->bif_refcnt);
316699281b5SAndrey V. Elsukov }
317699281b5SAndrey V. Elsukov 
318699281b5SAndrey V. Elsukov static void
319699281b5SAndrey V. Elsukov bpfif_rele(struct bpf_if *bp)
320699281b5SAndrey V. Elsukov {
321699281b5SAndrey V. Elsukov 
322699281b5SAndrey V. Elsukov 	if (!refcount_release(&bp->bif_refcnt))
323699281b5SAndrey V. Elsukov 		return;
3242a4bd982SGleb Smirnoff 	NET_EPOCH_CALL(bpfif_free, &bp->epoch_ctx);
325699281b5SAndrey V. Elsukov }
326699281b5SAndrey V. Elsukov 
327699281b5SAndrey V. Elsukov static void
328699281b5SAndrey V. Elsukov bpfd_ref(struct bpf_d *d)
329699281b5SAndrey V. Elsukov {
330699281b5SAndrey V. Elsukov 
331699281b5SAndrey V. Elsukov 	refcount_acquire(&d->bd_refcnt);
332699281b5SAndrey V. Elsukov }
333699281b5SAndrey V. Elsukov 
334699281b5SAndrey V. Elsukov static void
335699281b5SAndrey V. Elsukov bpfd_rele(struct bpf_d *d)
336699281b5SAndrey V. Elsukov {
337699281b5SAndrey V. Elsukov 
338699281b5SAndrey V. Elsukov 	if (!refcount_release(&d->bd_refcnt))
339699281b5SAndrey V. Elsukov 		return;
3402a4bd982SGleb Smirnoff 	NET_EPOCH_CALL(bpfd_free, &d->epoch_ctx);
341699281b5SAndrey V. Elsukov }
342699281b5SAndrey V. Elsukov 
343699281b5SAndrey V. Elsukov static struct bpf_program_buffer*
344699281b5SAndrey V. Elsukov bpf_program_buffer_alloc(size_t size, int flags)
345699281b5SAndrey V. Elsukov {
346699281b5SAndrey V. Elsukov 
347699281b5SAndrey V. Elsukov 	return (malloc(sizeof(struct bpf_program_buffer) + size,
348699281b5SAndrey V. Elsukov 	    M_BPF, flags));
349699281b5SAndrey V. Elsukov }
350699281b5SAndrey V. Elsukov 
351699281b5SAndrey V. Elsukov static void
352699281b5SAndrey V. Elsukov bpf_program_buffer_free(epoch_context_t ctx)
353699281b5SAndrey V. Elsukov {
354699281b5SAndrey V. Elsukov 	struct bpf_program_buffer *ptr;
355699281b5SAndrey V. Elsukov 
356699281b5SAndrey V. Elsukov 	ptr = __containerof(ctx, struct bpf_program_buffer, epoch_ctx);
357699281b5SAndrey V. Elsukov #ifdef BPF_JITTER
358699281b5SAndrey V. Elsukov 	if (ptr->func != NULL)
359699281b5SAndrey V. Elsukov 		bpf_destroy_jit_filter(ptr->func);
360699281b5SAndrey V. Elsukov #endif
361699281b5SAndrey V. Elsukov 	free(ptr, M_BPF);
362699281b5SAndrey V. Elsukov }
363699281b5SAndrey V. Elsukov 
3646c74ff0eSAlexander V. Chernikov /*
3654d621040SChristian S.J. Peron  * Wrapper functions for various buffering methods.  If the set of buffer
3664d621040SChristian S.J. Peron  * modes expands, we will probably want to introduce a switch data structure
3674d621040SChristian S.J. Peron  * similar to protosw, et.
3684d621040SChristian S.J. Peron  */
3694d621040SChristian S.J. Peron static void
3704d621040SChristian S.J. Peron bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
3714d621040SChristian S.J. Peron     u_int len)
3724d621040SChristian S.J. Peron {
3734d621040SChristian S.J. Peron 
374afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
3754d621040SChristian S.J. Peron 
3764d621040SChristian S.J. Peron 	switch (d->bd_bufmode) {
3774d621040SChristian S.J. Peron 	case BPF_BUFMODE_BUFFER:
3784d621040SChristian S.J. Peron 		return (bpf_buffer_append_bytes(d, buf, offset, src, len));
3794d621040SChristian S.J. Peron 
3804d621040SChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
381b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_zcopy, 1);
3824d621040SChristian S.J. Peron 		return (bpf_zerocopy_append_bytes(d, buf, offset, src, len));
3834d621040SChristian S.J. Peron 
3844d621040SChristian S.J. Peron 	default:
3854d621040SChristian S.J. Peron 		panic("bpf_buf_append_bytes");
3864d621040SChristian S.J. Peron 	}
3874d621040SChristian S.J. Peron }
3884d621040SChristian S.J. Peron 
3894d621040SChristian S.J. Peron static void
3904d621040SChristian S.J. Peron bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
3914d621040SChristian S.J. Peron     u_int len)
3924d621040SChristian S.J. Peron {
3934d621040SChristian S.J. Peron 
394afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
3954d621040SChristian S.J. Peron 
3964d621040SChristian S.J. Peron 	switch (d->bd_bufmode) {
3974d621040SChristian S.J. Peron 	case BPF_BUFMODE_BUFFER:
3984d621040SChristian S.J. Peron 		return (bpf_buffer_append_mbuf(d, buf, offset, src, len));
3994d621040SChristian S.J. Peron 
4004d621040SChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
401b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_zcopy, 1);
4024d621040SChristian S.J. Peron 		return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len));
4034d621040SChristian S.J. Peron 
4044d621040SChristian S.J. Peron 	default:
4054d621040SChristian S.J. Peron 		panic("bpf_buf_append_mbuf");
4064d621040SChristian S.J. Peron 	}
4074d621040SChristian S.J. Peron }
4084d621040SChristian S.J. Peron 
4094d621040SChristian S.J. Peron /*
41029f612ecSChristian S.J. Peron  * This function gets called when the free buffer is re-assigned.
41129f612ecSChristian S.J. Peron  */
41229f612ecSChristian S.J. Peron static void
41329f612ecSChristian S.J. Peron bpf_buf_reclaimed(struct bpf_d *d)
41429f612ecSChristian S.J. Peron {
41529f612ecSChristian S.J. Peron 
416afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
41729f612ecSChristian S.J. Peron 
41829f612ecSChristian S.J. Peron 	switch (d->bd_bufmode) {
41929f612ecSChristian S.J. Peron 	case BPF_BUFMODE_BUFFER:
42029f612ecSChristian S.J. Peron 		return;
42129f612ecSChristian S.J. Peron 
42229f612ecSChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
42329f612ecSChristian S.J. Peron 		bpf_zerocopy_buf_reclaimed(d);
42429f612ecSChristian S.J. Peron 		return;
42529f612ecSChristian S.J. Peron 
42629f612ecSChristian S.J. Peron 	default:
42729f612ecSChristian S.J. Peron 		panic("bpf_buf_reclaimed");
42829f612ecSChristian S.J. Peron 	}
42929f612ecSChristian S.J. Peron }
43029f612ecSChristian S.J. Peron 
43129f612ecSChristian S.J. Peron /*
4324d621040SChristian S.J. Peron  * If the buffer mechanism has a way to decide that a held buffer can be made
4334d621040SChristian S.J. Peron  * free, then it is exposed via the bpf_canfreebuf() interface.  (1) is
4344d621040SChristian S.J. Peron  * returned if the buffer can be discarded, (0) is returned if it cannot.
4354d621040SChristian S.J. Peron  */
4364d621040SChristian S.J. Peron static int
4374d621040SChristian S.J. Peron bpf_canfreebuf(struct bpf_d *d)
4384d621040SChristian S.J. Peron {
4394d621040SChristian S.J. Peron 
4404d621040SChristian S.J. Peron 	BPFD_LOCK_ASSERT(d);
4414d621040SChristian S.J. Peron 
4424d621040SChristian S.J. Peron 	switch (d->bd_bufmode) {
4434d621040SChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
4444d621040SChristian S.J. Peron 		return (bpf_zerocopy_canfreebuf(d));
4454d621040SChristian S.J. Peron 	}
4464d621040SChristian S.J. Peron 	return (0);
4474d621040SChristian S.J. Peron }
4484d621040SChristian S.J. Peron 
449a7a91e65SRobert Watson /*
450a7a91e65SRobert Watson  * Allow the buffer model to indicate that the current store buffer is
451a7a91e65SRobert Watson  * immutable, regardless of the appearance of space.  Return (1) if the
452a7a91e65SRobert Watson  * buffer is writable, and (0) if not.
453a7a91e65SRobert Watson  */
454a7a91e65SRobert Watson static int
455a7a91e65SRobert Watson bpf_canwritebuf(struct bpf_d *d)
456a7a91e65SRobert Watson {
457a7a91e65SRobert Watson 	BPFD_LOCK_ASSERT(d);
458a7a91e65SRobert Watson 
459a7a91e65SRobert Watson 	switch (d->bd_bufmode) {
460a7a91e65SRobert Watson 	case BPF_BUFMODE_ZBUF:
461a7a91e65SRobert Watson 		return (bpf_zerocopy_canwritebuf(d));
462a7a91e65SRobert Watson 	}
463a7a91e65SRobert Watson 	return (1);
464a7a91e65SRobert Watson }
465a7a91e65SRobert Watson 
466a7a91e65SRobert Watson /*
467a7a91e65SRobert Watson  * Notify buffer model that an attempt to write to the store buffer has
468a7a91e65SRobert Watson  * resulted in a dropped packet, in which case the buffer may be considered
469a7a91e65SRobert Watson  * full.
470a7a91e65SRobert Watson  */
471a7a91e65SRobert Watson static void
472a7a91e65SRobert Watson bpf_buffull(struct bpf_d *d)
473a7a91e65SRobert Watson {
474a7a91e65SRobert Watson 
475afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
476a7a91e65SRobert Watson 
477a7a91e65SRobert Watson 	switch (d->bd_bufmode) {
478a7a91e65SRobert Watson 	case BPF_BUFMODE_ZBUF:
479a7a91e65SRobert Watson 		bpf_zerocopy_buffull(d);
480a7a91e65SRobert Watson 		break;
481a7a91e65SRobert Watson 	}
482a7a91e65SRobert Watson }
483a7a91e65SRobert Watson 
484a7a91e65SRobert Watson /*
485a7a91e65SRobert Watson  * Notify the buffer model that a buffer has moved into the hold position.
486a7a91e65SRobert Watson  */
4874d621040SChristian S.J. Peron void
4884d621040SChristian S.J. Peron bpf_bufheld(struct bpf_d *d)
4894d621040SChristian S.J. Peron {
4904d621040SChristian S.J. Peron 
491afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
4924d621040SChristian S.J. Peron 
4934d621040SChristian S.J. Peron 	switch (d->bd_bufmode) {
4944d621040SChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
4954d621040SChristian S.J. Peron 		bpf_zerocopy_bufheld(d);
4964d621040SChristian S.J. Peron 		break;
4974d621040SChristian S.J. Peron 	}
4984d621040SChristian S.J. Peron }
4994d621040SChristian S.J. Peron 
5004d621040SChristian S.J. Peron static void
5014d621040SChristian S.J. Peron bpf_free(struct bpf_d *d)
5024d621040SChristian S.J. Peron {
5034d621040SChristian S.J. Peron 
5044d621040SChristian S.J. Peron 	switch (d->bd_bufmode) {
5054d621040SChristian S.J. Peron 	case BPF_BUFMODE_BUFFER:
5064d621040SChristian S.J. Peron 		return (bpf_buffer_free(d));
5074d621040SChristian S.J. Peron 
5084d621040SChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
5094d621040SChristian S.J. Peron 		return (bpf_zerocopy_free(d));
5104d621040SChristian S.J. Peron 
5114d621040SChristian S.J. Peron 	default:
5124d621040SChristian S.J. Peron 		panic("bpf_buf_free");
5134d621040SChristian S.J. Peron 	}
5144d621040SChristian S.J. Peron }
5154d621040SChristian S.J. Peron 
5164d621040SChristian S.J. Peron static int
5174d621040SChristian S.J. Peron bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio)
5184d621040SChristian S.J. Peron {
5194d621040SChristian S.J. Peron 
5204d621040SChristian S.J. Peron 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
5214d621040SChristian S.J. Peron 		return (EOPNOTSUPP);
5224d621040SChristian S.J. Peron 	return (bpf_buffer_uiomove(d, buf, len, uio));
5234d621040SChristian S.J. Peron }
5244d621040SChristian S.J. Peron 
5254d621040SChristian S.J. Peron static int
5264d621040SChristian S.J. Peron bpf_ioctl_sblen(struct bpf_d *d, u_int *i)
5274d621040SChristian S.J. Peron {
5284d621040SChristian S.J. Peron 
5294d621040SChristian S.J. Peron 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
5304d621040SChristian S.J. Peron 		return (EOPNOTSUPP);
5314d621040SChristian S.J. Peron 	return (bpf_buffer_ioctl_sblen(d, i));
5324d621040SChristian S.J. Peron }
5334d621040SChristian S.J. Peron 
5344d621040SChristian S.J. Peron static int
5354d621040SChristian S.J. Peron bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
5364d621040SChristian S.J. Peron {
5374d621040SChristian S.J. Peron 
5384d621040SChristian S.J. Peron 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
5394d621040SChristian S.J. Peron 		return (EOPNOTSUPP);
5404d621040SChristian S.J. Peron 	return (bpf_zerocopy_ioctl_getzmax(td, d, i));
5414d621040SChristian S.J. Peron }
5424d621040SChristian S.J. Peron 
5434d621040SChristian S.J. Peron static int
5444d621040SChristian S.J. Peron bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
5454d621040SChristian S.J. Peron {
5464d621040SChristian S.J. Peron 
5474d621040SChristian S.J. Peron 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
5484d621040SChristian S.J. Peron 		return (EOPNOTSUPP);
5494d621040SChristian S.J. Peron 	return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz));
5504d621040SChristian S.J. Peron }
5514d621040SChristian S.J. Peron 
5524d621040SChristian S.J. Peron static int
5534d621040SChristian S.J. Peron bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
5544d621040SChristian S.J. Peron {
5554d621040SChristian S.J. Peron 
5564d621040SChristian S.J. Peron 	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
5574d621040SChristian S.J. Peron 		return (EOPNOTSUPP);
5584d621040SChristian S.J. Peron 	return (bpf_zerocopy_ioctl_setzbuf(td, d, bz));
5594d621040SChristian S.J. Peron }
5604d621040SChristian S.J. Peron 
5614d621040SChristian S.J. Peron /*
5624d621040SChristian S.J. Peron  * General BPF functions.
5634d621040SChristian S.J. Peron  */
564df8bae1dSRodney W. Grimes static int
565cb44b6dfSAndrew Thompson bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
5664fb3a820SAlexander V. Chernikov     struct sockaddr *sockp, int *hdrlen, struct bpf_d *d)
567df8bae1dSRodney W. Grimes {
568246b5467SSam Leffler 	const struct ieee80211_bpf_params *p;
569cb44b6dfSAndrew Thompson 	struct ether_header *eh;
570df8bae1dSRodney W. Grimes 	struct mbuf *m;
571df8bae1dSRodney W. Grimes 	int error;
572df8bae1dSRodney W. Grimes 	int len;
573df8bae1dSRodney W. Grimes 	int hlen;
57493e39f0bSChristian S.J. Peron 	int slen;
575df8bae1dSRodney W. Grimes 
576df8bae1dSRodney W. Grimes 	/*
577df8bae1dSRodney W. Grimes 	 * Build a sockaddr based on the data link layer type.
578df8bae1dSRodney W. Grimes 	 * We do this at this level because the ethernet header
579df8bae1dSRodney W. Grimes 	 * is copied directly into the data field of the sockaddr.
580df8bae1dSRodney W. Grimes 	 * In the case of SLIP, there is no header and the packet
581df8bae1dSRodney W. Grimes 	 * is forwarded as is.
582df8bae1dSRodney W. Grimes 	 * Also, we are careful to leave room at the front of the mbuf
583df8bae1dSRodney W. Grimes 	 * for the link level header.
584df8bae1dSRodney W. Grimes 	 */
585df8bae1dSRodney W. Grimes 	switch (linktype) {
586df8bae1dSRodney W. Grimes 	case DLT_SLIP:
587df8bae1dSRodney W. Grimes 		sockp->sa_family = AF_INET;
588df8bae1dSRodney W. Grimes 		hlen = 0;
589df8bae1dSRodney W. Grimes 		break;
590df8bae1dSRodney W. Grimes 
591df8bae1dSRodney W. Grimes 	case DLT_EN10MB:
592df8bae1dSRodney W. Grimes 		sockp->sa_family = AF_UNSPEC;
593df8bae1dSRodney W. Grimes 		/* XXX Would MAXLINKHDR be better? */
594797f247bSMatthew N. Dodd 		hlen = ETHER_HDR_LEN;
595df8bae1dSRodney W. Grimes 		break;
596df8bae1dSRodney W. Grimes 
597df8bae1dSRodney W. Grimes 	case DLT_FDDI:
598d41f24e7SDavid Greenman 		sockp->sa_family = AF_IMPLINK;
599d41f24e7SDavid Greenman 		hlen = 0;
600df8bae1dSRodney W. Grimes 		break;
601df8bae1dSRodney W. Grimes 
60222f05c43SAndrey A. Chernov 	case DLT_RAW:
603df8bae1dSRodney W. Grimes 		sockp->sa_family = AF_UNSPEC;
604df8bae1dSRodney W. Grimes 		hlen = 0;
605df8bae1dSRodney W. Grimes 		break;
606df8bae1dSRodney W. Grimes 
60701399f34SDavid Malone 	case DLT_NULL:
60801399f34SDavid Malone 		/*
60901399f34SDavid Malone 		 * null interface types require a 4 byte pseudo header which
61001399f34SDavid Malone 		 * corresponds to the address family of the packet.
61101399f34SDavid Malone 		 */
61201399f34SDavid Malone 		sockp->sa_family = AF_UNSPEC;
61301399f34SDavid Malone 		hlen = 4;
61401399f34SDavid Malone 		break;
61501399f34SDavid Malone 
6164f53e3ccSKenjiro Cho 	case DLT_ATM_RFC1483:
6174f53e3ccSKenjiro Cho 		/*
6184f53e3ccSKenjiro Cho 		 * en atm driver requires 4-byte atm pseudo header.
6194f53e3ccSKenjiro Cho 		 * though it isn't standard, vpi:vci needs to be
6204f53e3ccSKenjiro Cho 		 * specified anyway.
6214f53e3ccSKenjiro Cho 		 */
6224f53e3ccSKenjiro Cho 		sockp->sa_family = AF_UNSPEC;
6234f53e3ccSKenjiro Cho 		hlen = 12;	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
6244f53e3ccSKenjiro Cho 		break;
6254f53e3ccSKenjiro Cho 
62630fa52a6SBrian Somers 	case DLT_PPP:
62730fa52a6SBrian Somers 		sockp->sa_family = AF_UNSPEC;
62830fa52a6SBrian Somers 		hlen = 4;	/* This should match PPP_HDRLEN */
62930fa52a6SBrian Somers 		break;
63030fa52a6SBrian Somers 
631246b5467SSam Leffler 	case DLT_IEEE802_11:		/* IEEE 802.11 wireless */
632246b5467SSam Leffler 		sockp->sa_family = AF_IEEE80211;
633246b5467SSam Leffler 		hlen = 0;
634246b5467SSam Leffler 		break;
635246b5467SSam Leffler 
636246b5467SSam Leffler 	case DLT_IEEE802_11_RADIO:	/* IEEE 802.11 wireless w/ phy params */
637246b5467SSam Leffler 		sockp->sa_family = AF_IEEE80211;
638246b5467SSam Leffler 		sockp->sa_len = 12;	/* XXX != 0 */
639246b5467SSam Leffler 		hlen = sizeof(struct ieee80211_bpf_params);
640246b5467SSam Leffler 		break;
641246b5467SSam Leffler 
642df8bae1dSRodney W. Grimes 	default:
643df8bae1dSRodney W. Grimes 		return (EIO);
644df8bae1dSRodney W. Grimes 	}
645df8bae1dSRodney W. Grimes 
646df8bae1dSRodney W. Grimes 	len = uio->uio_resid;
647ed63043bSGleb Smirnoff 	if (len < hlen || len - hlen > ifp->if_mtu)
64801399f34SDavid Malone 		return (EMSGSIZE);
64901399f34SDavid Malone 
650343bf78eSZhenlei Huang 	/* Allocate a mbuf, up to MJUM16BYTES bytes, for our write. */
651a051ca72SKristof Provost 	m = m_get3(len, M_WAITOK, MT_DATA, M_PKTHDR);
652ed63043bSGleb Smirnoff 	if (m == NULL)
653df8bae1dSRodney W. Grimes 		return (EIO);
654963e4c2aSGarrett Wollman 	m->m_pkthdr.len = m->m_len = len;
655df8bae1dSRodney W. Grimes 	*mp = m;
65624a229f4SSam Leffler 
65793e39f0bSChristian S.J. Peron 	error = uiomove(mtod(m, u_char *), len, uio);
65893e39f0bSChristian S.J. Peron 	if (error)
65993e39f0bSChristian S.J. Peron 		goto bad;
66093e39f0bSChristian S.J. Peron 
6614fb3a820SAlexander V. Chernikov 	slen = bpf_filter(d->bd_wfilter, mtod(m, u_char *), len, len);
66293e39f0bSChristian S.J. Peron 	if (slen == 0) {
66393e39f0bSChristian S.J. Peron 		error = EPERM;
66493e39f0bSChristian S.J. Peron 		goto bad;
66593e39f0bSChristian S.J. Peron 	}
66693e39f0bSChristian S.J. Peron 
667cb44b6dfSAndrew Thompson 	/* Check for multicast destination */
668cb44b6dfSAndrew Thompson 	switch (linktype) {
669cb44b6dfSAndrew Thompson 	case DLT_EN10MB:
670cb44b6dfSAndrew Thompson 		eh = mtod(m, struct ether_header *);
671cb44b6dfSAndrew Thompson 		if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
672cb44b6dfSAndrew Thompson 			if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost,
673cb44b6dfSAndrew Thompson 			    ETHER_ADDR_LEN) == 0)
674cb44b6dfSAndrew Thompson 				m->m_flags |= M_BCAST;
675cb44b6dfSAndrew Thompson 			else
676cb44b6dfSAndrew Thompson 				m->m_flags |= M_MCAST;
677cb44b6dfSAndrew Thompson 		}
6784fb3a820SAlexander V. Chernikov 		if (d->bd_hdrcmplt == 0) {
6794fb3a820SAlexander V. Chernikov 			memcpy(eh->ether_shost, IF_LLADDR(ifp),
6804fb3a820SAlexander V. Chernikov 			    sizeof(eh->ether_shost));
6814fb3a820SAlexander V. Chernikov 		}
682cb44b6dfSAndrew Thompson 		break;
683cb44b6dfSAndrew Thompson 	}
684cb44b6dfSAndrew Thompson 
685df8bae1dSRodney W. Grimes 	/*
68693e39f0bSChristian S.J. Peron 	 * Make room for link header, and copy it to sockaddr
687df8bae1dSRodney W. Grimes 	 */
688df8bae1dSRodney W. Grimes 	if (hlen != 0) {
689246b5467SSam Leffler 		if (sockp->sa_family == AF_IEEE80211) {
690246b5467SSam Leffler 			/*
691246b5467SSam Leffler 			 * Collect true length from the parameter header
692246b5467SSam Leffler 			 * NB: sockp is known to be zero'd so if we do a
693246b5467SSam Leffler 			 *     short copy unspecified parameters will be
694246b5467SSam Leffler 			 *     zero.
695246b5467SSam Leffler 			 * NB: packet may not be aligned after stripping
696246b5467SSam Leffler 			 *     bpf params
697246b5467SSam Leffler 			 * XXX check ibp_vers
698246b5467SSam Leffler 			 */
699246b5467SSam Leffler 			p = mtod(m, const struct ieee80211_bpf_params *);
700246b5467SSam Leffler 			hlen = p->ibp_len;
701246b5467SSam Leffler 			if (hlen > sizeof(sockp->sa_data)) {
702246b5467SSam Leffler 				error = EINVAL;
703246b5467SSam Leffler 				goto bad;
704246b5467SSam Leffler 			}
705246b5467SSam Leffler 		}
706a09968c4SAdrian Chadd 		bcopy(mtod(m, const void *), sockp->sa_data, hlen);
707df8bae1dSRodney W. Grimes 	}
708560a54e1SJung-uk Kim 	*hdrlen = hlen;
70993e39f0bSChristian S.J. Peron 
710df8bae1dSRodney W. Grimes 	return (0);
711df8bae1dSRodney W. Grimes bad:
712df8bae1dSRodney W. Grimes 	m_freem(m);
713df8bae1dSRodney W. Grimes 	return (error);
714df8bae1dSRodney W. Grimes }
715df8bae1dSRodney W. Grimes 
716df8bae1dSRodney W. Grimes /*
717699281b5SAndrey V. Elsukov  * Attach descriptor to the bpf interface, i.e. make d listen on bp,
718699281b5SAndrey V. Elsukov  * then reset its buffers and counters with reset_d().
719df8bae1dSRodney W. Grimes  */
720df8bae1dSRodney W. Grimes static void
72119ba8395SChristian S.J. Peron bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
722df8bae1dSRodney W. Grimes {
7236c74ff0eSAlexander V. Chernikov 	int op_w;
7246c74ff0eSAlexander V. Chernikov 
7256c74ff0eSAlexander V. Chernikov 	BPF_LOCK_ASSERT();
7266c74ff0eSAlexander V. Chernikov 
7276c74ff0eSAlexander V. Chernikov 	/*
7286c74ff0eSAlexander V. Chernikov 	 * Save sysctl value to protect from sysctl change
7296c74ff0eSAlexander V. Chernikov 	 * between reads
7306c74ff0eSAlexander V. Chernikov 	 */
73171448753SAlexander Motin 	op_w = V_bpf_optimize_writers || d->bd_writer;
7326c74ff0eSAlexander V. Chernikov 
7336c74ff0eSAlexander V. Chernikov 	if (d->bd_bif != NULL)
734699281b5SAndrey V. Elsukov 		bpf_detachd_locked(d, false);
735df8bae1dSRodney W. Grimes 	/*
73651ec1eb7SAlexander V. Chernikov 	 * Point d at bp, and add d to the interface's list.
737f87e372eSLuiz Otavio O Souza 	 * Since there are many applications using BPF for
73851ec1eb7SAlexander V. Chernikov 	 * sending raw packets only (dhcpd, cdpd are good examples)
73951ec1eb7SAlexander V. Chernikov 	 * we can delay adding d to the list of active listeners until
74051ec1eb7SAlexander V. Chernikov 	 * some filter is configured.
741df8bae1dSRodney W. Grimes 	 */
74251ec1eb7SAlexander V. Chernikov 
743afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
744699281b5SAndrey V. Elsukov 	/*
745699281b5SAndrey V. Elsukov 	 * Hold reference to bpif while descriptor uses this interface.
746699281b5SAndrey V. Elsukov 	 */
747699281b5SAndrey V. Elsukov 	bpfif_ref(bp);
7486c74ff0eSAlexander V. Chernikov 	d->bd_bif = bp;
7496c74ff0eSAlexander V. Chernikov 	if (op_w != 0) {
75051ec1eb7SAlexander V. Chernikov 		/* Add to writers-only list */
751699281b5SAndrey V. Elsukov 		CK_LIST_INSERT_HEAD(&bp->bif_wlist, d, bd_next);
75251ec1eb7SAlexander V. Chernikov 		/*
75351ec1eb7SAlexander V. Chernikov 		 * We decrement bd_writer on every filter set operation.
75451ec1eb7SAlexander V. Chernikov 		 * First BIOCSETF is done by pcap_open_live() to set up
755699281b5SAndrey V. Elsukov 		 * snap length. After that appliation usually sets its own
756699281b5SAndrey V. Elsukov 		 * filter.
75751ec1eb7SAlexander V. Chernikov 		 */
75851ec1eb7SAlexander V. Chernikov 		d->bd_writer = 2;
75951ec1eb7SAlexander V. Chernikov 	} else
760699281b5SAndrey V. Elsukov 		CK_LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
761df8bae1dSRodney W. Grimes 
762699281b5SAndrey V. Elsukov 	reset_d(d);
763426682b0SMark Johnston 
764426682b0SMark Johnston 	/* Trigger EVFILT_WRITE events. */
765426682b0SMark Johnston 	bpf_wakeup(d);
766426682b0SMark Johnston 
767afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
76851ec1eb7SAlexander V. Chernikov 	bpf_bpfd_cnt++;
76951ec1eb7SAlexander V. Chernikov 
77051ec1eb7SAlexander V. Chernikov 	CTR3(KTR_NET, "%s: bpf_attach called by pid %d, adding to %s list",
77151ec1eb7SAlexander V. Chernikov 	    __func__, d->bd_pid, d->bd_writer ? "writer" : "active");
77251ec1eb7SAlexander V. Chernikov 
7736c74ff0eSAlexander V. Chernikov 	if (op_w == 0)
77451ec1eb7SAlexander V. Chernikov 		EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1);
77551ec1eb7SAlexander V. Chernikov }
77651ec1eb7SAlexander V. Chernikov 
77751ec1eb7SAlexander V. Chernikov /*
778402000ffSAlexander V. Chernikov  * Check if we need to upgrade our descriptor @d from write-only mode.
779402000ffSAlexander V. Chernikov  */
780402000ffSAlexander V. Chernikov static int
781699281b5SAndrey V. Elsukov bpf_check_upgrade(u_long cmd, struct bpf_d *d, struct bpf_insn *fcode,
782699281b5SAndrey V. Elsukov     int flen)
783402000ffSAlexander V. Chernikov {
784402000ffSAlexander V. Chernikov 	int is_snap, need_upgrade;
785402000ffSAlexander V. Chernikov 
786402000ffSAlexander V. Chernikov 	/*
787402000ffSAlexander V. Chernikov 	 * Check if we've already upgraded or new filter is empty.
788402000ffSAlexander V. Chernikov 	 */
789402000ffSAlexander V. Chernikov 	if (d->bd_writer == 0 || fcode == NULL)
790402000ffSAlexander V. Chernikov 		return (0);
791402000ffSAlexander V. Chernikov 
792402000ffSAlexander V. Chernikov 	need_upgrade = 0;
793402000ffSAlexander V. Chernikov 
794402000ffSAlexander V. Chernikov 	/*
795402000ffSAlexander V. Chernikov 	 * Check if cmd looks like snaplen setting from
796402000ffSAlexander V. Chernikov 	 * pcap_bpf.c:pcap_open_live().
797402000ffSAlexander V. Chernikov 	 * Note we're not checking .k value here:
798caa7e52fSEitan Adler 	 * while pcap_open_live() definitely sets to non-zero value,
799402000ffSAlexander V. Chernikov 	 * we'd prefer to treat k=0 (deny ALL) case the same way: e.g.
800402000ffSAlexander V. Chernikov 	 * do not consider upgrading immediately
801402000ffSAlexander V. Chernikov 	 */
802699281b5SAndrey V. Elsukov 	if (cmd == BIOCSETF && flen == 1 &&
803699281b5SAndrey V. Elsukov 	    fcode[0].code == (BPF_RET | BPF_K))
804402000ffSAlexander V. Chernikov 		is_snap = 1;
805402000ffSAlexander V. Chernikov 	else
806402000ffSAlexander V. Chernikov 		is_snap = 0;
807402000ffSAlexander V. Chernikov 
808402000ffSAlexander V. Chernikov 	if (is_snap == 0) {
809402000ffSAlexander V. Chernikov 		/*
810402000ffSAlexander V. Chernikov 		 * We're setting first filter and it doesn't look like
811402000ffSAlexander V. Chernikov 		 * setting snaplen.  We're probably using bpf directly.
812402000ffSAlexander V. Chernikov 		 * Upgrade immediately.
813402000ffSAlexander V. Chernikov 		 */
814402000ffSAlexander V. Chernikov 		need_upgrade = 1;
815402000ffSAlexander V. Chernikov 	} else {
816402000ffSAlexander V. Chernikov 		/*
817402000ffSAlexander V. Chernikov 		 * Do not require upgrade by first BIOCSETF
818402000ffSAlexander V. Chernikov 		 * (used to set snaplen) by pcap_open_live().
819402000ffSAlexander V. Chernikov 		 */
820402000ffSAlexander V. Chernikov 
821402000ffSAlexander V. Chernikov 		if (--d->bd_writer == 0) {
822402000ffSAlexander V. Chernikov 			/*
823402000ffSAlexander V. Chernikov 			 * First snaplen filter has already
824402000ffSAlexander V. Chernikov 			 * been set. This is probably catch-all
825402000ffSAlexander V. Chernikov 			 * filter
826402000ffSAlexander V. Chernikov 			 */
827402000ffSAlexander V. Chernikov 			need_upgrade = 1;
828402000ffSAlexander V. Chernikov 		}
829402000ffSAlexander V. Chernikov 	}
830402000ffSAlexander V. Chernikov 
831402000ffSAlexander V. Chernikov 	CTR5(KTR_NET,
832402000ffSAlexander V. Chernikov 	    "%s: filter function set by pid %d, "
833402000ffSAlexander V. Chernikov 	    "bd_writer counter %d, snap %d upgrade %d",
834402000ffSAlexander V. Chernikov 	    __func__, d->bd_pid, d->bd_writer,
835402000ffSAlexander V. Chernikov 	    is_snap, need_upgrade);
836402000ffSAlexander V. Chernikov 
837402000ffSAlexander V. Chernikov 	return (need_upgrade);
838402000ffSAlexander V. Chernikov }
839402000ffSAlexander V. Chernikov 
840402000ffSAlexander V. Chernikov /*
841df8bae1dSRodney W. Grimes  * Detach a file from its interface.
842df8bae1dSRodney W. Grimes  */
843df8bae1dSRodney W. Grimes static void
84419ba8395SChristian S.J. Peron bpf_detachd(struct bpf_d *d)
845df8bae1dSRodney W. Grimes {
8466c74ff0eSAlexander V. Chernikov 	BPF_LOCK();
847699281b5SAndrey V. Elsukov 	bpf_detachd_locked(d, false);
8486c74ff0eSAlexander V. Chernikov 	BPF_UNLOCK();
8496c74ff0eSAlexander V. Chernikov }
8506c74ff0eSAlexander V. Chernikov 
8516c74ff0eSAlexander V. Chernikov static void
852699281b5SAndrey V. Elsukov bpf_detachd_locked(struct bpf_d *d, bool detached_ifp)
8536c74ff0eSAlexander V. Chernikov {
854df8bae1dSRodney W. Grimes 	struct bpf_if *bp;
85546448b5aSRobert Watson 	struct ifnet *ifp;
856699281b5SAndrey V. Elsukov 	int error;
85751ec1eb7SAlexander V. Chernikov 
858e4b3229aSAlexander V. Chernikov 	BPF_LOCK_ASSERT();
859699281b5SAndrey V. Elsukov 	CTR2(KTR_NET, "%s: detach required by pid %d", __func__, d->bd_pid);
860e4b3229aSAlexander V. Chernikov 
8616c74ff0eSAlexander V. Chernikov 	/* Check if descriptor is attached */
8626c74ff0eSAlexander V. Chernikov 	if ((bp = d->bd_bif) == NULL)
8636c74ff0eSAlexander V. Chernikov 		return;
8646c74ff0eSAlexander V. Chernikov 
865afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
866de253273SAndrey V. Elsukov 	/* Remove d from the interface's descriptor list. */
867de253273SAndrey V. Elsukov 	CK_LIST_REMOVE(d, bd_next);
86851ec1eb7SAlexander V. Chernikov 	/* Save bd_writer value */
86951ec1eb7SAlexander V. Chernikov 	error = d->bd_writer;
870e4b3229aSAlexander V. Chernikov 	ifp = bp->bif_ifp;
871572bde2aSRobert Watson 	d->bd_bif = NULL;
872699281b5SAndrey V. Elsukov 	if (detached_ifp) {
873699281b5SAndrey V. Elsukov 		/*
874699281b5SAndrey V. Elsukov 		 * Notify descriptor as it's detached, so that any
875699281b5SAndrey V. Elsukov 		 * sleepers wake up and get ENXIO.
876699281b5SAndrey V. Elsukov 		 */
877699281b5SAndrey V. Elsukov 		bpf_wakeup(d);
878699281b5SAndrey V. Elsukov 	}
879afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
880e4b3229aSAlexander V. Chernikov 	bpf_bpfd_cnt--;
88146448b5aSRobert Watson 
88251ec1eb7SAlexander V. Chernikov 	/* Call event handler iff d is attached */
88351ec1eb7SAlexander V. Chernikov 	if (error == 0)
8845ce8d970SSam Leffler 		EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0);
885b743c310SSam Leffler 
886df8bae1dSRodney W. Grimes 	/*
887df8bae1dSRodney W. Grimes 	 * Check if this descriptor had requested promiscuous mode.
888699281b5SAndrey V. Elsukov 	 * If so and ifnet is not detached, turn it off.
889df8bae1dSRodney W. Grimes 	 */
890699281b5SAndrey V. Elsukov 	if (d->bd_promisc && !detached_ifp) {
891df8bae1dSRodney W. Grimes 		d->bd_promisc = 0;
89297021c24SMarko Zec 		CURVNET_SET(ifp->if_vnet);
89346448b5aSRobert Watson 		error = ifpromisc(ifp, 0);
89497021c24SMarko Zec 		CURVNET_RESTORE();
8956e891d64SPoul-Henning Kamp 		if (error != 0 && error != ENXIO) {
896df8bae1dSRodney W. Grimes 			/*
8976e891d64SPoul-Henning Kamp 			 * ENXIO can happen if a pccard is unplugged
898df8bae1dSRodney W. Grimes 			 * Something is really wrong if we were able to put
899df8bae1dSRodney W. Grimes 			 * the driver into promiscuous mode, but can't
900df8bae1dSRodney W. Grimes 			 * take it out.
901df8bae1dSRodney W. Grimes 			 */
9028eab61f3SSam Leffler 			if_printf(bp->bif_ifp,
9038eab61f3SSam Leffler 				"bpf_detach: ifpromisc failed (%d)\n", error);
9046e891d64SPoul-Henning Kamp 		}
905df8bae1dSRodney W. Grimes 	}
906699281b5SAndrey V. Elsukov 	bpfif_rele(bp);
907df8bae1dSRodney W. Grimes }
908df8bae1dSRodney W. Grimes 
909df8bae1dSRodney W. Grimes /*
910136600feSEd Schouten  * Close the descriptor by detaching it from its interface,
911136600feSEd Schouten  * deallocating its buffers, and marking it free.
912136600feSEd Schouten  */
913136600feSEd Schouten static void
914136600feSEd Schouten bpf_dtor(void *data)
915136600feSEd Schouten {
916136600feSEd Schouten 	struct bpf_d *d = data;
917136600feSEd Schouten 
918afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
919136600feSEd Schouten 	if (d->bd_state == BPF_WAITING)
920136600feSEd Schouten 		callout_stop(&d->bd_callout);
921136600feSEd Schouten 	d->bd_state = BPF_IDLE;
922afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
923136600feSEd Schouten 	funsetown(&d->bd_sigio);
924136600feSEd Schouten 	bpf_detachd(d);
925136600feSEd Schouten #ifdef MAC
926136600feSEd Schouten 	mac_bpfdesc_destroy(d);
927136600feSEd Schouten #endif /* MAC */
9286aba400aSAttilio Rao 	seldrain(&d->bd_sel);
929136600feSEd Schouten 	knlist_destroy(&d->bd_sel.si_note);
9309fee1bd1SJung-uk Kim 	callout_drain(&d->bd_callout);
931699281b5SAndrey V. Elsukov 	bpfd_rele(d);
932136600feSEd Schouten }
933136600feSEd Schouten 
934136600feSEd Schouten /*
935df8bae1dSRodney W. Grimes  * Open ethernet device.  Returns ENXIO for illegal minor device number,
936df8bae1dSRodney W. Grimes  * EBUSY if file is open by another process.
937df8bae1dSRodney W. Grimes  */
938df8bae1dSRodney W. Grimes /* ARGSUSED */
93987f6c662SJulian Elischer static	int
94019ba8395SChristian S.J. Peron bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
941df8bae1dSRodney W. Grimes {
942e7bb21b3SJonathan Lemon 	struct bpf_d *d;
9434f42daa4SLuiz Otavio O Souza 	int error;
944df8bae1dSRodney W. Grimes 
9451ede983cSDag-Erling Smørgrav 	d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
946136600feSEd Schouten 	error = devfs_set_cdevpriv(d, bpf_dtor);
947136600feSEd Schouten 	if (error != 0) {
948136600feSEd Schouten 		free(d, M_BPF);
949136600feSEd Schouten 		return (error);
950136600feSEd Schouten 	}
9514d621040SChristian S.J. Peron 
952b2b7ca49SAlexander V. Chernikov 	/* Setup counters */
953b2b7ca49SAlexander V. Chernikov 	d->bd_rcount = counter_u64_alloc(M_WAITOK);
954b2b7ca49SAlexander V. Chernikov 	d->bd_dcount = counter_u64_alloc(M_WAITOK);
955b2b7ca49SAlexander V. Chernikov 	d->bd_fcount = counter_u64_alloc(M_WAITOK);
956b2b7ca49SAlexander V. Chernikov 	d->bd_wcount = counter_u64_alloc(M_WAITOK);
957b2b7ca49SAlexander V. Chernikov 	d->bd_wfcount = counter_u64_alloc(M_WAITOK);
958b2b7ca49SAlexander V. Chernikov 	d->bd_wdcount = counter_u64_alloc(M_WAITOK);
959b2b7ca49SAlexander V. Chernikov 	d->bd_zcopy = counter_u64_alloc(M_WAITOK);
960b2b7ca49SAlexander V. Chernikov 
9614d621040SChristian S.J. Peron 	/*
9624d621040SChristian S.J. Peron 	 * For historical reasons, perform a one-time initialization call to
9634d621040SChristian S.J. Peron 	 * the buffer routines, even though we're not yet committed to a
9644d621040SChristian S.J. Peron 	 * particular buffer method.
9654d621040SChristian S.J. Peron 	 */
9664d621040SChristian S.J. Peron 	bpf_buffer_init(d);
96771448753SAlexander Motin 	if ((flags & FREAD) == 0)
96871448753SAlexander Motin 		d->bd_writer = 2;
9693b3b91e7SGuy Helmer 	d->bd_hbuf_in_use = 0;
9704d621040SChristian S.J. Peron 	d->bd_bufmode = BPF_BUFMODE_BUFFER;
97100a83887SPaul Traina 	d->bd_sig = SIGIO;
972560a54e1SJung-uk Kim 	d->bd_direction = BPF_D_INOUT;
9738288117aSMateusz Guzik 	refcount_init(&d->bd_refcnt, 1);
974e4b3229aSAlexander V. Chernikov 	BPF_PID_REFRESH(d, td);
97582f4445dSRobert Watson #ifdef MAC
97630d239bcSRobert Watson 	mac_bpfdesc_init(d);
97730d239bcSRobert Watson 	mac_bpfdesc_create(td->td_ucred, d);
97882f4445dSRobert Watson #endif
979afa85850SAlexander V. Chernikov 	mtx_init(&d->bd_lock, devtoname(dev), "bpf cdev lock", MTX_DEF);
980afa85850SAlexander V. Chernikov 	callout_init_mtx(&d->bd_callout, &d->bd_lock, 0);
981afa85850SAlexander V. Chernikov 	knlist_init_mtx(&d->bd_sel.si_note, &d->bd_lock);
982df8bae1dSRodney W. Grimes 
9831e7fe2fbSLuiz Otavio O Souza 	/* Disable VLAN pcp tagging. */
9841e7fe2fbSLuiz Otavio O Souza 	d->bd_pcp = 0;
9851e7fe2fbSLuiz Otavio O Souza 
986df8bae1dSRodney W. Grimes 	return (0);
987df8bae1dSRodney W. Grimes }
988df8bae1dSRodney W. Grimes 
989df8bae1dSRodney W. Grimes /*
990df8bae1dSRodney W. Grimes  *  bpfread - read next chunk of packets from buffers
991df8bae1dSRodney W. Grimes  */
99287f6c662SJulian Elischer static	int
99319ba8395SChristian S.J. Peron bpfread(struct cdev *dev, struct uio *uio, int ioflag)
994df8bae1dSRodney W. Grimes {
995136600feSEd Schouten 	struct bpf_d *d;
996df8bae1dSRodney W. Grimes 	int error;
9978df67d77SJung-uk Kim 	int non_block;
9988df67d77SJung-uk Kim 	int timed_out;
999df8bae1dSRodney W. Grimes 
1000136600feSEd Schouten 	error = devfs_get_cdevpriv((void **)&d);
1001136600feSEd Schouten 	if (error != 0)
1002136600feSEd Schouten 		return (error);
1003136600feSEd Schouten 
1004df8bae1dSRodney W. Grimes 	/*
1005df8bae1dSRodney W. Grimes 	 * Restrict application to use a buffer the same size as
1006df8bae1dSRodney W. Grimes 	 * as kernel buffers.
1007df8bae1dSRodney W. Grimes 	 */
1008df8bae1dSRodney W. Grimes 	if (uio->uio_resid != d->bd_bufsize)
1009df8bae1dSRodney W. Grimes 		return (EINVAL);
1010df8bae1dSRodney W. Grimes 
10118df67d77SJung-uk Kim 	non_block = ((ioflag & O_NONBLOCK) != 0);
10128df67d77SJung-uk Kim 
1013afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
1014e4b3229aSAlexander V. Chernikov 	BPF_PID_REFRESH_CUR(d);
10154d621040SChristian S.J. Peron 	if (d->bd_bufmode != BPF_BUFMODE_BUFFER) {
1016afa85850SAlexander V. Chernikov 		BPFD_UNLOCK(d);
10174d621040SChristian S.J. Peron 		return (EOPNOTSUPP);
10184d621040SChristian S.J. Peron 	}
101981bda851SJohn Polstra 	if (d->bd_state == BPF_WAITING)
102081bda851SJohn Polstra 		callout_stop(&d->bd_callout);
102181bda851SJohn Polstra 	timed_out = (d->bd_state == BPF_TIMED_OUT);
102281bda851SJohn Polstra 	d->bd_state = BPF_IDLE;
1023d013d902SGuy Helmer 	while (d->bd_hbuf_in_use) {
1024d013d902SGuy Helmer 		error = mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
10253b3b91e7SGuy Helmer 		    PRINET | PCATCH, "bd_hbuf", 0);
1026d013d902SGuy Helmer 		if (error != 0) {
1027d013d902SGuy Helmer 			BPFD_UNLOCK(d);
1028d013d902SGuy Helmer 			return (error);
1029d013d902SGuy Helmer 		}
1030d013d902SGuy Helmer 	}
1031df8bae1dSRodney W. Grimes 	/*
1032df8bae1dSRodney W. Grimes 	 * If the hold buffer is empty, then do a timed sleep, which
1033df8bae1dSRodney W. Grimes 	 * ends when the timeout expires or when enough packets
1034df8bae1dSRodney W. Grimes 	 * have arrived to fill the store buffer.
1035df8bae1dSRodney W. Grimes 	 */
1036572bde2aSRobert Watson 	while (d->bd_hbuf == NULL) {
10378df67d77SJung-uk Kim 		if (d->bd_slen != 0) {
1038df8bae1dSRodney W. Grimes 			/*
1039df8bae1dSRodney W. Grimes 			 * A packet(s) either arrived since the previous
1040df8bae1dSRodney W. Grimes 			 * read or arrived while we were asleep.
10418df67d77SJung-uk Kim 			 */
10428df67d77SJung-uk Kim 			if (d->bd_immediate || non_block || timed_out) {
10438df67d77SJung-uk Kim 				/*
10448df67d77SJung-uk Kim 				 * Rotate the buffers and return what's here
10458df67d77SJung-uk Kim 				 * if we are in immediate mode, non-blocking
10468df67d77SJung-uk Kim 				 * flag is set, or this descriptor timed out.
1047df8bae1dSRodney W. Grimes 				 */
1048df8bae1dSRodney W. Grimes 				ROTATE_BUFFERS(d);
1049df8bae1dSRodney W. Grimes 				break;
1050df8bae1dSRodney W. Grimes 			}
10518df67d77SJung-uk Kim 		}
1052de5d9935SRobert Watson 
1053de5d9935SRobert Watson 		/*
1054de5d9935SRobert Watson 		 * No data is available, check to see if the bpf device
1055de5d9935SRobert Watson 		 * is still pointed at a real interface.  If not, return
1056de5d9935SRobert Watson 		 * ENXIO so that the userland process knows to rebind
1057de5d9935SRobert Watson 		 * it before using it again.
1058de5d9935SRobert Watson 		 */
1059de5d9935SRobert Watson 		if (d->bd_bif == NULL) {
1060afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
1061de5d9935SRobert Watson 			return (ENXIO);
1062de5d9935SRobert Watson 		}
1063de5d9935SRobert Watson 
10648df67d77SJung-uk Kim 		if (non_block) {
1065afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
1066fba3cfdeSJohn Polstra 			return (EWOULDBLOCK);
1067fba3cfdeSJohn Polstra 		}
1068afa85850SAlexander V. Chernikov 		error = msleep(d, &d->bd_lock, PRINET | PCATCH,
1069e7bb21b3SJonathan Lemon 		     "bpf", d->bd_rtout);
1070df8bae1dSRodney W. Grimes 		if (error == EINTR || error == ERESTART) {
1071afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
1072df8bae1dSRodney W. Grimes 			return (error);
1073df8bae1dSRodney W. Grimes 		}
1074df8bae1dSRodney W. Grimes 		if (error == EWOULDBLOCK) {
1075df8bae1dSRodney W. Grimes 			/*
1076df8bae1dSRodney W. Grimes 			 * On a timeout, return what's in the buffer,
1077df8bae1dSRodney W. Grimes 			 * which may be nothing.  If there is something
1078df8bae1dSRodney W. Grimes 			 * in the store buffer, we can rotate the buffers.
1079df8bae1dSRodney W. Grimes 			 */
1080df8bae1dSRodney W. Grimes 			if (d->bd_hbuf)
1081df8bae1dSRodney W. Grimes 				/*
1082df8bae1dSRodney W. Grimes 				 * We filled up the buffer in between
1083df8bae1dSRodney W. Grimes 				 * getting the timeout and arriving
1084df8bae1dSRodney W. Grimes 				 * here, so we don't need to rotate.
1085df8bae1dSRodney W. Grimes 				 */
1086df8bae1dSRodney W. Grimes 				break;
1087df8bae1dSRodney W. Grimes 
1088df8bae1dSRodney W. Grimes 			if (d->bd_slen == 0) {
1089afa85850SAlexander V. Chernikov 				BPFD_UNLOCK(d);
1090df8bae1dSRodney W. Grimes 				return (0);
1091df8bae1dSRodney W. Grimes 			}
1092df8bae1dSRodney W. Grimes 			ROTATE_BUFFERS(d);
1093df8bae1dSRodney W. Grimes 			break;
1094df8bae1dSRodney W. Grimes 		}
1095df8bae1dSRodney W. Grimes 	}
1096df8bae1dSRodney W. Grimes 	/*
1097df8bae1dSRodney W. Grimes 	 * At this point, we know we have something in the hold slot.
1098df8bae1dSRodney W. Grimes 	 */
10993b3b91e7SGuy Helmer 	d->bd_hbuf_in_use = 1;
1100afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
1101df8bae1dSRodney W. Grimes 
1102df8bae1dSRodney W. Grimes 	/*
1103df8bae1dSRodney W. Grimes 	 * Move data from hold buffer into user space.
1104df8bae1dSRodney W. Grimes 	 * We know the entire buffer is transferred since
1105df8bae1dSRodney W. Grimes 	 * we checked above that the read buffer is bpf_bufsize bytes.
110631b32e6dSRobert Watson   	 *
11073b3b91e7SGuy Helmer 	 * We do not have to worry about simultaneous reads because
11083b3b91e7SGuy Helmer 	 * we waited for sole access to the hold buffer above.
1109df8bae1dSRodney W. Grimes 	 */
11104d621040SChristian S.J. Peron 	error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio);
1111df8bae1dSRodney W. Grimes 
1112afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
11133b3b91e7SGuy Helmer 	KASSERT(d->bd_hbuf != NULL, ("bpfread: lost bd_hbuf"));
1114df8bae1dSRodney W. Grimes 	d->bd_fbuf = d->bd_hbuf;
1115572bde2aSRobert Watson 	d->bd_hbuf = NULL;
1116df8bae1dSRodney W. Grimes 	d->bd_hlen = 0;
111729f612ecSChristian S.J. Peron 	bpf_buf_reclaimed(d);
11183b3b91e7SGuy Helmer 	d->bd_hbuf_in_use = 0;
11193b3b91e7SGuy Helmer 	wakeup(&d->bd_hbuf_in_use);
1120afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
1121df8bae1dSRodney W. Grimes 
1122df8bae1dSRodney W. Grimes 	return (error);
1123df8bae1dSRodney W. Grimes }
1124df8bae1dSRodney W. Grimes 
1125df8bae1dSRodney W. Grimes /*
1126df8bae1dSRodney W. Grimes  * If there are processes sleeping on this descriptor, wake them up.
1127df8bae1dSRodney W. Grimes  */
1128e7bb21b3SJonathan Lemon static __inline void
112919ba8395SChristian S.J. Peron bpf_wakeup(struct bpf_d *d)
1130df8bae1dSRodney W. Grimes {
1131a3272e3cSChristian S.J. Peron 
1132afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
113381bda851SJohn Polstra 	if (d->bd_state == BPF_WAITING) {
113481bda851SJohn Polstra 		callout_stop(&d->bd_callout);
113581bda851SJohn Polstra 		d->bd_state = BPF_IDLE;
113681bda851SJohn Polstra 	}
1137521f364bSDag-Erling Smørgrav 	wakeup(d);
1138831d27a9SDon Lewis 	if (d->bd_async && d->bd_sig && d->bd_sigio)
1139f1320723SAlfred Perlstein 		pgsigio(&d->bd_sigio, d->bd_sig, 0);
114000a83887SPaul Traina 
1141512824f8SSeigo Tanimura 	selwakeuppri(&d->bd_sel, PRINET);
1142ad3b9257SJohn-Mark Gurney 	KNOTE_LOCKED(&d->bd_sel.si_note, 0);
1143df8bae1dSRodney W. Grimes }
1144df8bae1dSRodney W. Grimes 
114581bda851SJohn Polstra static void
114619ba8395SChristian S.J. Peron bpf_timed_out(void *arg)
114781bda851SJohn Polstra {
114881bda851SJohn Polstra 	struct bpf_d *d = (struct bpf_d *)arg;
114981bda851SJohn Polstra 
1150afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
11519fee1bd1SJung-uk Kim 
1152699281b5SAndrey V. Elsukov 	if (callout_pending(&d->bd_callout) ||
1153699281b5SAndrey V. Elsukov 	    !callout_active(&d->bd_callout))
11549fee1bd1SJung-uk Kim 		return;
115581bda851SJohn Polstra 	if (d->bd_state == BPF_WAITING) {
115681bda851SJohn Polstra 		d->bd_state = BPF_TIMED_OUT;
115781bda851SJohn Polstra 		if (d->bd_slen != 0)
115881bda851SJohn Polstra 			bpf_wakeup(d);
115981bda851SJohn Polstra 	}
116081bda851SJohn Polstra }
116181bda851SJohn Polstra 
116287f6c662SJulian Elischer static int
11634d621040SChristian S.J. Peron bpf_ready(struct bpf_d *d)
11644d621040SChristian S.J. Peron {
11654d621040SChristian S.J. Peron 
1166afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
11674d621040SChristian S.J. Peron 
11684d621040SChristian S.J. Peron 	if (!bpf_canfreebuf(d) && d->bd_hlen != 0)
11694d621040SChristian S.J. Peron 		return (1);
11704d621040SChristian S.J. Peron 	if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
11714d621040SChristian S.J. Peron 	    d->bd_slen != 0)
11724d621040SChristian S.J. Peron 		return (1);
11734d621040SChristian S.J. Peron 	return (0);
11744d621040SChristian S.J. Peron }
11754d621040SChristian S.J. Peron 
11764d621040SChristian S.J. Peron static int
117719ba8395SChristian S.J. Peron bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
1178df8bae1dSRodney W. Grimes {
1179699281b5SAndrey V. Elsukov 	struct route ro;
1180699281b5SAndrey V. Elsukov 	struct sockaddr dst;
1181699281b5SAndrey V. Elsukov 	struct epoch_tracker et;
1182699281b5SAndrey V. Elsukov 	struct bpf_if *bp;
1183136600feSEd Schouten 	struct bpf_d *d;
1184df8bae1dSRodney W. Grimes 	struct ifnet *ifp;
1185560a54e1SJung-uk Kim 	struct mbuf *m, *mc;
1186560a54e1SJung-uk Kim 	int error, hlen;
1187df8bae1dSRodney W. Grimes 
1188136600feSEd Schouten 	error = devfs_get_cdevpriv((void **)&d);
1189136600feSEd Schouten 	if (error != 0)
1190136600feSEd Schouten 		return (error);
1191136600feSEd Schouten 
1192699281b5SAndrey V. Elsukov 	NET_EPOCH_ENTER(et);
1193699281b5SAndrey V. Elsukov 	BPFD_LOCK(d);
1194e4b3229aSAlexander V. Chernikov 	BPF_PID_REFRESH_CUR(d);
1195b2b7ca49SAlexander V. Chernikov 	counter_u64_add(d->bd_wcount, 1);
1196699281b5SAndrey V. Elsukov 	if ((bp = d->bd_bif) == NULL) {
1197699281b5SAndrey V. Elsukov 		error = ENXIO;
1198699281b5SAndrey V. Elsukov 		goto out_locked;
11994d621040SChristian S.J. Peron 	}
1200df8bae1dSRodney W. Grimes 
1201699281b5SAndrey V. Elsukov 	ifp = bp->bif_ifp;
12024d621040SChristian S.J. Peron 	if ((ifp->if_flags & IFF_UP) == 0) {
1203699281b5SAndrey V. Elsukov 		error = ENETDOWN;
1204699281b5SAndrey V. Elsukov 		goto out_locked;
12054d621040SChristian S.J. Peron 	}
12063518d220SSam Leffler 
1207699281b5SAndrey V. Elsukov 	if (uio->uio_resid == 0)
1208699281b5SAndrey V. Elsukov 		goto out_locked;
1209df8bae1dSRodney W. Grimes 
12108240bf1eSRobert Watson 	bzero(&dst, sizeof(dst));
1211d83e603aSChristian S.J. Peron 	m = NULL;
1212d83e603aSChristian S.J. Peron 	hlen = 0;
1213699281b5SAndrey V. Elsukov 
1214699281b5SAndrey V. Elsukov 	/*
1215699281b5SAndrey V. Elsukov 	 * Take extra reference, unlock d and exit from epoch section,
1216699281b5SAndrey V. Elsukov 	 * since bpf_movein() can sleep.
1217699281b5SAndrey V. Elsukov 	 */
1218699281b5SAndrey V. Elsukov 	bpfd_ref(d);
1219699281b5SAndrey V. Elsukov 	NET_EPOCH_EXIT(et);
1220699281b5SAndrey V. Elsukov 	BPFD_UNLOCK(d);
1221699281b5SAndrey V. Elsukov 
1222699281b5SAndrey V. Elsukov 	error = bpf_movein(uio, (int)bp->bif_dlt, ifp,
12234fb3a820SAlexander V. Chernikov 	    &m, &dst, &hlen, d);
1224699281b5SAndrey V. Elsukov 
1225699281b5SAndrey V. Elsukov 	if (error != 0) {
1226b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_wdcount, 1);
1227699281b5SAndrey V. Elsukov 		bpfd_rele(d);
1228df8bae1dSRodney W. Grimes 		return (error);
12294d621040SChristian S.J. Peron 	}
1230699281b5SAndrey V. Elsukov 
1231699281b5SAndrey V. Elsukov 	BPFD_LOCK(d);
1232699281b5SAndrey V. Elsukov 	/*
1233699281b5SAndrey V. Elsukov 	 * Check that descriptor is still attached to the interface.
1234699281b5SAndrey V. Elsukov 	 * This can happen on bpfdetach(). To avoid access to detached
1235699281b5SAndrey V. Elsukov 	 * ifnet, free mbuf and return ENXIO.
1236699281b5SAndrey V. Elsukov 	 */
1237699281b5SAndrey V. Elsukov 	if (d->bd_bif == NULL) {
1238699281b5SAndrey V. Elsukov 		counter_u64_add(d->bd_wdcount, 1);
1239699281b5SAndrey V. Elsukov 		BPFD_UNLOCK(d);
1240699281b5SAndrey V. Elsukov 		bpfd_rele(d);
1241699281b5SAndrey V. Elsukov 		m_freem(m);
1242699281b5SAndrey V. Elsukov 		return (ENXIO);
1243699281b5SAndrey V. Elsukov 	}
1244b2b7ca49SAlexander V. Chernikov 	counter_u64_add(d->bd_wfcount, 1);
1245114ae644SMike Smith 	if (d->bd_hdrcmplt)
1246114ae644SMike Smith 		dst.sa_family = pseudo_AF_HDRCMPLT;
1247114ae644SMike Smith 
1248560a54e1SJung-uk Kim 	if (d->bd_feedback) {
1249eb1b1807SGleb Smirnoff 		mc = m_dup(m, M_NOWAIT);
1250560a54e1SJung-uk Kim 		if (mc != NULL)
1251560a54e1SJung-uk Kim 			mc->m_pkthdr.rcvif = ifp;
12528cd892f7SJung-uk Kim 		/* Set M_PROMISC for outgoing packets to be discarded. */
12538cd892f7SJung-uk Kim 		if (d->bd_direction == BPF_D_INOUT)
12548cd892f7SJung-uk Kim 			m->m_flags |= M_PROMISC;
1255560a54e1SJung-uk Kim 	} else
1256560a54e1SJung-uk Kim 		mc = NULL;
1257560a54e1SJung-uk Kim 
1258560a54e1SJung-uk Kim 	m->m_pkthdr.len -= hlen;
1259560a54e1SJung-uk Kim 	m->m_len -= hlen;
1260560a54e1SJung-uk Kim 	m->m_data += hlen;	/* XXX */
1261560a54e1SJung-uk Kim 
126221ca7b57SMarko Zec 	CURVNET_SET(ifp->if_vnet);
126382f4445dSRobert Watson #ifdef MAC
126430d239bcSRobert Watson 	mac_bpfdesc_create_mbuf(d, m);
1265560a54e1SJung-uk Kim 	if (mc != NULL)
126630d239bcSRobert Watson 		mac_bpfdesc_create_mbuf(d, mc);
126782f4445dSRobert Watson #endif
1268560a54e1SJung-uk Kim 
12694fb3a820SAlexander V. Chernikov 	bzero(&ro, sizeof(ro));
12704fb3a820SAlexander V. Chernikov 	if (hlen != 0) {
12714fb3a820SAlexander V. Chernikov 		ro.ro_prepend = (u_char *)&dst.sa_data;
12724fb3a820SAlexander V. Chernikov 		ro.ro_plen = hlen;
12734fb3a820SAlexander V. Chernikov 		ro.ro_flags = RT_HAS_HEADER;
12744fb3a820SAlexander V. Chernikov 	}
12754fb3a820SAlexander V. Chernikov 
12761e7fe2fbSLuiz Otavio O Souza 	if (d->bd_pcp != 0)
12779ef8cd0bSKristof Provost 		vlan_set_pcp(m, d->bd_pcp);
12781e7fe2fbSLuiz Otavio O Souza 
127982d7bf6bSAndrey V. Elsukov 	/* Avoid possible recursion on BPFD_LOCK(). */
128082d7bf6bSAndrey V. Elsukov 	NET_EPOCH_ENTER(et);
128182d7bf6bSAndrey V. Elsukov 	BPFD_UNLOCK(d);
12824fb3a820SAlexander V. Chernikov 	error = (*ifp->if_output)(ifp, m, &dst, &ro);
12834d621040SChristian S.J. Peron 	if (error)
1284b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_wdcount, 1);
1285560a54e1SJung-uk Kim 
1286560a54e1SJung-uk Kim 	if (mc != NULL) {
12870bf686c1SRobert Watson 		if (error == 0)
1288560a54e1SJung-uk Kim 			(*ifp->if_input)(ifp, mc);
12890bf686c1SRobert Watson 		else
1290560a54e1SJung-uk Kim 			m_freem(mc);
1291560a54e1SJung-uk Kim 	}
129282d7bf6bSAndrey V. Elsukov 	NET_EPOCH_EXIT(et);
129321ca7b57SMarko Zec 	CURVNET_RESTORE();
1294699281b5SAndrey V. Elsukov 	bpfd_rele(d);
1295699281b5SAndrey V. Elsukov 	return (error);
1296560a54e1SJung-uk Kim 
1297699281b5SAndrey V. Elsukov out_locked:
1298699281b5SAndrey V. Elsukov 	counter_u64_add(d->bd_wdcount, 1);
1299699281b5SAndrey V. Elsukov 	NET_EPOCH_EXIT(et);
1300699281b5SAndrey V. Elsukov 	BPFD_UNLOCK(d);
1301df8bae1dSRodney W. Grimes 	return (error);
1302df8bae1dSRodney W. Grimes }
1303df8bae1dSRodney W. Grimes 
1304df8bae1dSRodney W. Grimes /*
1305e82669d9SRobert Watson  * Reset a descriptor by flushing its packet buffer and clearing the receive
1306e82669d9SRobert Watson  * and drop counts.  This is doable for kernel-only buffers, but with
1307e82669d9SRobert Watson  * zero-copy buffers, we can't write to (or rotate) buffers that are
1308e82669d9SRobert Watson  * currently owned by userspace.  It would be nice if we could encapsulate
1309e82669d9SRobert Watson  * this logic in the buffer code rather than here.
1310df8bae1dSRodney W. Grimes  */
1311df8bae1dSRodney W. Grimes static void
131219ba8395SChristian S.J. Peron reset_d(struct bpf_d *d)
1313df8bae1dSRodney W. Grimes {
1314e7bb21b3SJonathan Lemon 
1315afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
1316e82669d9SRobert Watson 
13173b3b91e7SGuy Helmer 	while (d->bd_hbuf_in_use)
13183b3b91e7SGuy Helmer 		mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock, PRINET,
13193b3b91e7SGuy Helmer 		    "bd_hbuf", 0);
1320e82669d9SRobert Watson 	if ((d->bd_hbuf != NULL) &&
1321e82669d9SRobert Watson 	    (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) {
1322df8bae1dSRodney W. Grimes 		/* Free the hold buffer. */
1323df8bae1dSRodney W. Grimes 		d->bd_fbuf = d->bd_hbuf;
1324572bde2aSRobert Watson 		d->bd_hbuf = NULL;
1325e82669d9SRobert Watson 		d->bd_hlen = 0;
132629f612ecSChristian S.J. Peron 		bpf_buf_reclaimed(d);
1327df8bae1dSRodney W. Grimes 	}
1328e82669d9SRobert Watson 	if (bpf_canwritebuf(d))
1329df8bae1dSRodney W. Grimes 		d->bd_slen = 0;
1330b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_rcount);
1331b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_dcount);
1332b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_fcount);
1333b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_wcount);
1334b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_wfcount);
1335b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_wdcount);
1336b2b7ca49SAlexander V. Chernikov 	counter_u64_zero(d->bd_zcopy);
1337df8bae1dSRodney W. Grimes }
1338df8bae1dSRodney W. Grimes 
1339df8bae1dSRodney W. Grimes /*
1340df8bae1dSRodney W. Grimes  *  FIONREAD		Check for read packet available.
1341df8bae1dSRodney W. Grimes  *  BIOCGBLEN		Get buffer len [for read()].
1342f11c3508SDavid Malone  *  BIOCSETF		Set read filter.
1343f11c3508SDavid Malone  *  BIOCSETFNR		Set read filter without resetting descriptor.
1344f11c3508SDavid Malone  *  BIOCSETWF		Set write filter.
1345df8bae1dSRodney W. Grimes  *  BIOCFLUSH		Flush read packet buffer.
1346df8bae1dSRodney W. Grimes  *  BIOCPROMISC		Put interface into promiscuous mode.
1347df8bae1dSRodney W. Grimes  *  BIOCGDLT		Get link layer type.
1348df8bae1dSRodney W. Grimes  *  BIOCGETIF		Get interface name.
1349df8bae1dSRodney W. Grimes  *  BIOCSETIF		Set interface.
1350df8bae1dSRodney W. Grimes  *  BIOCSRTIMEOUT	Set read timeout.
1351df8bae1dSRodney W. Grimes  *  BIOCGRTIMEOUT	Get read timeout.
1352df8bae1dSRodney W. Grimes  *  BIOCGSTATS		Get packet stats.
1353df8bae1dSRodney W. Grimes  *  BIOCIMMEDIATE	Set immediate mode.
1354df8bae1dSRodney W. Grimes  *  BIOCVERSION		Get filter language version.
1355114ae644SMike Smith  *  BIOCGHDRCMPLT	Get "header already complete" flag
1356114ae644SMike Smith  *  BIOCSHDRCMPLT	Set "header already complete" flag
1357560a54e1SJung-uk Kim  *  BIOCGDIRECTION	Get packet direction flag
1358560a54e1SJung-uk Kim  *  BIOCSDIRECTION	Set packet direction flag
1359547d94bdSJung-uk Kim  *  BIOCGTSTAMP		Get time stamp format and resolution.
1360547d94bdSJung-uk Kim  *  BIOCSTSTAMP		Set time stamp format and resolution.
136193e39f0bSChristian S.J. Peron  *  BIOCLOCK		Set "locked" flag
1362560a54e1SJung-uk Kim  *  BIOCFEEDBACK	Set packet feedback mode.
13634d621040SChristian S.J. Peron  *  BIOCSETZBUF		Set current zero-copy buffer locations.
13644d621040SChristian S.J. Peron  *  BIOCGETZMAX		Get maximum zero-copy buffer size.
13654d621040SChristian S.J. Peron  *  BIOCROTZBUF		Force rotation of zero-copy buffer
13664d621040SChristian S.J. Peron  *  BIOCSETBUFMODE	Set buffer mode.
13674d621040SChristian S.J. Peron  *  BIOCGETBUFMODE	Get current buffer mode.
13681e7fe2fbSLuiz Otavio O Souza  *  BIOCSETVLANPCP	Set VLAN PCP tag.
1369df8bae1dSRodney W. Grimes  */
1370df8bae1dSRodney W. Grimes /* ARGSUSED */
137187f6c662SJulian Elischer static	int
137219ba8395SChristian S.J. Peron bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
137319ba8395SChristian S.J. Peron     struct thread *td)
1374df8bae1dSRodney W. Grimes {
1375136600feSEd Schouten 	struct bpf_d *d;
1376136600feSEd Schouten 	int error;
1377136600feSEd Schouten 
1378136600feSEd Schouten 	error = devfs_get_cdevpriv((void **)&d);
1379136600feSEd Schouten 	if (error != 0)
1380136600feSEd Schouten 		return (error);
1381df8bae1dSRodney W. Grimes 
1382b75a24a0SChristian S.J. Peron 	/*
1383b75a24a0SChristian S.J. Peron 	 * Refresh PID associated with this descriptor.
1384b75a24a0SChristian S.J. Peron 	 */
1385afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
1386e4b3229aSAlexander V. Chernikov 	BPF_PID_REFRESH(d, td);
138781bda851SJohn Polstra 	if (d->bd_state == BPF_WAITING)
138881bda851SJohn Polstra 		callout_stop(&d->bd_callout);
138981bda851SJohn Polstra 	d->bd_state = BPF_IDLE;
1390afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
139181bda851SJohn Polstra 
139293e39f0bSChristian S.J. Peron 	if (d->bd_locked == 1) {
139393e39f0bSChristian S.J. Peron 		switch (cmd) {
139493e39f0bSChristian S.J. Peron 		case BIOCGBLEN:
139593e39f0bSChristian S.J. Peron 		case BIOCFLUSH:
139693e39f0bSChristian S.J. Peron 		case BIOCGDLT:
139793e39f0bSChristian S.J. Peron 		case BIOCGDLTLIST:
1398fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
1399fc0a61a4SKonstantin Belousov 		case BIOCGDLTLIST32:
1400fc0a61a4SKonstantin Belousov #endif
140193e39f0bSChristian S.J. Peron 		case BIOCGETIF:
140293e39f0bSChristian S.J. Peron 		case BIOCGRTIMEOUT:
1403b4366092SJustin Hibbits #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1404fc0a61a4SKonstantin Belousov 		case BIOCGRTIMEOUT32:
1405fc0a61a4SKonstantin Belousov #endif
140693e39f0bSChristian S.J. Peron 		case BIOCGSTATS:
140793e39f0bSChristian S.J. Peron 		case BIOCVERSION:
140893e39f0bSChristian S.J. Peron 		case BIOCGRSIG:
140993e39f0bSChristian S.J. Peron 		case BIOCGHDRCMPLT:
1410547d94bdSJung-uk Kim 		case BIOCSTSTAMP:
1411560a54e1SJung-uk Kim 		case BIOCFEEDBACK:
141293e39f0bSChristian S.J. Peron 		case FIONREAD:
141393e39f0bSChristian S.J. Peron 		case BIOCLOCK:
141493e39f0bSChristian S.J. Peron 		case BIOCSRTIMEOUT:
1415b4366092SJustin Hibbits #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1416fc0a61a4SKonstantin Belousov 		case BIOCSRTIMEOUT32:
1417fc0a61a4SKonstantin Belousov #endif
141893e39f0bSChristian S.J. Peron 		case BIOCIMMEDIATE:
141993e39f0bSChristian S.J. Peron 		case TIOCGPGRP:
14204d621040SChristian S.J. Peron 		case BIOCROTZBUF:
142193e39f0bSChristian S.J. Peron 			break;
142293e39f0bSChristian S.J. Peron 		default:
142393e39f0bSChristian S.J. Peron 			return (EPERM);
142493e39f0bSChristian S.J. Peron 		}
142593e39f0bSChristian S.J. Peron 	}
1426fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
1427fc0a61a4SKonstantin Belousov 	/*
1428fc0a61a4SKonstantin Belousov 	 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so
1429fc0a61a4SKonstantin Belousov 	 * that it will get 32-bit packet headers.
1430fc0a61a4SKonstantin Belousov 	 */
1431fc0a61a4SKonstantin Belousov 	switch (cmd) {
1432fc0a61a4SKonstantin Belousov 	case BIOCSETF32:
1433fc0a61a4SKonstantin Belousov 	case BIOCSETFNR32:
1434fc0a61a4SKonstantin Belousov 	case BIOCSETWF32:
1435fc0a61a4SKonstantin Belousov 	case BIOCGDLTLIST32:
1436fc0a61a4SKonstantin Belousov 	case BIOCGRTIMEOUT32:
1437fc0a61a4SKonstantin Belousov 	case BIOCSRTIMEOUT32:
143874549d4bSWojciech Macek 		if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
143997aacec6SAlexander V. Chernikov 			BPFD_LOCK(d);
1440fc0a61a4SKonstantin Belousov 			d->bd_compat32 = 1;
144197aacec6SAlexander V. Chernikov 			BPFD_UNLOCK(d);
1442fc0a61a4SKonstantin Belousov 		}
144374549d4bSWojciech Macek 	}
1444fc0a61a4SKonstantin Belousov #endif
1445fc0a61a4SKonstantin Belousov 
144697021c24SMarko Zec 	CURVNET_SET(TD_TO_VNET(td));
1447df8bae1dSRodney W. Grimes 	switch (cmd) {
1448df8bae1dSRodney W. Grimes 	default:
1449df8bae1dSRodney W. Grimes 		error = EINVAL;
1450df8bae1dSRodney W. Grimes 		break;
1451df8bae1dSRodney W. Grimes 
1452df8bae1dSRodney W. Grimes 	/*
1453df8bae1dSRodney W. Grimes 	 * Check for read packet available.
1454df8bae1dSRodney W. Grimes 	 */
1455df8bae1dSRodney W. Grimes 	case FIONREAD:
1456df8bae1dSRodney W. Grimes 		{
1457df8bae1dSRodney W. Grimes 			int n;
1458df8bae1dSRodney W. Grimes 
1459afa85850SAlexander V. Chernikov 			BPFD_LOCK(d);
1460df8bae1dSRodney W. Grimes 			n = d->bd_slen;
14613b3b91e7SGuy Helmer 			while (d->bd_hbuf_in_use)
14623b3b91e7SGuy Helmer 				mtx_sleep(&d->bd_hbuf_in_use, &d->bd_lock,
14633b3b91e7SGuy Helmer 				    PRINET, "bd_hbuf", 0);
1464df8bae1dSRodney W. Grimes 			if (d->bd_hbuf)
1465df8bae1dSRodney W. Grimes 				n += d->bd_hlen;
1466afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
1467df8bae1dSRodney W. Grimes 
1468df8bae1dSRodney W. Grimes 			*(int *)addr = n;
1469df8bae1dSRodney W. Grimes 			break;
1470df8bae1dSRodney W. Grimes 		}
1471df8bae1dSRodney W. Grimes 
1472df8bae1dSRodney W. Grimes 	/*
1473df8bae1dSRodney W. Grimes 	 * Get buffer len [for read()].
1474df8bae1dSRodney W. Grimes 	 */
1475df8bae1dSRodney W. Grimes 	case BIOCGBLEN:
147697aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1477df8bae1dSRodney W. Grimes 		*(u_int *)addr = d->bd_bufsize;
147897aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1479df8bae1dSRodney W. Grimes 		break;
1480df8bae1dSRodney W. Grimes 
1481df8bae1dSRodney W. Grimes 	/*
1482df8bae1dSRodney W. Grimes 	 * Set buffer length.
1483df8bae1dSRodney W. Grimes 	 */
1484df8bae1dSRodney W. Grimes 	case BIOCSBLEN:
14854d621040SChristian S.J. Peron 		error = bpf_ioctl_sblen(d, (u_int *)addr);
1486df8bae1dSRodney W. Grimes 		break;
1487df8bae1dSRodney W. Grimes 
1488df8bae1dSRodney W. Grimes 	/*
1489df8bae1dSRodney W. Grimes 	 * Set link layer read filter.
1490df8bae1dSRodney W. Grimes 	 */
1491df8bae1dSRodney W. Grimes 	case BIOCSETF:
1492f11c3508SDavid Malone 	case BIOCSETFNR:
149393e39f0bSChristian S.J. Peron 	case BIOCSETWF:
1494fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
1495fc0a61a4SKonstantin Belousov 	case BIOCSETF32:
1496fc0a61a4SKonstantin Belousov 	case BIOCSETFNR32:
1497fc0a61a4SKonstantin Belousov 	case BIOCSETWF32:
1498fc0a61a4SKonstantin Belousov #endif
149993e39f0bSChristian S.J. Peron 		error = bpf_setf(d, (struct bpf_program *)addr, cmd);
1500df8bae1dSRodney W. Grimes 		break;
1501df8bae1dSRodney W. Grimes 
1502df8bae1dSRodney W. Grimes 	/*
1503df8bae1dSRodney W. Grimes 	 * Flush read packet buffer.
1504df8bae1dSRodney W. Grimes 	 */
1505df8bae1dSRodney W. Grimes 	case BIOCFLUSH:
1506afa85850SAlexander V. Chernikov 		BPFD_LOCK(d);
1507df8bae1dSRodney W. Grimes 		reset_d(d);
1508afa85850SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1509df8bae1dSRodney W. Grimes 		break;
1510df8bae1dSRodney W. Grimes 
1511df8bae1dSRodney W. Grimes 	/*
1512df8bae1dSRodney W. Grimes 	 * Put interface into promiscuous mode.
1513df8bae1dSRodney W. Grimes 	 */
1514df8bae1dSRodney W. Grimes 	case BIOCPROMISC:
1515220818acSMark Johnston 		BPF_LOCK();
1516572bde2aSRobert Watson 		if (d->bd_bif == NULL) {
1517df8bae1dSRodney W. Grimes 			/*
1518df8bae1dSRodney W. Grimes 			 * No interface attached yet.
1519df8bae1dSRodney W. Grimes 			 */
1520df8bae1dSRodney W. Grimes 			error = EINVAL;
1521220818acSMark Johnston 		} else if (d->bd_promisc == 0) {
1522df8bae1dSRodney W. Grimes 			error = ifpromisc(d->bd_bif->bif_ifp, 1);
1523df8bae1dSRodney W. Grimes 			if (error == 0)
1524df8bae1dSRodney W. Grimes 				d->bd_promisc = 1;
1525df8bae1dSRodney W. Grimes 		}
1526220818acSMark Johnston 		BPF_UNLOCK();
1527df8bae1dSRodney W. Grimes 		break;
1528df8bae1dSRodney W. Grimes 
1529df8bae1dSRodney W. Grimes 	/*
15308eab61f3SSam Leffler 	 * Get current data link type.
1531df8bae1dSRodney W. Grimes 	 */
1532df8bae1dSRodney W. Grimes 	case BIOCGDLT:
153397aacec6SAlexander V. Chernikov 		BPF_LOCK();
1534572bde2aSRobert Watson 		if (d->bd_bif == NULL)
1535df8bae1dSRodney W. Grimes 			error = EINVAL;
1536df8bae1dSRodney W. Grimes 		else
1537df8bae1dSRodney W. Grimes 			*(u_int *)addr = d->bd_bif->bif_dlt;
153897aacec6SAlexander V. Chernikov 		BPF_UNLOCK();
1539df8bae1dSRodney W. Grimes 		break;
1540df8bae1dSRodney W. Grimes 
1541df8bae1dSRodney W. Grimes 	/*
15428eab61f3SSam Leffler 	 * Get a list of supported data link types.
15438eab61f3SSam Leffler 	 */
1544fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
1545fc0a61a4SKonstantin Belousov 	case BIOCGDLTLIST32:
1546fc0a61a4SKonstantin Belousov 		{
1547fc0a61a4SKonstantin Belousov 			struct bpf_dltlist32 *list32;
1548fc0a61a4SKonstantin Belousov 			struct bpf_dltlist dltlist;
1549fc0a61a4SKonstantin Belousov 
1550fc0a61a4SKonstantin Belousov 			list32 = (struct bpf_dltlist32 *)addr;
1551fc0a61a4SKonstantin Belousov 			dltlist.bfl_len = list32->bfl_len;
1552fc0a61a4SKonstantin Belousov 			dltlist.bfl_list = PTRIN(list32->bfl_list);
155397aacec6SAlexander V. Chernikov 			BPF_LOCK();
1554fc0a61a4SKonstantin Belousov 			if (d->bd_bif == NULL)
1555fc0a61a4SKonstantin Belousov 				error = EINVAL;
1556fc0a61a4SKonstantin Belousov 			else {
1557fc0a61a4SKonstantin Belousov 				error = bpf_getdltlist(d, &dltlist);
1558fc0a61a4SKonstantin Belousov 				if (error == 0)
1559fc0a61a4SKonstantin Belousov 					list32->bfl_len = dltlist.bfl_len;
1560fc0a61a4SKonstantin Belousov 			}
156197aacec6SAlexander V. Chernikov 			BPF_UNLOCK();
1562fc0a61a4SKonstantin Belousov 			break;
1563fc0a61a4SKonstantin Belousov 		}
1564fc0a61a4SKonstantin Belousov #endif
1565fc0a61a4SKonstantin Belousov 
15668eab61f3SSam Leffler 	case BIOCGDLTLIST:
156797aacec6SAlexander V. Chernikov 		BPF_LOCK();
1568572bde2aSRobert Watson 		if (d->bd_bif == NULL)
15698eab61f3SSam Leffler 			error = EINVAL;
15708eab61f3SSam Leffler 		else
15718eab61f3SSam Leffler 			error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
157297aacec6SAlexander V. Chernikov 		BPF_UNLOCK();
15738eab61f3SSam Leffler 		break;
15748eab61f3SSam Leffler 
15758eab61f3SSam Leffler 	/*
15768eab61f3SSam Leffler 	 * Set data link type.
15778eab61f3SSam Leffler 	 */
15788eab61f3SSam Leffler 	case BIOCSDLT:
15796c74ff0eSAlexander V. Chernikov 		BPF_LOCK();
1580572bde2aSRobert Watson 		if (d->bd_bif == NULL)
15818eab61f3SSam Leffler 			error = EINVAL;
15828eab61f3SSam Leffler 		else
15838eab61f3SSam Leffler 			error = bpf_setdlt(d, *(u_int *)addr);
15846c74ff0eSAlexander V. Chernikov 		BPF_UNLOCK();
15858eab61f3SSam Leffler 		break;
15868eab61f3SSam Leffler 
15878eab61f3SSam Leffler 	/*
15881dd0feaaSArchie Cobbs 	 * Get interface name.
1589df8bae1dSRodney W. Grimes 	 */
1590df8bae1dSRodney W. Grimes 	case BIOCGETIF:
159197aacec6SAlexander V. Chernikov 		BPF_LOCK();
1592572bde2aSRobert Watson 		if (d->bd_bif == NULL)
1593df8bae1dSRodney W. Grimes 			error = EINVAL;
15941dd0feaaSArchie Cobbs 		else {
15951dd0feaaSArchie Cobbs 			struct ifnet *const ifp = d->bd_bif->bif_ifp;
15961dd0feaaSArchie Cobbs 			struct ifreq *const ifr = (struct ifreq *)addr;
15971dd0feaaSArchie Cobbs 
15989bf40edeSBrooks Davis 			strlcpy(ifr->ifr_name, ifp->if_xname,
15999bf40edeSBrooks Davis 			    sizeof(ifr->ifr_name));
16001dd0feaaSArchie Cobbs 		}
160197aacec6SAlexander V. Chernikov 		BPF_UNLOCK();
1602df8bae1dSRodney W. Grimes 		break;
1603df8bae1dSRodney W. Grimes 
1604df8bae1dSRodney W. Grimes 	/*
1605df8bae1dSRodney W. Grimes 	 * Set interface.
1606df8bae1dSRodney W. Grimes 	 */
1607df8bae1dSRodney W. Grimes 	case BIOCSETIF:
16084f42daa4SLuiz Otavio O Souza 		{
16094f42daa4SLuiz Otavio O Souza 			int alloc_buf, size;
16104f42daa4SLuiz Otavio O Souza 
16114f42daa4SLuiz Otavio O Souza 			/*
16124f42daa4SLuiz Otavio O Souza 			 * Behavior here depends on the buffering model.  If
16134f42daa4SLuiz Otavio O Souza 			 * we're using kernel memory buffers, then we can
16144f42daa4SLuiz Otavio O Souza 			 * allocate them here.  If we're using zero-copy,
16154f42daa4SLuiz Otavio O Souza 			 * then the user process must have registered buffers
16164f42daa4SLuiz Otavio O Souza 			 * by the time we get here.
16174f42daa4SLuiz Otavio O Souza 			 */
16184f42daa4SLuiz Otavio O Souza 			alloc_buf = 0;
16194f42daa4SLuiz Otavio O Souza 			BPFD_LOCK(d);
16204f42daa4SLuiz Otavio O Souza 			if (d->bd_bufmode == BPF_BUFMODE_BUFFER &&
16214f42daa4SLuiz Otavio O Souza 			    d->bd_sbuf == NULL)
16224f42daa4SLuiz Otavio O Souza 				alloc_buf = 1;
16234f42daa4SLuiz Otavio O Souza 			BPFD_UNLOCK(d);
16244f42daa4SLuiz Otavio O Souza 			if (alloc_buf) {
16254f42daa4SLuiz Otavio O Souza 				size = d->bd_bufsize;
16264f42daa4SLuiz Otavio O Souza 				error = bpf_buffer_ioctl_sblen(d, &size);
16274f42daa4SLuiz Otavio O Souza 				if (error != 0)
16284f42daa4SLuiz Otavio O Souza 					break;
16294f42daa4SLuiz Otavio O Souza 			}
16306c74ff0eSAlexander V. Chernikov 			BPF_LOCK();
1631df8bae1dSRodney W. Grimes 			error = bpf_setif(d, (struct ifreq *)addr);
16326c74ff0eSAlexander V. Chernikov 			BPF_UNLOCK();
1633df8bae1dSRodney W. Grimes 			break;
16344f42daa4SLuiz Otavio O Souza 		}
1635df8bae1dSRodney W. Grimes 
1636df8bae1dSRodney W. Grimes 	/*
1637df8bae1dSRodney W. Grimes 	 * Set read timeout.
1638df8bae1dSRodney W. Grimes 	 */
1639df8bae1dSRodney W. Grimes 	case BIOCSRTIMEOUT:
1640b4366092SJustin Hibbits #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1641fc0a61a4SKonstantin Belousov 	case BIOCSRTIMEOUT32:
1642fc0a61a4SKonstantin Belousov #endif
1643df8bae1dSRodney W. Grimes 		{
1644df8bae1dSRodney W. Grimes 			struct timeval *tv = (struct timeval *)addr;
164533755776SMateusz Guzik #if defined(COMPAT_FREEBSD32)
1646fc0a61a4SKonstantin Belousov 			struct timeval32 *tv32;
1647fc0a61a4SKonstantin Belousov 			struct timeval tv64;
1648fc0a61a4SKonstantin Belousov 
1649fc0a61a4SKonstantin Belousov 			if (cmd == BIOCSRTIMEOUT32) {
1650fc0a61a4SKonstantin Belousov 				tv32 = (struct timeval32 *)addr;
1651fc0a61a4SKonstantin Belousov 				tv = &tv64;
1652fc0a61a4SKonstantin Belousov 				tv->tv_sec = tv32->tv_sec;
1653fc0a61a4SKonstantin Belousov 				tv->tv_usec = tv32->tv_usec;
1654fc0a61a4SKonstantin Belousov 			} else
1655fc0a61a4SKonstantin Belousov #endif
1656fc0a61a4SKonstantin Belousov 				tv = (struct timeval *)addr;
1657df8bae1dSRodney W. Grimes 
1658bdc2cdc5SAlexander Langer 			/*
1659bdc2cdc5SAlexander Langer 			 * Subtract 1 tick from tvtohz() since this isn't
1660bdc2cdc5SAlexander Langer 			 * a one-shot timer.
1661bdc2cdc5SAlexander Langer 			 */
1662bdc2cdc5SAlexander Langer 			if ((error = itimerfix(tv)) == 0)
1663bdc2cdc5SAlexander Langer 				d->bd_rtout = tvtohz(tv) - 1;
1664df8bae1dSRodney W. Grimes 			break;
1665df8bae1dSRodney W. Grimes 		}
1666df8bae1dSRodney W. Grimes 
1667df8bae1dSRodney W. Grimes 	/*
1668df8bae1dSRodney W. Grimes 	 * Get read timeout.
1669df8bae1dSRodney W. Grimes 	 */
1670df8bae1dSRodney W. Grimes 	case BIOCGRTIMEOUT:
1671b4366092SJustin Hibbits #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1672fc0a61a4SKonstantin Belousov 	case BIOCGRTIMEOUT32:
1673fc0a61a4SKonstantin Belousov #endif
1674df8bae1dSRodney W. Grimes 		{
1675fc0a61a4SKonstantin Belousov 			struct timeval *tv;
1676b4366092SJustin Hibbits #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1677fc0a61a4SKonstantin Belousov 			struct timeval32 *tv32;
1678fc0a61a4SKonstantin Belousov 			struct timeval tv64;
1679fc0a61a4SKonstantin Belousov 
1680fc0a61a4SKonstantin Belousov 			if (cmd == BIOCGRTIMEOUT32)
1681fc0a61a4SKonstantin Belousov 				tv = &tv64;
1682fc0a61a4SKonstantin Belousov 			else
1683fc0a61a4SKonstantin Belousov #endif
1684fc0a61a4SKonstantin Belousov 				tv = (struct timeval *)addr;
1685df8bae1dSRodney W. Grimes 
1686bdc2cdc5SAlexander Langer 			tv->tv_sec = d->bd_rtout / hz;
1687bdc2cdc5SAlexander Langer 			tv->tv_usec = (d->bd_rtout % hz) * tick;
1688b4366092SJustin Hibbits #if defined(COMPAT_FREEBSD32) && defined(__amd64__)
1689fc0a61a4SKonstantin Belousov 			if (cmd == BIOCGRTIMEOUT32) {
1690fc0a61a4SKonstantin Belousov 				tv32 = (struct timeval32 *)addr;
1691fc0a61a4SKonstantin Belousov 				tv32->tv_sec = tv->tv_sec;
1692fc0a61a4SKonstantin Belousov 				tv32->tv_usec = tv->tv_usec;
1693fc0a61a4SKonstantin Belousov 			}
1694fc0a61a4SKonstantin Belousov #endif
1695fc0a61a4SKonstantin Belousov 
1696df8bae1dSRodney W. Grimes 			break;
1697df8bae1dSRodney W. Grimes 		}
1698df8bae1dSRodney W. Grimes 
1699df8bae1dSRodney W. Grimes 	/*
1700df8bae1dSRodney W. Grimes 	 * Get packet stats.
1701df8bae1dSRodney W. Grimes 	 */
1702df8bae1dSRodney W. Grimes 	case BIOCGSTATS:
1703df8bae1dSRodney W. Grimes 		{
1704df8bae1dSRodney W. Grimes 			struct bpf_stat *bs = (struct bpf_stat *)addr;
1705df8bae1dSRodney W. Grimes 
17064d621040SChristian S.J. Peron 			/* XXXCSJP overflow */
1707b2b7ca49SAlexander V. Chernikov 			bs->bs_recv = (u_int)counter_u64_fetch(d->bd_rcount);
1708b2b7ca49SAlexander V. Chernikov 			bs->bs_drop = (u_int)counter_u64_fetch(d->bd_dcount);
1709df8bae1dSRodney W. Grimes 			break;
1710df8bae1dSRodney W. Grimes 		}
1711df8bae1dSRodney W. Grimes 
1712df8bae1dSRodney W. Grimes 	/*
1713df8bae1dSRodney W. Grimes 	 * Set immediate mode.
1714df8bae1dSRodney W. Grimes 	 */
1715df8bae1dSRodney W. Grimes 	case BIOCIMMEDIATE:
171697aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1717df8bae1dSRodney W. Grimes 		d->bd_immediate = *(u_int *)addr;
171897aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1719df8bae1dSRodney W. Grimes 		break;
1720df8bae1dSRodney W. Grimes 
1721df8bae1dSRodney W. Grimes 	case BIOCVERSION:
1722df8bae1dSRodney W. Grimes 		{
1723df8bae1dSRodney W. Grimes 			struct bpf_version *bv = (struct bpf_version *)addr;
1724df8bae1dSRodney W. Grimes 
1725df8bae1dSRodney W. Grimes 			bv->bv_major = BPF_MAJOR_VERSION;
1726df8bae1dSRodney W. Grimes 			bv->bv_minor = BPF_MINOR_VERSION;
1727df8bae1dSRodney W. Grimes 			break;
1728df8bae1dSRodney W. Grimes 		}
172900a83887SPaul Traina 
1730114ae644SMike Smith 	/*
1731114ae644SMike Smith 	 * Get "header already complete" flag
1732114ae644SMike Smith 	 */
1733114ae644SMike Smith 	case BIOCGHDRCMPLT:
173497aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1735114ae644SMike Smith 		*(u_int *)addr = d->bd_hdrcmplt;
173697aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1737114ae644SMike Smith 		break;
1738114ae644SMike Smith 
1739114ae644SMike Smith 	/*
1740114ae644SMike Smith 	 * Set "header already complete" flag
1741114ae644SMike Smith 	 */
1742114ae644SMike Smith 	case BIOCSHDRCMPLT:
174397aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1744114ae644SMike Smith 		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
174597aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1746114ae644SMike Smith 		break;
1747114ae644SMike Smith 
17488ed3828cSRobert Watson 	/*
1749560a54e1SJung-uk Kim 	 * Get packet direction flag
17508ed3828cSRobert Watson 	 */
1751560a54e1SJung-uk Kim 	case BIOCGDIRECTION:
175297aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1753560a54e1SJung-uk Kim 		*(u_int *)addr = d->bd_direction;
175497aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
17558ed3828cSRobert Watson 		break;
17568ed3828cSRobert Watson 
17578ed3828cSRobert Watson 	/*
1758560a54e1SJung-uk Kim 	 * Set packet direction flag
17598ed3828cSRobert Watson 	 */
1760560a54e1SJung-uk Kim 	case BIOCSDIRECTION:
1761560a54e1SJung-uk Kim 		{
1762560a54e1SJung-uk Kim 			u_int	direction;
1763560a54e1SJung-uk Kim 
1764560a54e1SJung-uk Kim 			direction = *(u_int *)addr;
1765560a54e1SJung-uk Kim 			switch (direction) {
1766560a54e1SJung-uk Kim 			case BPF_D_IN:
1767560a54e1SJung-uk Kim 			case BPF_D_INOUT:
1768560a54e1SJung-uk Kim 			case BPF_D_OUT:
176997aacec6SAlexander V. Chernikov 				BPFD_LOCK(d);
1770560a54e1SJung-uk Kim 				d->bd_direction = direction;
177197aacec6SAlexander V. Chernikov 				BPFD_UNLOCK(d);
1772560a54e1SJung-uk Kim 				break;
1773560a54e1SJung-uk Kim 			default:
1774560a54e1SJung-uk Kim 				error = EINVAL;
1775560a54e1SJung-uk Kim 			}
1776560a54e1SJung-uk Kim 		}
1777560a54e1SJung-uk Kim 		break;
1778560a54e1SJung-uk Kim 
1779547d94bdSJung-uk Kim 	/*
1780d0d7bcdfSJung-uk Kim 	 * Get packet timestamp format and resolution.
1781547d94bdSJung-uk Kim 	 */
1782547d94bdSJung-uk Kim 	case BIOCGTSTAMP:
178397aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1784547d94bdSJung-uk Kim 		*(u_int *)addr = d->bd_tstamp;
178597aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1786547d94bdSJung-uk Kim 		break;
1787547d94bdSJung-uk Kim 
1788547d94bdSJung-uk Kim 	/*
1789547d94bdSJung-uk Kim 	 * Set packet timestamp format and resolution.
1790547d94bdSJung-uk Kim 	 */
1791547d94bdSJung-uk Kim 	case BIOCSTSTAMP:
1792547d94bdSJung-uk Kim 		{
1793547d94bdSJung-uk Kim 			u_int	func;
1794547d94bdSJung-uk Kim 
1795547d94bdSJung-uk Kim 			func = *(u_int *)addr;
1796547d94bdSJung-uk Kim 			if (BPF_T_VALID(func))
1797547d94bdSJung-uk Kim 				d->bd_tstamp = func;
1798547d94bdSJung-uk Kim 			else
1799547d94bdSJung-uk Kim 				error = EINVAL;
1800547d94bdSJung-uk Kim 		}
1801547d94bdSJung-uk Kim 		break;
1802547d94bdSJung-uk Kim 
1803560a54e1SJung-uk Kim 	case BIOCFEEDBACK:
180497aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1805560a54e1SJung-uk Kim 		d->bd_feedback = *(u_int *)addr;
180697aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1807560a54e1SJung-uk Kim 		break;
1808560a54e1SJung-uk Kim 
1809560a54e1SJung-uk Kim 	case BIOCLOCK:
181097aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
1811560a54e1SJung-uk Kim 		d->bd_locked = 1;
181297aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
18138ed3828cSRobert Watson 		break;
18148ed3828cSRobert Watson 
181500a83887SPaul Traina 	case FIONBIO:		/* Non-blocking I/O */
181600a83887SPaul Traina 		break;
181700a83887SPaul Traina 
181800a83887SPaul Traina 	case FIOASYNC:		/* Send signal on receive packets */
181997aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
182000a83887SPaul Traina 		d->bd_async = *(int *)addr;
182197aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
182200a83887SPaul Traina 		break;
182300a83887SPaul Traina 
1824831d27a9SDon Lewis 	case FIOSETOWN:
182597aacec6SAlexander V. Chernikov 		/*
182697aacec6SAlexander V. Chernikov 		 * XXX: Add some sort of locking here?
182797aacec6SAlexander V. Chernikov 		 * fsetown() can sleep.
182897aacec6SAlexander V. Chernikov 		 */
1829831d27a9SDon Lewis 		error = fsetown(*(int *)addr, &d->bd_sigio);
183000a83887SPaul Traina 		break;
183100a83887SPaul Traina 
1832831d27a9SDon Lewis 	case FIOGETOWN:
183397aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
183491e97a82SDon Lewis 		*(int *)addr = fgetown(&d->bd_sigio);
183597aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
1836831d27a9SDon Lewis 		break;
1837831d27a9SDon Lewis 
1838831d27a9SDon Lewis 	/* This is deprecated, FIOSETOWN should be used instead. */
1839831d27a9SDon Lewis 	case TIOCSPGRP:
1840831d27a9SDon Lewis 		error = fsetown(-(*(int *)addr), &d->bd_sigio);
1841831d27a9SDon Lewis 		break;
1842831d27a9SDon Lewis 
1843831d27a9SDon Lewis 	/* This is deprecated, FIOGETOWN should be used instead. */
184400a83887SPaul Traina 	case TIOCGPGRP:
184591e97a82SDon Lewis 		*(int *)addr = -fgetown(&d->bd_sigio);
184600a83887SPaul Traina 		break;
184700a83887SPaul Traina 
184800a83887SPaul Traina 	case BIOCSRSIG:		/* Set receive signal */
184900a83887SPaul Traina 		{
185000a83887SPaul Traina 			u_int sig;
185100a83887SPaul Traina 
185200a83887SPaul Traina 			sig = *(u_int *)addr;
185300a83887SPaul Traina 
185400a83887SPaul Traina 			if (sig >= NSIG)
185500a83887SPaul Traina 				error = EINVAL;
185697aacec6SAlexander V. Chernikov 			else {
185797aacec6SAlexander V. Chernikov 				BPFD_LOCK(d);
185800a83887SPaul Traina 				d->bd_sig = sig;
185997aacec6SAlexander V. Chernikov 				BPFD_UNLOCK(d);
186097aacec6SAlexander V. Chernikov 			}
186100a83887SPaul Traina 			break;
186200a83887SPaul Traina 		}
186300a83887SPaul Traina 	case BIOCGRSIG:
186497aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
186500a83887SPaul Traina 		*(u_int *)addr = d->bd_sig;
186697aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
186700a83887SPaul Traina 		break;
18684d621040SChristian S.J. Peron 
18694d621040SChristian S.J. Peron 	case BIOCGETBUFMODE:
187097aacec6SAlexander V. Chernikov 		BPFD_LOCK(d);
18714d621040SChristian S.J. Peron 		*(u_int *)addr = d->bd_bufmode;
187297aacec6SAlexander V. Chernikov 		BPFD_UNLOCK(d);
18734d621040SChristian S.J. Peron 		break;
18744d621040SChristian S.J. Peron 
18754d621040SChristian S.J. Peron 	case BIOCSETBUFMODE:
18764d621040SChristian S.J. Peron 		/*
18774d621040SChristian S.J. Peron 		 * Allow the buffering mode to be changed as long as we
18784d621040SChristian S.J. Peron 		 * haven't yet committed to a particular mode.  Our
18794d621040SChristian S.J. Peron 		 * definition of commitment, for now, is whether or not a
18804d621040SChristian S.J. Peron 		 * buffer has been allocated or an interface attached, since
18814d621040SChristian S.J. Peron 		 * that's the point where things get tricky.
18824d621040SChristian S.J. Peron 		 */
18834d621040SChristian S.J. Peron 		switch (*(u_int *)addr) {
18844d621040SChristian S.J. Peron 		case BPF_BUFMODE_BUFFER:
18854d621040SChristian S.J. Peron 			break;
18864d621040SChristian S.J. Peron 
18874d621040SChristian S.J. Peron 		case BPF_BUFMODE_ZBUF:
18884d621040SChristian S.J. Peron 			if (bpf_zerocopy_enable)
18894d621040SChristian S.J. Peron 				break;
18904d621040SChristian S.J. Peron 			/* FALLSTHROUGH */
18914d621040SChristian S.J. Peron 
18924d621040SChristian S.J. Peron 		default:
18931b610a74SBjoern A. Zeeb 			CURVNET_RESTORE();
18944d621040SChristian S.J. Peron 			return (EINVAL);
18954d621040SChristian S.J. Peron 		}
18964d621040SChristian S.J. Peron 
1897afa85850SAlexander V. Chernikov 		BPFD_LOCK(d);
18984d621040SChristian S.J. Peron 		if (d->bd_sbuf != NULL || d->bd_hbuf != NULL ||
18994d621040SChristian S.J. Peron 		    d->bd_fbuf != NULL || d->bd_bif != NULL) {
1900afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
19011b610a74SBjoern A. Zeeb 			CURVNET_RESTORE();
19024d621040SChristian S.J. Peron 			return (EBUSY);
19034d621040SChristian S.J. Peron 		}
19044d621040SChristian S.J. Peron 		d->bd_bufmode = *(u_int *)addr;
1905afa85850SAlexander V. Chernikov 		BPFD_UNLOCK(d);
19064d621040SChristian S.J. Peron 		break;
19074d621040SChristian S.J. Peron 
19084d621040SChristian S.J. Peron 	case BIOCGETZMAX:
19091b610a74SBjoern A. Zeeb 		error = bpf_ioctl_getzmax(td, d, (size_t *)addr);
19101b610a74SBjoern A. Zeeb 		break;
19114d621040SChristian S.J. Peron 
19124d621040SChristian S.J. Peron 	case BIOCSETZBUF:
19131b610a74SBjoern A. Zeeb 		error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr);
19141b610a74SBjoern A. Zeeb 		break;
19154d621040SChristian S.J. Peron 
19164d621040SChristian S.J. Peron 	case BIOCROTZBUF:
19171b610a74SBjoern A. Zeeb 		error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr);
19181b610a74SBjoern A. Zeeb 		break;
19191e7fe2fbSLuiz Otavio O Souza 
19201e7fe2fbSLuiz Otavio O Souza 	case BIOCSETVLANPCP:
19211e7fe2fbSLuiz Otavio O Souza 		{
19221e7fe2fbSLuiz Otavio O Souza 			u_int pcp;
19231e7fe2fbSLuiz Otavio O Souza 
19241e7fe2fbSLuiz Otavio O Souza 			pcp = *(u_int *)addr;
19251e7fe2fbSLuiz Otavio O Souza 			if (pcp > BPF_PRIO_MAX || pcp < 0) {
19261e7fe2fbSLuiz Otavio O Souza 				error = EINVAL;
19271e7fe2fbSLuiz Otavio O Souza 				break;
19281e7fe2fbSLuiz Otavio O Souza 			}
19291e7fe2fbSLuiz Otavio O Souza 			d->bd_pcp = pcp;
19301e7fe2fbSLuiz Otavio O Souza 			break;
19311e7fe2fbSLuiz Otavio O Souza 		}
1932df8bae1dSRodney W. Grimes 	}
193397021c24SMarko Zec 	CURVNET_RESTORE();
1934df8bae1dSRodney W. Grimes 	return (error);
1935df8bae1dSRodney W. Grimes }
1936df8bae1dSRodney W. Grimes 
1937df8bae1dSRodney W. Grimes /*
1938df8bae1dSRodney W. Grimes  * Set d's packet filter program to fp. If this file already has a filter,
1939df8bae1dSRodney W. Grimes  * free it and replace it. Returns EINVAL for bogus requests.
1940784292f8SAlexander V. Chernikov  *
1941699281b5SAndrey V. Elsukov  * Note we use global lock here to serialize bpf_setf() and bpf_setif()
1942699281b5SAndrey V. Elsukov  * calls.
1943df8bae1dSRodney W. Grimes  */
1944f708ef1bSPoul-Henning Kamp static int
194519ba8395SChristian S.J. Peron bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1946df8bae1dSRodney W. Grimes {
19479b7d4a7fSJung-uk Kim #ifdef COMPAT_FREEBSD32
19489b7d4a7fSJung-uk Kim 	struct bpf_program fp_swab;
19499b7d4a7fSJung-uk Kim 	struct bpf_program32 *fp32;
19509b7d4a7fSJung-uk Kim #endif
1951699281b5SAndrey V. Elsukov 	struct bpf_program_buffer *fcode;
1952699281b5SAndrey V. Elsukov 	struct bpf_insn *filter;
1953293c06a1SRuslan Ermilov #ifdef BPF_JITTER
1954699281b5SAndrey V. Elsukov 	bpf_jit_filter *jfunc;
1955ae275efcSJung-uk Kim #endif
19568b04b48aSJung-uk Kim 	size_t size;
19578b04b48aSJung-uk Kim 	u_int flen;
1958699281b5SAndrey V. Elsukov 	bool track_event;
1959df8bae1dSRodney W. Grimes 
19609b7d4a7fSJung-uk Kim #ifdef COMPAT_FREEBSD32
19616f731135SJung-uk Kim 	switch (cmd) {
19626f731135SJung-uk Kim 	case BIOCSETF32:
19636f731135SJung-uk Kim 	case BIOCSETWF32:
19646f731135SJung-uk Kim 	case BIOCSETFNR32:
1965fc0a61a4SKonstantin Belousov 		fp32 = (struct bpf_program32 *)fp;
1966fc0a61a4SKonstantin Belousov 		fp_swab.bf_len = fp32->bf_len;
1967699281b5SAndrey V. Elsukov 		fp_swab.bf_insns =
1968699281b5SAndrey V. Elsukov 		    (struct bpf_insn *)(uintptr_t)fp32->bf_insns;
1969fc0a61a4SKonstantin Belousov 		fp = &fp_swab;
19706f731135SJung-uk Kim 		switch (cmd) {
19716f731135SJung-uk Kim 		case BIOCSETF32:
19726f731135SJung-uk Kim 			cmd = BIOCSETF;
19736f731135SJung-uk Kim 			break;
19746f731135SJung-uk Kim 		case BIOCSETWF32:
1975fc0a61a4SKonstantin Belousov 			cmd = BIOCSETWF;
19766f731135SJung-uk Kim 			break;
19776f731135SJung-uk Kim 		}
19786f731135SJung-uk Kim 		break;
1979fc0a61a4SKonstantin Belousov 	}
1980fc0a61a4SKonstantin Belousov #endif
19818b04b48aSJung-uk Kim 
1982699281b5SAndrey V. Elsukov 	filter = NULL;
19838b04b48aSJung-uk Kim #ifdef BPF_JITTER
1984699281b5SAndrey V. Elsukov 	jfunc = NULL;
19858b04b48aSJung-uk Kim #endif
19866c74ff0eSAlexander V. Chernikov 	/*
19876c74ff0eSAlexander V. Chernikov 	 * Check new filter validness before acquiring any locks.
19886c74ff0eSAlexander V. Chernikov 	 * Allocate memory for new filter, if needed.
19896c74ff0eSAlexander V. Chernikov 	 */
19906c74ff0eSAlexander V. Chernikov 	flen = fp->bf_len;
19919b7d4a7fSJung-uk Kim 	if (flen > bpf_maxinsns || (fp->bf_insns == NULL && flen != 0))
19926c74ff0eSAlexander V. Chernikov 		return (EINVAL);
19936c74ff0eSAlexander V. Chernikov 	size = flen * sizeof(*fp->bf_insns);
1994a86227d1SAlexander V. Chernikov 	if (size > 0) {
19959b7d4a7fSJung-uk Kim 		/* We're setting up new filter. Copy and check actual data. */
1996699281b5SAndrey V. Elsukov 		fcode = bpf_program_buffer_alloc(size, M_WAITOK);
1997699281b5SAndrey V. Elsukov 		filter = (struct bpf_insn *)fcode->buffer;
1998699281b5SAndrey V. Elsukov 		if (copyin(fp->bf_insns, filter, size) != 0 ||
1999699281b5SAndrey V. Elsukov 		    !bpf_validate(filter, flen)) {
2000a86227d1SAlexander V. Chernikov 			free(fcode, M_BPF);
2001a86227d1SAlexander V. Chernikov 			return (EINVAL);
2002a86227d1SAlexander V. Chernikov 		}
2003c7b0200eSAlexander V. Chernikov #ifdef BPF_JITTER
200416a227c7SJonathan T. Looney 		if (cmd != BIOCSETWF) {
200516a227c7SJonathan T. Looney 			/*
200616a227c7SJonathan T. Looney 			 * Filter is copied inside fcode and is
200716a227c7SJonathan T. Looney 			 * perfectly valid.
200816a227c7SJonathan T. Looney 			 */
2009699281b5SAndrey V. Elsukov 			jfunc = bpf_jitter(filter, flen);
201016a227c7SJonathan T. Looney 		}
2011c7b0200eSAlexander V. Chernikov #endif
20128b04b48aSJung-uk Kim 	}
2013c7b0200eSAlexander V. Chernikov 
2014699281b5SAndrey V. Elsukov 	track_event = false;
2015699281b5SAndrey V. Elsukov 	fcode = NULL;
20166c74ff0eSAlexander V. Chernikov 
2017699281b5SAndrey V. Elsukov 	BPF_LOCK();
2018afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
2019699281b5SAndrey V. Elsukov 	/* Set up new filter. */
20208b04b48aSJung-uk Kim 	if (cmd == BIOCSETWF) {
2021699281b5SAndrey V. Elsukov 		if (d->bd_wfilter != NULL) {
2022699281b5SAndrey V. Elsukov 			fcode = __containerof((void *)d->bd_wfilter,
2023699281b5SAndrey V. Elsukov 			    struct bpf_program_buffer, buffer);
2024293c06a1SRuslan Ermilov #ifdef BPF_JITTER
2025699281b5SAndrey V. Elsukov 			fcode->func = NULL;
2026699281b5SAndrey V. Elsukov #endif
2027699281b5SAndrey V. Elsukov 		}
2028699281b5SAndrey V. Elsukov 		d->bd_wfilter = filter;
2029699281b5SAndrey V. Elsukov 	} else {
2030699281b5SAndrey V. Elsukov 		if (d->bd_rfilter != NULL) {
2031699281b5SAndrey V. Elsukov 			fcode = __containerof((void *)d->bd_rfilter,
2032699281b5SAndrey V. Elsukov 			    struct bpf_program_buffer, buffer);
2033699281b5SAndrey V. Elsukov #ifdef BPF_JITTER
2034699281b5SAndrey V. Elsukov 			fcode->func = d->bd_bfilter;
2035699281b5SAndrey V. Elsukov #endif
2036699281b5SAndrey V. Elsukov 		}
2037699281b5SAndrey V. Elsukov 		d->bd_rfilter = filter;
2038699281b5SAndrey V. Elsukov #ifdef BPF_JITTER
2039c7b0200eSAlexander V. Chernikov 		d->bd_bfilter = jfunc;
2040ae275efcSJung-uk Kim #endif
2041f11c3508SDavid Malone 		if (cmd == BIOCSETF)
2042df8bae1dSRodney W. Grimes 			reset_d(d);
204351ec1eb7SAlexander V. Chernikov 
2044699281b5SAndrey V. Elsukov 		if (bpf_check_upgrade(cmd, d, filter, flen) != 0) {
2045699281b5SAndrey V. Elsukov 			/*
2046699281b5SAndrey V. Elsukov 			 * Filter can be set several times without
2047699281b5SAndrey V. Elsukov 			 * specifying interface. In this case just mark d
2048699281b5SAndrey V. Elsukov 			 * as reader.
2049699281b5SAndrey V. Elsukov 			 */
2050699281b5SAndrey V. Elsukov 			d->bd_writer = 0;
2051699281b5SAndrey V. Elsukov 			if (d->bd_bif != NULL) {
2052699281b5SAndrey V. Elsukov 				/*
2053699281b5SAndrey V. Elsukov 				 * Remove descriptor from writers-only list
2054699281b5SAndrey V. Elsukov 				 * and add it to active readers list.
2055699281b5SAndrey V. Elsukov 				 */
2056699281b5SAndrey V. Elsukov 				CK_LIST_REMOVE(d, bd_next);
2057699281b5SAndrey V. Elsukov 				CK_LIST_INSERT_HEAD(&d->bd_bif->bif_dlist,
2058699281b5SAndrey V. Elsukov 				    d, bd_next);
2059699281b5SAndrey V. Elsukov 				CTR2(KTR_NET,
2060699281b5SAndrey V. Elsukov 				    "%s: upgrade required by pid %d",
2061699281b5SAndrey V. Elsukov 				    __func__, d->bd_pid);
2062699281b5SAndrey V. Elsukov 				track_event = true;
2063699281b5SAndrey V. Elsukov 			}
2064699281b5SAndrey V. Elsukov 		}
20658b04b48aSJung-uk Kim 	}
2066afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
2067df8bae1dSRodney W. Grimes 
2068699281b5SAndrey V. Elsukov 	if (fcode != NULL)
20692a4bd982SGleb Smirnoff 		NET_EPOCH_CALL(bpf_program_buffer_free, &fcode->epoch_ctx);
2070699281b5SAndrey V. Elsukov 
2071699281b5SAndrey V. Elsukov 	if (track_event)
2072699281b5SAndrey V. Elsukov 		EVENTHANDLER_INVOKE(bpf_track,
2073699281b5SAndrey V. Elsukov 		    d->bd_bif->bif_ifp, d->bd_bif->bif_dlt, 1);
207451ec1eb7SAlexander V. Chernikov 
20756c74ff0eSAlexander V. Chernikov 	BPF_UNLOCK();
2076df8bae1dSRodney W. Grimes 	return (0);
2077df8bae1dSRodney W. Grimes }
2078df8bae1dSRodney W. Grimes 
2079df8bae1dSRodney W. Grimes /*
2080df8bae1dSRodney W. Grimes  * Detach a file from its current interface (if attached at all) and attach
2081df8bae1dSRodney W. Grimes  * to the interface indicated by the name stored in ifr.
2082df8bae1dSRodney W. Grimes  * Return an errno or 0.
2083df8bae1dSRodney W. Grimes  */
2084df8bae1dSRodney W. Grimes static int
208519ba8395SChristian S.J. Peron bpf_setif(struct bpf_d *d, struct ifreq *ifr)
2086df8bae1dSRodney W. Grimes {
2087df8bae1dSRodney W. Grimes 	struct bpf_if *bp;
20889b44ff22SGarrett Wollman 	struct ifnet *theywant;
2089df8bae1dSRodney W. Grimes 
20906c74ff0eSAlexander V. Chernikov 	BPF_LOCK_ASSERT();
20916c74ff0eSAlexander V. Chernikov 
20929b44ff22SGarrett Wollman 	theywant = ifunit(ifr->ifr_name);
209316d878ccSChristian S.J. Peron 	if (theywant == NULL || theywant->if_bpf == NULL)
209416d878ccSChristian S.J. Peron 		return (ENXIO);
20959b44ff22SGarrett Wollman 
209616d878ccSChristian S.J. Peron 	bp = theywant->if_bpf;
2097df8bae1dSRodney W. Grimes 	/*
20984f42daa4SLuiz Otavio O Souza 	 * At this point, we expect the buffer is already allocated.  If not,
20994f42daa4SLuiz Otavio O Souza 	 * return an error.
2100df8bae1dSRodney W. Grimes 	 */
21014d621040SChristian S.J. Peron 	switch (d->bd_bufmode) {
21024d621040SChristian S.J. Peron 	case BPF_BUFMODE_BUFFER:
21034d621040SChristian S.J. Peron 	case BPF_BUFMODE_ZBUF:
21044d621040SChristian S.J. Peron 		if (d->bd_sbuf == NULL)
21054d621040SChristian S.J. Peron 			return (EINVAL);
21064d621040SChristian S.J. Peron 		break;
21074d621040SChristian S.J. Peron 
21084d621040SChristian S.J. Peron 	default:
21094d621040SChristian S.J. Peron 		panic("bpf_setif: bufmode %d", d->bd_bufmode);
21104d621040SChristian S.J. Peron 	}
21116c74ff0eSAlexander V. Chernikov 	if (bp != d->bd_bif)
2112df8bae1dSRodney W. Grimes 		bpf_attachd(d, bp);
2113699281b5SAndrey V. Elsukov 	else {
2114afa85850SAlexander V. Chernikov 		BPFD_LOCK(d);
2115df8bae1dSRodney W. Grimes 		reset_d(d);
2116afa85850SAlexander V. Chernikov 		BPFD_UNLOCK(d);
2117699281b5SAndrey V. Elsukov 	}
2118df8bae1dSRodney W. Grimes 	return (0);
2119df8bae1dSRodney W. Grimes }
2120df8bae1dSRodney W. Grimes 
2121df8bae1dSRodney W. Grimes /*
2122243ac7d8SPeter Wemm  * Support for select() and poll() system calls
2123df8bae1dSRodney W. Grimes  *
2124df8bae1dSRodney W. Grimes  * Return true iff the specific operation will not block indefinitely.
2125df8bae1dSRodney W. Grimes  * Otherwise, return false but make a note that a selwakeup() must be done.
2126df8bae1dSRodney W. Grimes  */
212737c84183SPoul-Henning Kamp static int
212819ba8395SChristian S.J. Peron bpfpoll(struct cdev *dev, int events, struct thread *td)
2129df8bae1dSRodney W. Grimes {
2130e7bb21b3SJonathan Lemon 	struct bpf_d *d;
21310832fc64SGarance A Drosehn 	int revents;
2132df8bae1dSRodney W. Grimes 
2133136600feSEd Schouten 	if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL)
2134136600feSEd Schouten 		return (events &
2135136600feSEd Schouten 		    (POLLHUP | POLLIN | POLLRDNORM | POLLOUT | POLLWRNORM));
2136de5d9935SRobert Watson 
2137b75a24a0SChristian S.J. Peron 	/*
2138b75a24a0SChristian S.J. Peron 	 * Refresh PID associated with this descriptor.
2139b75a24a0SChristian S.J. Peron 	 */
21400832fc64SGarance A Drosehn 	revents = events & (POLLOUT | POLLWRNORM);
2141afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
2142e4b3229aSAlexander V. Chernikov 	BPF_PID_REFRESH(d, td);
214375c13541SPoul-Henning Kamp 	if (events & (POLLIN | POLLRDNORM)) {
214495aab9ccSJohn-Mark Gurney 		if (bpf_ready(d))
2145243ac7d8SPeter Wemm 			revents |= events & (POLLIN | POLLRDNORM);
214681bda851SJohn Polstra 		else {
2147ed01445dSJohn Baldwin 			selrecord(td, &d->bd_sel);
214881bda851SJohn Polstra 			/* Start the read timeout if necessary. */
214981bda851SJohn Polstra 			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
215081bda851SJohn Polstra 				callout_reset(&d->bd_callout, d->bd_rtout,
215181bda851SJohn Polstra 				    bpf_timed_out, d);
215281bda851SJohn Polstra 				d->bd_state = BPF_WAITING;
215381bda851SJohn Polstra 			}
215481bda851SJohn Polstra 		}
215575c13541SPoul-Henning Kamp 	}
2156afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
2157243ac7d8SPeter Wemm 	return (revents);
2158df8bae1dSRodney W. Grimes }
2159df8bae1dSRodney W. Grimes 
2160df8bae1dSRodney W. Grimes /*
216195aab9ccSJohn-Mark Gurney  * Support for kevent() system call.  Register EVFILT_READ filters and
216295aab9ccSJohn-Mark Gurney  * reject all others.
216395aab9ccSJohn-Mark Gurney  */
216495aab9ccSJohn-Mark Gurney int
216519ba8395SChristian S.J. Peron bpfkqfilter(struct cdev *dev, struct knote *kn)
216695aab9ccSJohn-Mark Gurney {
2167136600feSEd Schouten 	struct bpf_d *d;
216895aab9ccSJohn-Mark Gurney 
2169ded77e02SHartmut Brandt 	if (devfs_get_cdevpriv((void **)&d) != 0)
217095aab9ccSJohn-Mark Gurney 		return (1);
217195aab9ccSJohn-Mark Gurney 
2172ded77e02SHartmut Brandt 	switch (kn->kn_filter) {
2173ded77e02SHartmut Brandt 	case EVFILT_READ:
2174ded77e02SHartmut Brandt 		kn->kn_fop = &bpfread_filtops;
2175ded77e02SHartmut Brandt 		break;
2176ded77e02SHartmut Brandt 
2177ded77e02SHartmut Brandt 	case EVFILT_WRITE:
2178ded77e02SHartmut Brandt 		kn->kn_fop = &bpfwrite_filtops;
2179ded77e02SHartmut Brandt 		break;
2180ded77e02SHartmut Brandt 
2181ded77e02SHartmut Brandt 	default:
2182ded77e02SHartmut Brandt 		return (1);
2183ded77e02SHartmut Brandt 	}
2184ded77e02SHartmut Brandt 
2185b75a24a0SChristian S.J. Peron 	/*
2186b75a24a0SChristian S.J. Peron 	 * Refresh PID associated with this descriptor.
2187b75a24a0SChristian S.J. Peron 	 */
2188afa85850SAlexander V. Chernikov 	BPFD_LOCK(d);
2189e4b3229aSAlexander V. Chernikov 	BPF_PID_REFRESH_CUR(d);
219095aab9ccSJohn-Mark Gurney 	kn->kn_hook = d;
21914b19419eSChristian S.J. Peron 	knlist_add(&d->bd_sel.si_note, kn, 1);
2192afa85850SAlexander V. Chernikov 	BPFD_UNLOCK(d);
219395aab9ccSJohn-Mark Gurney 
219495aab9ccSJohn-Mark Gurney 	return (0);
219595aab9ccSJohn-Mark Gurney }
219695aab9ccSJohn-Mark Gurney 
219795aab9ccSJohn-Mark Gurney static void
219819ba8395SChristian S.J. Peron filt_bpfdetach(struct knote *kn)
219995aab9ccSJohn-Mark Gurney {
220095aab9ccSJohn-Mark Gurney 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
220195aab9ccSJohn-Mark Gurney 
2202ad3b9257SJohn-Mark Gurney 	knlist_remove(&d->bd_sel.si_note, kn, 0);
220395aab9ccSJohn-Mark Gurney }
220495aab9ccSJohn-Mark Gurney 
220595aab9ccSJohn-Mark Gurney static int
220619ba8395SChristian S.J. Peron filt_bpfread(struct knote *kn, long hint)
220795aab9ccSJohn-Mark Gurney {
220895aab9ccSJohn-Mark Gurney 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
220995aab9ccSJohn-Mark Gurney 	int ready;
221095aab9ccSJohn-Mark Gurney 
2211afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
221295aab9ccSJohn-Mark Gurney 	ready = bpf_ready(d);
221395aab9ccSJohn-Mark Gurney 	if (ready) {
221495aab9ccSJohn-Mark Gurney 		kn->kn_data = d->bd_slen;
221592242172SLuiz Otavio O Souza 		/*
221692242172SLuiz Otavio O Souza 		 * Ignore the hold buffer if it is being copied to user space.
221792242172SLuiz Otavio O Souza 		 */
221892242172SLuiz Otavio O Souza 		if (!d->bd_hbuf_in_use && d->bd_hbuf)
221995aab9ccSJohn-Mark Gurney 			kn->kn_data += d->bd_hlen;
22205d7af3a1SJung-uk Kim 	} else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
222195aab9ccSJohn-Mark Gurney 		callout_reset(&d->bd_callout, d->bd_rtout,
222295aab9ccSJohn-Mark Gurney 		    bpf_timed_out, d);
222395aab9ccSJohn-Mark Gurney 		d->bd_state = BPF_WAITING;
222495aab9ccSJohn-Mark Gurney 	}
222595aab9ccSJohn-Mark Gurney 
222695aab9ccSJohn-Mark Gurney 	return (ready);
222795aab9ccSJohn-Mark Gurney }
222895aab9ccSJohn-Mark Gurney 
2229ded77e02SHartmut Brandt static int
2230ded77e02SHartmut Brandt filt_bpfwrite(struct knote *kn, long hint)
2231ded77e02SHartmut Brandt {
2232ded77e02SHartmut Brandt 	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2233426682b0SMark Johnston 
2234ded77e02SHartmut Brandt 	BPFD_LOCK_ASSERT(d);
2235ded77e02SHartmut Brandt 
2236426682b0SMark Johnston 	if (d->bd_bif == NULL) {
2237426682b0SMark Johnston 		kn->kn_data = 0;
2238426682b0SMark Johnston 		return (0);
2239426682b0SMark Johnston 	} else {
2240ded77e02SHartmut Brandt 		kn->kn_data = d->bd_bif->bif_ifp->if_mtu;
2241ded77e02SHartmut Brandt 		return (1);
2242ded77e02SHartmut Brandt 	}
2243426682b0SMark Johnston }
2244ded77e02SHartmut Brandt 
2245253a3814SLawrence Stewart #define	BPF_TSTAMP_NONE		0
2246253a3814SLawrence Stewart #define	BPF_TSTAMP_FAST		1
2247253a3814SLawrence Stewart #define	BPF_TSTAMP_NORMAL	2
2248253a3814SLawrence Stewart #define	BPF_TSTAMP_EXTERN	3
2249253a3814SLawrence Stewart 
2250253a3814SLawrence Stewart static int
2251253a3814SLawrence Stewart bpf_ts_quality(int tstype)
2252253a3814SLawrence Stewart {
2253253a3814SLawrence Stewart 
2254253a3814SLawrence Stewart 	if (tstype == BPF_T_NONE)
2255253a3814SLawrence Stewart 		return (BPF_TSTAMP_NONE);
2256253a3814SLawrence Stewart 	if ((tstype & BPF_T_FAST) != 0)
2257253a3814SLawrence Stewart 		return (BPF_TSTAMP_FAST);
2258253a3814SLawrence Stewart 
2259253a3814SLawrence Stewart 	return (BPF_TSTAMP_NORMAL);
2260253a3814SLawrence Stewart }
2261253a3814SLawrence Stewart 
2262253a3814SLawrence Stewart static int
2263253a3814SLawrence Stewart bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
2264253a3814SLawrence Stewart {
226556cdab33SJung-uk Kim 	struct timespec ts;
2266253a3814SLawrence Stewart 	struct m_tag *tag;
2267253a3814SLawrence Stewart 	int quality;
2268253a3814SLawrence Stewart 
2269253a3814SLawrence Stewart 	quality = bpf_ts_quality(tstype);
2270253a3814SLawrence Stewart 	if (quality == BPF_TSTAMP_NONE)
2271253a3814SLawrence Stewart 		return (quality);
2272253a3814SLawrence Stewart 
2273253a3814SLawrence Stewart 	if (m != NULL) {
227456cdab33SJung-uk Kim 		if ((m->m_flags & (M_PKTHDR | M_TSTMP)) == (M_PKTHDR | M_TSTMP)) {
227556cdab33SJung-uk Kim 			mbuf_tstmp2timespec(m, &ts);
227656cdab33SJung-uk Kim 			timespec2bintime(&ts, bt);
227756cdab33SJung-uk Kim 			return (BPF_TSTAMP_EXTERN);
227856cdab33SJung-uk Kim 		}
2279253a3814SLawrence Stewart 		tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL);
2280253a3814SLawrence Stewart 		if (tag != NULL) {
2281253a3814SLawrence Stewart 			*bt = *(struct bintime *)(tag + 1);
2282253a3814SLawrence Stewart 			return (BPF_TSTAMP_EXTERN);
2283253a3814SLawrence Stewart 		}
2284253a3814SLawrence Stewart 	}
2285253a3814SLawrence Stewart 	if (quality == BPF_TSTAMP_NORMAL)
2286253a3814SLawrence Stewart 		binuptime(bt);
2287253a3814SLawrence Stewart 	else
2288253a3814SLawrence Stewart 		getbinuptime(bt);
2289253a3814SLawrence Stewart 
2290253a3814SLawrence Stewart 	return (quality);
2291253a3814SLawrence Stewart }
2292253a3814SLawrence Stewart 
229395aab9ccSJohn-Mark Gurney /*
2294df8bae1dSRodney W. Grimes  * Incoming linkage from device drivers.  Process the packet pkt, of length
2295df8bae1dSRodney W. Grimes  * pktlen, which is stored in a contiguous buffer.  The packet is parsed
2296df8bae1dSRodney W. Grimes  * by each process' filter, and if accepted, stashed into the corresponding
2297df8bae1dSRodney W. Grimes  * buffer.
2298df8bae1dSRodney W. Grimes  */
2299df8bae1dSRodney W. Grimes void
230019ba8395SChristian S.J. Peron bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2301df8bae1dSRodney W. Grimes {
2302699281b5SAndrey V. Elsukov 	struct epoch_tracker et;
2303547d94bdSJung-uk Kim 	struct bintime bt;
23048994a245SDag-Erling Smørgrav 	struct bpf_d *d;
2305a36599ccSJung-uk Kim #ifdef BPF_JITTER
2306a36599ccSJung-uk Kim 	bpf_jit_filter *bf;
2307a36599ccSJung-uk Kim #endif
2308253a3814SLawrence Stewart 	u_int slen;
2309253a3814SLawrence Stewart 	int gottime;
2310e7bb21b3SJonathan Lemon 
2311253a3814SLawrence Stewart 	gottime = BPF_TSTAMP_NONE;
2312699281b5SAndrey V. Elsukov 	NET_EPOCH_ENTER(et);
2313699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2314b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_rcount, 1);
2315a05cf8c6SChristian S.J. Peron 		/*
2316699281b5SAndrey V. Elsukov 		 * NB: We dont call BPF_CHECK_DIRECTION() here since there
2317699281b5SAndrey V. Elsukov 		 * is no way for the caller to indiciate to us whether this
2318699281b5SAndrey V. Elsukov 		 * packet is inbound or outbound. In the bpf_mtap() routines,
2319699281b5SAndrey V. Elsukov 		 * we use the interface pointers on the mbuf to figure it out.
2320a05cf8c6SChristian S.J. Peron 		 */
2321ae275efcSJung-uk Kim #ifdef BPF_JITTER
2322a36599ccSJung-uk Kim 		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
2323a36599ccSJung-uk Kim 		if (bf != NULL)
2324a36599ccSJung-uk Kim 			slen = (*(bf->func))(pkt, pktlen, pktlen);
2325ae275efcSJung-uk Kim 		else
2326ae275efcSJung-uk Kim #endif
232793e39f0bSChristian S.J. Peron 		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
2328ec272d87SRobert Watson 		if (slen != 0) {
2329e4b3229aSAlexander V. Chernikov 			/*
2330e4b3229aSAlexander V. Chernikov 			 * Filter matches. Let's to acquire write lock.
2331e4b3229aSAlexander V. Chernikov 			 */
2332afa85850SAlexander V. Chernikov 			BPFD_LOCK(d);
2333b2b7ca49SAlexander V. Chernikov 			counter_u64_add(d->bd_fcount, 1);
2334253a3814SLawrence Stewart 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2335699281b5SAndrey V. Elsukov 				gottime = bpf_gettime(&bt, d->bd_tstamp,
2336699281b5SAndrey V. Elsukov 				    NULL);
2337ec272d87SRobert Watson #ifdef MAC
233830d239bcSRobert Watson 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2339ec272d87SRobert Watson #endif
23404d621040SChristian S.J. Peron 				catchpacket(d, pkt, pktlen, slen,
2341547d94bdSJung-uk Kim 				    bpf_append_bytes, &bt);
2342afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
2343ec272d87SRobert Watson 		}
2344df8bae1dSRodney W. Grimes 	}
2345699281b5SAndrey V. Elsukov 	NET_EPOCH_EXIT(et);
2346df8bae1dSRodney W. Grimes }
2347df8bae1dSRodney W. Grimes 
2348950cc1f4SJustin Hibbits void
2349950cc1f4SJustin Hibbits bpf_tap_if(if_t ifp, u_char *pkt, u_int pktlen)
2350950cc1f4SJustin Hibbits {
2351950cc1f4SJustin Hibbits 	if (bpf_peers_present(ifp->if_bpf))
2352950cc1f4SJustin Hibbits 		bpf_tap(ifp->if_bpf, pkt, pktlen);
2353950cc1f4SJustin Hibbits }
2354950cc1f4SJustin Hibbits 
2355f81a2a49SJung-uk Kim #define	BPF_CHECK_DIRECTION(d, r, i)				\
2356f81a2a49SJung-uk Kim 	    (((d)->bd_direction == BPF_D_IN && (r) != (i)) ||	\
2357f81a2a49SJung-uk Kim 	    ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
2358560a54e1SJung-uk Kim 
2359df8bae1dSRodney W. Grimes /*
2360df8bae1dSRodney W. Grimes  * Incoming linkage from device drivers, when packet is in an mbuf chain.
2361e4b3229aSAlexander V. Chernikov  * Locking model is explained in bpf_tap().
2362df8bae1dSRodney W. Grimes  */
2363df8bae1dSRodney W. Grimes void
236419ba8395SChristian S.J. Peron bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2365df8bae1dSRodney W. Grimes {
2366699281b5SAndrey V. Elsukov 	struct epoch_tracker et;
2367547d94bdSJung-uk Kim 	struct bintime bt;
2368df8bae1dSRodney W. Grimes 	struct bpf_d *d;
2369a36599ccSJung-uk Kim #ifdef BPF_JITTER
2370a36599ccSJung-uk Kim 	bpf_jit_filter *bf;
2371a36599ccSJung-uk Kim #endif
2372253a3814SLawrence Stewart 	u_int pktlen, slen;
2373253a3814SLawrence Stewart 	int gottime;
237491433904SDavid Malone 
23758cd892f7SJung-uk Kim 	/* Skip outgoing duplicate packets. */
2376fb3bc596SJohn Baldwin 	if ((m->m_flags & M_PROMISC) != 0 && m_rcvif(m) == NULL) {
23778cd892f7SJung-uk Kim 		m->m_flags &= ~M_PROMISC;
23788cd892f7SJung-uk Kim 		return;
23798cd892f7SJung-uk Kim 	}
23808cd892f7SJung-uk Kim 
2381f0e2422bSPoul-Henning Kamp 	pktlen = m_length(m, NULL);
2382253a3814SLawrence Stewart 	gottime = BPF_TSTAMP_NONE;
2383e4b3229aSAlexander V. Chernikov 
2384699281b5SAndrey V. Elsukov 	NET_EPOCH_ENTER(et);
2385699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2386fb3bc596SJohn Baldwin 		if (BPF_CHECK_DIRECTION(d, m_rcvif(m), bp->bif_ifp))
23878ed3828cSRobert Watson 			continue;
2388b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_rcount, 1);
2389ae275efcSJung-uk Kim #ifdef BPF_JITTER
2390a36599ccSJung-uk Kim 		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
2391ae275efcSJung-uk Kim 		/* XXX We cannot handle multiple mbufs. */
2392a36599ccSJung-uk Kim 		if (bf != NULL && m->m_next == NULL)
2393699281b5SAndrey V. Elsukov 			slen = (*(bf->func))(mtod(m, u_char *), pktlen,
2394699281b5SAndrey V. Elsukov 			    pktlen);
2395ae275efcSJung-uk Kim 		else
2396ae275efcSJung-uk Kim #endif
239793e39f0bSChristian S.J. Peron 		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
23984ddfb531SChristian S.J. Peron 		if (slen != 0) {
2399afa85850SAlexander V. Chernikov 			BPFD_LOCK(d);
2400e4b3229aSAlexander V. Chernikov 
2401b2b7ca49SAlexander V. Chernikov 			counter_u64_add(d->bd_fcount, 1);
2402253a3814SLawrence Stewart 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2403253a3814SLawrence Stewart 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
24040c7fb534SRobert Watson #ifdef MAC
240530d239bcSRobert Watson 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
24060c7fb534SRobert Watson #endif
24070c7fb534SRobert Watson 				catchpacket(d, (u_char *)m, pktlen, slen,
2408547d94bdSJung-uk Kim 				    bpf_append_mbuf, &bt);
2409afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
24104ddfb531SChristian S.J. Peron 		}
2411df8bae1dSRodney W. Grimes 	}
2412699281b5SAndrey V. Elsukov 	NET_EPOCH_EXIT(et);
2413df8bae1dSRodney W. Grimes }
2414df8bae1dSRodney W. Grimes 
2415950cc1f4SJustin Hibbits void
2416950cc1f4SJustin Hibbits bpf_mtap_if(if_t ifp, struct mbuf *m)
2417950cc1f4SJustin Hibbits {
2418950cc1f4SJustin Hibbits 	if (bpf_peers_present(ifp->if_bpf)) {
2419950cc1f4SJustin Hibbits 		M_ASSERTVALID(m);
2420950cc1f4SJustin Hibbits 		bpf_mtap(ifp->if_bpf, m);
2421950cc1f4SJustin Hibbits 	}
2422950cc1f4SJustin Hibbits }
2423950cc1f4SJustin Hibbits 
2424df8bae1dSRodney W. Grimes /*
2425437ffe18SSam Leffler  * Incoming linkage from device drivers, when packet is in
2426437ffe18SSam Leffler  * an mbuf chain and to be prepended by a contiguous header.
2427437ffe18SSam Leffler  */
2428437ffe18SSam Leffler void
242919ba8395SChristian S.J. Peron bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
2430437ffe18SSam Leffler {
2431699281b5SAndrey V. Elsukov 	struct epoch_tracker et;
2432547d94bdSJung-uk Kim 	struct bintime bt;
2433437ffe18SSam Leffler 	struct mbuf mb;
2434437ffe18SSam Leffler 	struct bpf_d *d;
2435253a3814SLawrence Stewart 	u_int pktlen, slen;
2436253a3814SLawrence Stewart 	int gottime;
243791433904SDavid Malone 
24388cd892f7SJung-uk Kim 	/* Skip outgoing duplicate packets. */
24398cd892f7SJung-uk Kim 	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
24408cd892f7SJung-uk Kim 		m->m_flags &= ~M_PROMISC;
24418cd892f7SJung-uk Kim 		return;
24428cd892f7SJung-uk Kim 	}
24438cd892f7SJung-uk Kim 
2444437ffe18SSam Leffler 	pktlen = m_length(m, NULL);
2445437ffe18SSam Leffler 	/*
2446437ffe18SSam Leffler 	 * Craft on-stack mbuf suitable for passing to bpf_filter.
2447437ffe18SSam Leffler 	 * Note that we cut corners here; we only setup what's
2448437ffe18SSam Leffler 	 * absolutely needed--this mbuf should never go anywhere else.
2449437ffe18SSam Leffler 	 */
245082334850SJohn Baldwin 	mb.m_flags = 0;
2451437ffe18SSam Leffler 	mb.m_next = m;
2452437ffe18SSam Leffler 	mb.m_data = data;
2453437ffe18SSam Leffler 	mb.m_len = dlen;
2454437ffe18SSam Leffler 	pktlen += dlen;
2455437ffe18SSam Leffler 
2456253a3814SLawrence Stewart 	gottime = BPF_TSTAMP_NONE;
2457e4b3229aSAlexander V. Chernikov 
2458699281b5SAndrey V. Elsukov 	NET_EPOCH_ENTER(et);
2459699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
2460f81a2a49SJung-uk Kim 		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
2461437ffe18SSam Leffler 			continue;
2462b2b7ca49SAlexander V. Chernikov 		counter_u64_add(d->bd_rcount, 1);
246393e39f0bSChristian S.J. Peron 		slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
24644ddfb531SChristian S.J. Peron 		if (slen != 0) {
2465afa85850SAlexander V. Chernikov 			BPFD_LOCK(d);
2466e4b3229aSAlexander V. Chernikov 
2467b2b7ca49SAlexander V. Chernikov 			counter_u64_add(d->bd_fcount, 1);
2468253a3814SLawrence Stewart 			if (gottime < bpf_ts_quality(d->bd_tstamp))
2469253a3814SLawrence Stewart 				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
2470437ffe18SSam Leffler #ifdef MAC
247130d239bcSRobert Watson 			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
2472437ffe18SSam Leffler #endif
2473437ffe18SSam Leffler 				catchpacket(d, (u_char *)&mb, pktlen, slen,
2474547d94bdSJung-uk Kim 				    bpf_append_mbuf, &bt);
2475afa85850SAlexander V. Chernikov 			BPFD_UNLOCK(d);
24764ddfb531SChristian S.J. Peron 		}
2477437ffe18SSam Leffler 	}
2478699281b5SAndrey V. Elsukov 	NET_EPOCH_EXIT(et);
2479437ffe18SSam Leffler }
2480437ffe18SSam Leffler 
2481950cc1f4SJustin Hibbits void
2482950cc1f4SJustin Hibbits bpf_mtap2_if(if_t ifp, void *data, u_int dlen, struct mbuf *m)
2483950cc1f4SJustin Hibbits {
2484950cc1f4SJustin Hibbits 	if (bpf_peers_present(ifp->if_bpf)) {
2485950cc1f4SJustin Hibbits 		M_ASSERTVALID(m);
2486950cc1f4SJustin Hibbits 		bpf_mtap2(ifp->if_bpf, data, dlen, m);
2487950cc1f4SJustin Hibbits 	}
2488950cc1f4SJustin Hibbits }
2489950cc1f4SJustin Hibbits 
2490560a54e1SJung-uk Kim #undef	BPF_CHECK_DIRECTION
2491253a3814SLawrence Stewart #undef	BPF_TSTAMP_NONE
2492253a3814SLawrence Stewart #undef	BPF_TSTAMP_FAST
2493253a3814SLawrence Stewart #undef	BPF_TSTAMP_NORMAL
2494253a3814SLawrence Stewart #undef	BPF_TSTAMP_EXTERN
2495253a3814SLawrence Stewart 
2496547d94bdSJung-uk Kim static int
2497547d94bdSJung-uk Kim bpf_hdrlen(struct bpf_d *d)
2498547d94bdSJung-uk Kim {
2499547d94bdSJung-uk Kim 	int hdrlen;
2500547d94bdSJung-uk Kim 
2501547d94bdSJung-uk Kim 	hdrlen = d->bd_bif->bif_hdrlen;
2502547d94bdSJung-uk Kim #ifndef BURN_BRIDGES
2503547d94bdSJung-uk Kim 	if (d->bd_tstamp == BPF_T_NONE ||
2504547d94bdSJung-uk Kim 	    BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME)
2505547d94bdSJung-uk Kim #ifdef COMPAT_FREEBSD32
2506547d94bdSJung-uk Kim 		if (d->bd_compat32)
2507547d94bdSJung-uk Kim 			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32);
2508547d94bdSJung-uk Kim 		else
2509547d94bdSJung-uk Kim #endif
2510547d94bdSJung-uk Kim 			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr);
2511547d94bdSJung-uk Kim 	else
2512547d94bdSJung-uk Kim #endif
2513547d94bdSJung-uk Kim 		hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr);
2514547d94bdSJung-uk Kim #ifdef COMPAT_FREEBSD32
2515547d94bdSJung-uk Kim 	if (d->bd_compat32)
2516547d94bdSJung-uk Kim 		hdrlen = BPF_WORDALIGN32(hdrlen);
2517547d94bdSJung-uk Kim 	else
2518547d94bdSJung-uk Kim #endif
2519547d94bdSJung-uk Kim 		hdrlen = BPF_WORDALIGN(hdrlen);
2520547d94bdSJung-uk Kim 
2521547d94bdSJung-uk Kim 	return (hdrlen - d->bd_bif->bif_hdrlen);
2522547d94bdSJung-uk Kim }
2523547d94bdSJung-uk Kim 
2524547d94bdSJung-uk Kim static void
2525547d94bdSJung-uk Kim bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
2526547d94bdSJung-uk Kim {
2527584b675eSKonstantin Belousov 	struct bintime bt2, boottimebin;
2528547d94bdSJung-uk Kim 	struct timeval tsm;
2529547d94bdSJung-uk Kim 	struct timespec tsn;
2530547d94bdSJung-uk Kim 
2531253a3814SLawrence Stewart 	if ((tstype & BPF_T_MONOTONIC) == 0) {
2532253a3814SLawrence Stewart 		bt2 = *bt;
2533584b675eSKonstantin Belousov 		getboottimebin(&boottimebin);
2534253a3814SLawrence Stewart 		bintime_add(&bt2, &boottimebin);
2535253a3814SLawrence Stewart 		bt = &bt2;
2536253a3814SLawrence Stewart 	}
2537547d94bdSJung-uk Kim 	switch (BPF_T_FORMAT(tstype)) {
2538547d94bdSJung-uk Kim 	case BPF_T_MICROTIME:
2539547d94bdSJung-uk Kim 		bintime2timeval(bt, &tsm);
2540547d94bdSJung-uk Kim 		ts->bt_sec = tsm.tv_sec;
2541547d94bdSJung-uk Kim 		ts->bt_frac = tsm.tv_usec;
2542547d94bdSJung-uk Kim 		break;
2543547d94bdSJung-uk Kim 	case BPF_T_NANOTIME:
2544547d94bdSJung-uk Kim 		bintime2timespec(bt, &tsn);
2545547d94bdSJung-uk Kim 		ts->bt_sec = tsn.tv_sec;
2546547d94bdSJung-uk Kim 		ts->bt_frac = tsn.tv_nsec;
2547547d94bdSJung-uk Kim 		break;
2548547d94bdSJung-uk Kim 	case BPF_T_BINTIME:
2549547d94bdSJung-uk Kim 		ts->bt_sec = bt->sec;
2550547d94bdSJung-uk Kim 		ts->bt_frac = bt->frac;
2551547d94bdSJung-uk Kim 		break;
2552547d94bdSJung-uk Kim 	}
2553547d94bdSJung-uk Kim }
2554547d94bdSJung-uk Kim 
2555437ffe18SSam Leffler /*
2556df8bae1dSRodney W. Grimes  * Move the packet data from interface memory (pkt) into the
25579e610888SDag-Erling Smørgrav  * store buffer.  "cpfn" is the routine called to do the actual data
2558df8bae1dSRodney W. Grimes  * transfer.  bcopy is passed in to copy contiguous chunks, while
25594d621040SChristian S.J. Peron  * bpf_append_mbuf is passed in to copy mbuf chains.  In the latter case,
2560df8bae1dSRodney W. Grimes  * pkt is really an mbuf.
2561df8bae1dSRodney W. Grimes  */
2562df8bae1dSRodney W. Grimes static void
256319ba8395SChristian S.J. Peron catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
25644d621040SChristian S.J. Peron     void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
2565547d94bdSJung-uk Kim     struct bintime *bt)
2566df8bae1dSRodney W. Grimes {
256760b4ad4bSMark Johnston 	static char zeroes[BPF_ALIGNMENT];
2568547d94bdSJung-uk Kim 	struct bpf_xhdr hdr;
2569547d94bdSJung-uk Kim #ifndef BURN_BRIDGES
2570547d94bdSJung-uk Kim 	struct bpf_hdr hdr_old;
2571fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
2572547d94bdSJung-uk Kim 	struct bpf_hdr32 hdr32_old;
2573fc0a61a4SKonstantin Belousov #endif
2574547d94bdSJung-uk Kim #endif
257560b4ad4bSMark Johnston 	int caplen, curlen, hdrlen, pad, totlen;
25767819da79SJohn-Mark Gurney 	int do_wakeup = 0;
2577547d94bdSJung-uk Kim 	int do_timestamp;
2578547d94bdSJung-uk Kim 	int tstype;
25799e610888SDag-Erling Smørgrav 
2580afa85850SAlexander V. Chernikov 	BPFD_LOCK_ASSERT(d);
2581de253273SAndrey V. Elsukov 	if (d->bd_bif == NULL) {
2582de253273SAndrey V. Elsukov 		/* Descriptor was detached in concurrent thread */
2583de253273SAndrey V. Elsukov 		counter_u64_add(d->bd_dcount, 1);
2584de253273SAndrey V. Elsukov 		return;
2585de253273SAndrey V. Elsukov 	}
25864d621040SChristian S.J. Peron 
25874d621040SChristian S.J. Peron 	/*
25884d621040SChristian S.J. Peron 	 * Detect whether user space has released a buffer back to us, and if
25894d621040SChristian S.J. Peron 	 * so, move it from being a hold buffer to a free buffer.  This may
25904d621040SChristian S.J. Peron 	 * not be the best place to do it (for example, we might only want to
25914d621040SChristian S.J. Peron 	 * run this check if we need the space), but for now it's a reliable
25924d621040SChristian S.J. Peron 	 * spot to do it.
25934d621040SChristian S.J. Peron 	 */
2594fa0c2b34SRobert Watson 	if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) {
25954d621040SChristian S.J. Peron 		d->bd_fbuf = d->bd_hbuf;
25964d621040SChristian S.J. Peron 		d->bd_hbuf = NULL;
25974d621040SChristian S.J. Peron 		d->bd_hlen = 0;
259829f612ecSChristian S.J. Peron 		bpf_buf_reclaimed(d);
25994d621040SChristian S.J. Peron 	}
26004d621040SChristian S.J. Peron 
2601df8bae1dSRodney W. Grimes 	/*
2602df8bae1dSRodney W. Grimes 	 * Figure out how many bytes to move.  If the packet is
2603df8bae1dSRodney W. Grimes 	 * greater or equal to the snapshot length, transfer that
2604df8bae1dSRodney W. Grimes 	 * much.  Otherwise, transfer the whole packet (unless
2605df8bae1dSRodney W. Grimes 	 * we hit the buffer size limit).
2606df8bae1dSRodney W. Grimes 	 */
2607547d94bdSJung-uk Kim 	hdrlen = bpf_hdrlen(d);
2608df8bae1dSRodney W. Grimes 	totlen = hdrlen + min(snaplen, pktlen);
2609df8bae1dSRodney W. Grimes 	if (totlen > d->bd_bufsize)
2610df8bae1dSRodney W. Grimes 		totlen = d->bd_bufsize;
2611df8bae1dSRodney W. Grimes 
2612df8bae1dSRodney W. Grimes 	/*
2613df8bae1dSRodney W. Grimes 	 * Round up the end of the previous packet to the next longword.
2614a7a91e65SRobert Watson 	 *
2615a7a91e65SRobert Watson 	 * Drop the packet if there's no room and no hope of room
2616a7a91e65SRobert Watson 	 * If the packet would overflow the storage buffer or the storage
2617a7a91e65SRobert Watson 	 * buffer is considered immutable by the buffer model, try to rotate
2618a7a91e65SRobert Watson 	 * the buffer and wakeup pending processes.
2619df8bae1dSRodney W. Grimes 	 */
2620fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
2621fc0a61a4SKonstantin Belousov 	if (d->bd_compat32)
2622fc0a61a4SKonstantin Belousov 		curlen = BPF_WORDALIGN32(d->bd_slen);
2623fc0a61a4SKonstantin Belousov 	else
2624fc0a61a4SKonstantin Belousov #endif
2625df8bae1dSRodney W. Grimes 		curlen = BPF_WORDALIGN(d->bd_slen);
2626a7a91e65SRobert Watson 	if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) {
2627572bde2aSRobert Watson 		if (d->bd_fbuf == NULL) {
2628df8bae1dSRodney W. Grimes 			/*
2629a7a91e65SRobert Watson 			 * There's no room in the store buffer, and no
2630a7a91e65SRobert Watson 			 * prospect of room, so drop the packet.  Notify the
2631a7a91e65SRobert Watson 			 * buffer model.
2632df8bae1dSRodney W. Grimes 			 */
2633a7a91e65SRobert Watson 			bpf_buffull(d);
2634b2b7ca49SAlexander V. Chernikov 			counter_u64_add(d->bd_dcount, 1);
2635df8bae1dSRodney W. Grimes 			return;
2636df8bae1dSRodney W. Grimes 		}
263798fa5d85SLuiz Otavio O Souza 		KASSERT(!d->bd_hbuf_in_use, ("hold buffer is in use"));
2638df8bae1dSRodney W. Grimes 		ROTATE_BUFFERS(d);
26397819da79SJohn-Mark Gurney 		do_wakeup = 1;
2640df8bae1dSRodney W. Grimes 		curlen = 0;
264160b4ad4bSMark Johnston 	} else {
264260b4ad4bSMark Johnston 		if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
2643df8bae1dSRodney W. Grimes 			/*
264460b4ad4bSMark Johnston 			 * Immediate mode is set, or the read timeout has
264560b4ad4bSMark Johnston 			 * already expired during a select call.  A packet
264660b4ad4bSMark Johnston 			 * arrived, so the reader should be woken up.
2647df8bae1dSRodney W. Grimes 			 */
26487819da79SJohn-Mark Gurney 			do_wakeup = 1;
264960b4ad4bSMark Johnston 		}
265060b4ad4bSMark Johnston 		pad = curlen - d->bd_slen;
265160b4ad4bSMark Johnston 		KASSERT(pad >= 0 && pad <= sizeof(zeroes),
265260b4ad4bSMark Johnston 		    ("%s: invalid pad byte count %d", __func__, pad));
265360b4ad4bSMark Johnston 		if (pad > 0) {
265460b4ad4bSMark Johnston 			/* Zero pad bytes. */
265560b4ad4bSMark Johnston 			bpf_append_bytes(d, d->bd_sbuf, d->bd_slen, zeroes,
265660b4ad4bSMark Johnston 			    pad);
265760b4ad4bSMark Johnston 		}
265860b4ad4bSMark Johnston 	}
265960b4ad4bSMark Johnston 
2660547d94bdSJung-uk Kim 	caplen = totlen - hdrlen;
2661547d94bdSJung-uk Kim 	tstype = d->bd_tstamp;
2662547d94bdSJung-uk Kim 	do_timestamp = tstype != BPF_T_NONE;
2663547d94bdSJung-uk Kim #ifndef BURN_BRIDGES
2664547d94bdSJung-uk Kim 	if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) {
2665547d94bdSJung-uk Kim 		struct bpf_ts ts;
2666547d94bdSJung-uk Kim 		if (do_timestamp)
2667547d94bdSJung-uk Kim 			bpf_bintime2ts(bt, &ts, tstype);
2668fc0a61a4SKonstantin Belousov #ifdef COMPAT_FREEBSD32
2669fc0a61a4SKonstantin Belousov 		if (d->bd_compat32) {
2670547d94bdSJung-uk Kim 			bzero(&hdr32_old, sizeof(hdr32_old));
2671547d94bdSJung-uk Kim 			if (do_timestamp) {
2672547d94bdSJung-uk Kim 				hdr32_old.bh_tstamp.tv_sec = ts.bt_sec;
2673547d94bdSJung-uk Kim 				hdr32_old.bh_tstamp.tv_usec = ts.bt_frac;
2674547d94bdSJung-uk Kim 			}
2675547d94bdSJung-uk Kim 			hdr32_old.bh_datalen = pktlen;
2676547d94bdSJung-uk Kim 			hdr32_old.bh_hdrlen = hdrlen;
2677547d94bdSJung-uk Kim 			hdr32_old.bh_caplen = caplen;
2678547d94bdSJung-uk Kim 			bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old,
2679547d94bdSJung-uk Kim 			    sizeof(hdr32_old));
2680547d94bdSJung-uk Kim 			goto copy;
2681547d94bdSJung-uk Kim 		}
2682547d94bdSJung-uk Kim #endif
2683547d94bdSJung-uk Kim 		bzero(&hdr_old, sizeof(hdr_old));
2684547d94bdSJung-uk Kim 		if (do_timestamp) {
2685547d94bdSJung-uk Kim 			hdr_old.bh_tstamp.tv_sec = ts.bt_sec;
2686547d94bdSJung-uk Kim 			hdr_old.bh_tstamp.tv_usec = ts.bt_frac;
2687547d94bdSJung-uk Kim 		}
2688547d94bdSJung-uk Kim 		hdr_old.bh_datalen = pktlen;
2689547d94bdSJung-uk Kim 		hdr_old.bh_hdrlen = hdrlen;
2690547d94bdSJung-uk Kim 		hdr_old.bh_caplen = caplen;
2691547d94bdSJung-uk Kim 		bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old,
2692547d94bdSJung-uk Kim 		    sizeof(hdr_old));
2693fc0a61a4SKonstantin Belousov 		goto copy;
2694fc0a61a4SKonstantin Belousov 	}
2695fc0a61a4SKonstantin Belousov #endif
2696df8bae1dSRodney W. Grimes 
2697df8bae1dSRodney W. Grimes 	/*
26984d621040SChristian S.J. Peron 	 * Append the bpf header.  Note we append the actual header size, but
26994d621040SChristian S.J. Peron 	 * move forward the length of the header plus padding.
2700df8bae1dSRodney W. Grimes 	 */
27014d621040SChristian S.J. Peron 	bzero(&hdr, sizeof(hdr));
2702547d94bdSJung-uk Kim 	if (do_timestamp)
2703547d94bdSJung-uk Kim 		bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype);
27044d621040SChristian S.J. Peron 	hdr.bh_datalen = pktlen;
27054d621040SChristian S.J. Peron 	hdr.bh_hdrlen = hdrlen;
2706547d94bdSJung-uk Kim 	hdr.bh_caplen = caplen;
27074d621040SChristian S.J. Peron 	bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr));
27084d621040SChristian S.J. Peron 
2709df8bae1dSRodney W. Grimes 	/*
2710df8bae1dSRodney W. Grimes 	 * Copy the packet data into the store buffer and update its length.
2711df8bae1dSRodney W. Grimes 	 */
2712547d94bdSJung-uk Kim #ifndef BURN_BRIDGES
2713fc0a61a4SKonstantin Belousov copy:
2714fc0a61a4SKonstantin Belousov #endif
2715547d94bdSJung-uk Kim 	(*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen);
2716df8bae1dSRodney W. Grimes 	d->bd_slen = curlen + totlen;
27177819da79SJohn-Mark Gurney 
27187819da79SJohn-Mark Gurney 	if (do_wakeup)
27197819da79SJohn-Mark Gurney 		bpf_wakeup(d);
2720df8bae1dSRodney W. Grimes }
2721df8bae1dSRodney W. Grimes 
2722df8bae1dSRodney W. Grimes /*
2723df8bae1dSRodney W. Grimes  * Free buffers currently in use by a descriptor.
2724df8bae1dSRodney W. Grimes  * Called on close.
2725df8bae1dSRodney W. Grimes  */
2726df8bae1dSRodney W. Grimes static void
2727699281b5SAndrey V. Elsukov bpfd_free(epoch_context_t ctx)
2728df8bae1dSRodney W. Grimes {
2729699281b5SAndrey V. Elsukov 	struct bpf_d *d;
2730699281b5SAndrey V. Elsukov 	struct bpf_program_buffer *p;
27314d621040SChristian S.J. Peron 
2732df8bae1dSRodney W. Grimes 	/*
2733df8bae1dSRodney W. Grimes 	 * We don't need to lock out interrupts since this descriptor has
2734df8bae1dSRodney W. Grimes 	 * been detached from its interface and it yet hasn't been marked
2735df8bae1dSRodney W. Grimes 	 * free.
2736df8bae1dSRodney W. Grimes 	 */
2737699281b5SAndrey V. Elsukov 	d = __containerof(ctx, struct bpf_d, epoch_ctx);
27384d621040SChristian S.J. Peron 	bpf_free(d);
273970485847SJung-uk Kim 	if (d->bd_rfilter != NULL) {
2740699281b5SAndrey V. Elsukov 		p = __containerof((void *)d->bd_rfilter,
2741699281b5SAndrey V. Elsukov 		    struct bpf_program_buffer, buffer);
2742af1f58dfSAndrey V. Elsukov #ifdef BPF_JITTER
2743af1f58dfSAndrey V. Elsukov 		p->func = d->bd_bfilter;
2744af1f58dfSAndrey V. Elsukov #endif
2745699281b5SAndrey V. Elsukov 		bpf_program_buffer_free(&p->epoch_ctx);
2746ae275efcSJung-uk Kim 	}
2747699281b5SAndrey V. Elsukov 	if (d->bd_wfilter != NULL) {
2748699281b5SAndrey V. Elsukov 		p = __containerof((void *)d->bd_wfilter,
2749699281b5SAndrey V. Elsukov 		    struct bpf_program_buffer, buffer);
2750af1f58dfSAndrey V. Elsukov #ifdef BPF_JITTER
2751af1f58dfSAndrey V. Elsukov 		p->func = NULL;
2752af1f58dfSAndrey V. Elsukov #endif
2753699281b5SAndrey V. Elsukov 		bpf_program_buffer_free(&p->epoch_ctx);
2754699281b5SAndrey V. Elsukov 	}
2755b2b7ca49SAlexander V. Chernikov 
2756699281b5SAndrey V. Elsukov 	mtx_destroy(&d->bd_lock);
2757b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_rcount);
2758b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_dcount);
2759b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_fcount);
2760b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_wcount);
2761b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_wfcount);
2762b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_wdcount);
2763b2b7ca49SAlexander V. Chernikov 	counter_u64_free(d->bd_zcopy);
2764699281b5SAndrey V. Elsukov 	free(d, M_BPF);
2765df8bae1dSRodney W. Grimes }
2766df8bae1dSRodney W. Grimes 
2767df8bae1dSRodney W. Grimes /*
276824a229f4SSam Leffler  * Attach an interface to bpf.  dlt is the link layer type; hdrlen is the
276924a229f4SSam Leffler  * fixed size of the link header (variable length headers not yet supported).
2770df8bae1dSRodney W. Grimes  */
2771df8bae1dSRodney W. Grimes void
277219ba8395SChristian S.J. Peron bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2773df8bae1dSRodney W. Grimes {
277424a229f4SSam Leffler 
277524a229f4SSam Leffler 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
277624a229f4SSam Leffler }
277724a229f4SSam Leffler 
277824a229f4SSam Leffler /*
277924a229f4SSam Leffler  * Attach an interface to bpf.  ifp is a pointer to the structure
278024a229f4SSam Leffler  * defining the interface to be attached, dlt is the link layer type,
278124a229f4SSam Leffler  * and hdrlen is the fixed size of the link header (variable length
278224a229f4SSam Leffler  * headers are not yet supporrted).
278324a229f4SSam Leffler  */
278424a229f4SSam Leffler void
2785699281b5SAndrey V. Elsukov bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen,
2786699281b5SAndrey V. Elsukov     struct bpf_if **driverp)
278724a229f4SSam Leffler {
2788df8bae1dSRodney W. Grimes 	struct bpf_if *bp;
278919ba8395SChristian S.J. Peron 
2790699281b5SAndrey V. Elsukov 	KASSERT(*driverp == NULL,
2791699281b5SAndrey V. Elsukov 	    ("bpfattach2: driverp already initialized"));
2792df8bae1dSRodney W. Grimes 
2793c5be49daSAndrey V. Elsukov 	bp = malloc(sizeof(*bp), M_BPF, M_WAITOK | M_ZERO);
2794c5be49daSAndrey V. Elsukov 
2795699281b5SAndrey V. Elsukov 	CK_LIST_INIT(&bp->bif_dlist);
2796699281b5SAndrey V. Elsukov 	CK_LIST_INIT(&bp->bif_wlist);
2797df8bae1dSRodney W. Grimes 	bp->bif_ifp = ifp;
2798df8bae1dSRodney W. Grimes 	bp->bif_dlt = dlt;
2799c5be49daSAndrey V. Elsukov 	bp->bif_hdrlen = hdrlen;
28009ce40d32SKristof Provost 	bp->bif_bpf = driverp;
28018288117aSMateusz Guzik 	refcount_init(&bp->bif_refcnt, 1);
280216d878ccSChristian S.J. Peron 	*driverp = bp;
2803699281b5SAndrey V. Elsukov 	/*
2804699281b5SAndrey V. Elsukov 	 * Reference ifnet pointer, so it won't freed until
2805699281b5SAndrey V. Elsukov 	 * we release it.
2806699281b5SAndrey V. Elsukov 	 */
2807699281b5SAndrey V. Elsukov 	if_ref(ifp);
2808e4b3229aSAlexander V. Chernikov 	BPF_LOCK();
2809699281b5SAndrey V. Elsukov 	CK_LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
2810e4b3229aSAlexander V. Chernikov 	BPF_UNLOCK();
2811df8bae1dSRodney W. Grimes 
2812616bc4f4SBjoern A. Zeeb 	if (bootverbose && IS_DEFAULT_VNET(curvnet))
281324a229f4SSam Leffler 		if_printf(ifp, "bpf attached\n");
2814df8bae1dSRodney W. Grimes }
281553ac6efbSJulian Elischer 
281605fc4164SBjoern A. Zeeb #ifdef VIMAGE
281705fc4164SBjoern A. Zeeb /*
281805fc4164SBjoern A. Zeeb  * When moving interfaces between vnet instances we need a way to
281905fc4164SBjoern A. Zeeb  * query the dlt and hdrlen before detach so we can re-attch the if_bpf
282005fc4164SBjoern A. Zeeb  * after the vmove.  We unfortunately have no device driver infrastructure
282105fc4164SBjoern A. Zeeb  * to query the interface for these values after creation/attach, thus
282205fc4164SBjoern A. Zeeb  * add this as a workaround.
282305fc4164SBjoern A. Zeeb  */
282405fc4164SBjoern A. Zeeb int
282505fc4164SBjoern A. Zeeb bpf_get_bp_params(struct bpf_if *bp, u_int *bif_dlt, u_int *bif_hdrlen)
282605fc4164SBjoern A. Zeeb {
282705fc4164SBjoern A. Zeeb 
282805fc4164SBjoern A. Zeeb 	if (bp == NULL)
282905fc4164SBjoern A. Zeeb 		return (ENXIO);
283005fc4164SBjoern A. Zeeb 	if (bif_dlt == NULL && bif_hdrlen == NULL)
283105fc4164SBjoern A. Zeeb 		return (0);
283205fc4164SBjoern A. Zeeb 
283305fc4164SBjoern A. Zeeb 	if (bif_dlt != NULL)
283405fc4164SBjoern A. Zeeb 		*bif_dlt = bp->bif_dlt;
283505fc4164SBjoern A. Zeeb 	if (bif_hdrlen != NULL)
283605fc4164SBjoern A. Zeeb 		*bif_hdrlen = bp->bif_hdrlen;
283705fc4164SBjoern A. Zeeb 
283805fc4164SBjoern A. Zeeb 	return (0);
283905fc4164SBjoern A. Zeeb }
284005fc4164SBjoern A. Zeeb #endif
284105fc4164SBjoern A. Zeeb 
2842de5d9935SRobert Watson /*
2843de5d9935SRobert Watson  * Detach bpf from an interface. This involves detaching each descriptor
28446c74ff0eSAlexander V. Chernikov  * associated with the interface. Notify each descriptor as it's detached
28456c74ff0eSAlexander V. Chernikov  * so that any sleepers wake up and get ENXIO.
2846de5d9935SRobert Watson  */
2847de5d9935SRobert Watson void
284819ba8395SChristian S.J. Peron bpfdetach(struct ifnet *ifp)
2849de5d9935SRobert Watson {
2850f079a0faSAlexander V. Chernikov 	struct bpf_if *bp, *bp_temp;
2851de5d9935SRobert Watson 	struct bpf_d *d;
28529a7e6bacSLawrence Stewart 
2853afa85850SAlexander V. Chernikov 	BPF_LOCK();
28549a7e6bacSLawrence Stewart 	/* Find all bpf_if struct's which reference ifp and detach them. */
2855699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH_SAFE(bp, &bpf_iflist, bif_next, bp_temp) {
2856f079a0faSAlexander V. Chernikov 		if (ifp != bp->bif_ifp)
2857f079a0faSAlexander V. Chernikov 			continue;
2858de5d9935SRobert Watson 
2859699281b5SAndrey V. Elsukov 		CK_LIST_REMOVE(bp, bif_next);
28607a974a64SZhenlei Huang 		*bp->bif_bpf = __DECONST(struct bpf_if *, &dead_bpf_if);
2861f079a0faSAlexander V. Chernikov 
2862699281b5SAndrey V. Elsukov 		CTR4(KTR_NET,
2863699281b5SAndrey V. Elsukov 		    "%s: sheduling free for encap %d (%p) for if %p",
2864f079a0faSAlexander V. Chernikov 		    __func__, bp->bif_dlt, bp, ifp);
2865f079a0faSAlexander V. Chernikov 
2866699281b5SAndrey V. Elsukov 		/* Detach common descriptors */
2867699281b5SAndrey V. Elsukov 		while ((d = CK_LIST_FIRST(&bp->bif_dlist)) != NULL) {
2868699281b5SAndrey V. Elsukov 			bpf_detachd_locked(d, true);
2869e7bb21b3SJonathan Lemon 		}
2870f079a0faSAlexander V. Chernikov 
2871699281b5SAndrey V. Elsukov 		/* Detach writer-only descriptors */
2872699281b5SAndrey V. Elsukov 		while ((d = CK_LIST_FIRST(&bp->bif_wlist)) != NULL) {
2873699281b5SAndrey V. Elsukov 			bpf_detachd_locked(d, true);
28746c74ff0eSAlexander V. Chernikov 		}
2875699281b5SAndrey V. Elsukov 		bpfif_rele(bp);
2876f079a0faSAlexander V. Chernikov 	}
2877f079a0faSAlexander V. Chernikov 	BPF_UNLOCK();
2878afa85850SAlexander V. Chernikov }
2879afa85850SAlexander V. Chernikov 
28808f31b879SJustin Hibbits bool
28818f31b879SJustin Hibbits bpf_peers_present_if(struct ifnet *ifp)
28828f31b879SJustin Hibbits {
288389204d9dSZhenlei Huang 	return (bpf_peers_present(ifp->if_bpf));
28848f31b879SJustin Hibbits }
28858f31b879SJustin Hibbits 
2886afa85850SAlexander V. Chernikov /*
28878eab61f3SSam Leffler  * Get a list of available data link type of the interface.
28888eab61f3SSam Leffler  */
28898eab61f3SSam Leffler static int
289019ba8395SChristian S.J. Peron bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
28918eab61f3SSam Leffler {
28928eab61f3SSam Leffler 	struct ifnet *ifp;
28938eab61f3SSam Leffler 	struct bpf_if *bp;
289470209acaSKonstantin Belousov 	u_int *lst;
289570209acaSKonstantin Belousov 	int error, n, n1;
28968eab61f3SSam Leffler 
289797aacec6SAlexander V. Chernikov 	BPF_LOCK_ASSERT();
289897aacec6SAlexander V. Chernikov 
28998eab61f3SSam Leffler 	ifp = d->bd_bif->bif_ifp;
290070209acaSKonstantin Belousov 	n1 = 0;
2901699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
290270209acaSKonstantin Belousov 		if (bp->bif_ifp == ifp)
290370209acaSKonstantin Belousov 			n1++;
290470209acaSKonstantin Belousov 	}
290570209acaSKonstantin Belousov 	if (bfl->bfl_list == NULL) {
290670209acaSKonstantin Belousov 		bfl->bfl_len = n1;
290770209acaSKonstantin Belousov 		return (0);
290870209acaSKonstantin Belousov 	}
290970209acaSKonstantin Belousov 	if (n1 > bfl->bfl_len)
291070209acaSKonstantin Belousov 		return (ENOMEM);
2911699281b5SAndrey V. Elsukov 
291270209acaSKonstantin Belousov 	lst = malloc(n1 * sizeof(u_int), M_TEMP, M_WAITOK);
29138eab61f3SSam Leffler 	n = 0;
2914699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
29158eab61f3SSam Leffler 		if (bp->bif_ifp != ifp)
29168eab61f3SSam Leffler 			continue;
2917699281b5SAndrey V. Elsukov 		lst[n++] = bp->bif_dlt;
29188eab61f3SSam Leffler 	}
291970209acaSKonstantin Belousov 	error = copyout(lst, bfl->bfl_list, sizeof(u_int) * n);
292070209acaSKonstantin Belousov 	free(lst, M_TEMP);
29218eab61f3SSam Leffler 	bfl->bfl_len = n;
29228eab61f3SSam Leffler 	return (error);
29238eab61f3SSam Leffler }
29248eab61f3SSam Leffler 
29258eab61f3SSam Leffler /*
29268eab61f3SSam Leffler  * Set the data link type of a BPF instance.
29278eab61f3SSam Leffler  */
29288eab61f3SSam Leffler static int
292919ba8395SChristian S.J. Peron bpf_setdlt(struct bpf_d *d, u_int dlt)
29308eab61f3SSam Leffler {
29318eab61f3SSam Leffler 	int error, opromisc;
29328eab61f3SSam Leffler 	struct ifnet *ifp;
29338eab61f3SSam Leffler 	struct bpf_if *bp;
29348eab61f3SSam Leffler 
29356c74ff0eSAlexander V. Chernikov 	BPF_LOCK_ASSERT();
2936699281b5SAndrey V. Elsukov 	MPASS(d->bd_bif != NULL);
29376c74ff0eSAlexander V. Chernikov 
2938699281b5SAndrey V. Elsukov 	/*
2939699281b5SAndrey V. Elsukov 	 * It is safe to check bd_bif without BPFD_LOCK, it can not be
2940699281b5SAndrey V. Elsukov 	 * changed while we hold global lock.
2941699281b5SAndrey V. Elsukov 	 */
29428eab61f3SSam Leffler 	if (d->bd_bif->bif_dlt == dlt)
29438eab61f3SSam Leffler 		return (0);
29446c74ff0eSAlexander V. Chernikov 
2945699281b5SAndrey V. Elsukov 	ifp = d->bd_bif->bif_ifp;
2946699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
29478eab61f3SSam Leffler 		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
29488eab61f3SSam Leffler 			break;
29498eab61f3SSam Leffler 	}
2950699281b5SAndrey V. Elsukov 	if (bp == NULL)
2951699281b5SAndrey V. Elsukov 		return (EINVAL);
29526c74ff0eSAlexander V. Chernikov 
29538eab61f3SSam Leffler 	opromisc = d->bd_promisc;
29548eab61f3SSam Leffler 	bpf_attachd(d, bp);
29558eab61f3SSam Leffler 	if (opromisc) {
29568eab61f3SSam Leffler 		error = ifpromisc(bp->bif_ifp, 1);
29578eab61f3SSam Leffler 		if (error)
2958699281b5SAndrey V. Elsukov 			if_printf(bp->bif_ifp, "%s: ifpromisc failed (%d)\n",
2959699281b5SAndrey V. Elsukov 			    __func__, error);
29608eab61f3SSam Leffler 		else
29618eab61f3SSam Leffler 			d->bd_promisc = 1;
29628eab61f3SSam Leffler 	}
2963699281b5SAndrey V. Elsukov 	return (0);
2964de5d9935SRobert Watson }
2965de5d9935SRobert Watson 
29663f54a085SPoul-Henning Kamp static void
296719ba8395SChristian S.J. Peron bpf_drvinit(void *unused)
296853ac6efbSJulian Elischer {
2969136600feSEd Schouten 	struct cdev *dev;
297053ac6efbSJulian Elischer 
2971f422673eSStephen Hurd 	sx_init(&bpf_sx, "bpf global lock");
2972699281b5SAndrey V. Elsukov 	CK_LIST_INIT(&bpf_iflist);
2973136600feSEd Schouten 
2974136600feSEd Schouten 	dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf");
2975136600feSEd Schouten 	/* For compatibility */
2976136600feSEd Schouten 	make_dev_alias(dev, "bpf0");
29777198bf47SJulian Elischer }
297853ac6efbSJulian Elischer 
29790e37f3e1SChristian S.J. Peron /*
29800e37f3e1SChristian S.J. Peron  * Zero out the various packet counters associated with all of the bpf
29810e37f3e1SChristian S.J. Peron  * descriptors.  At some point, we will probably want to get a bit more
29820e37f3e1SChristian S.J. Peron  * granular and allow the user to specify descriptors to be zeroed.
29830e37f3e1SChristian S.J. Peron  */
29840e37f3e1SChristian S.J. Peron static void
29850e37f3e1SChristian S.J. Peron bpf_zero_counters(void)
29860e37f3e1SChristian S.J. Peron {
29870e37f3e1SChristian S.J. Peron 	struct bpf_if *bp;
29880e37f3e1SChristian S.J. Peron 	struct bpf_d *bd;
29890e37f3e1SChristian S.J. Peron 
2990e4b3229aSAlexander V. Chernikov 	BPF_LOCK();
2991699281b5SAndrey V. Elsukov 	/*
2992699281b5SAndrey V. Elsukov 	 * We are protected by global lock here, interfaces and
2993699281b5SAndrey V. Elsukov 	 * descriptors can not be deleted while we hold it.
2994699281b5SAndrey V. Elsukov 	 */
2995699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2996699281b5SAndrey V. Elsukov 		CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2997b2b7ca49SAlexander V. Chernikov 			counter_u64_zero(bd->bd_rcount);
2998b2b7ca49SAlexander V. Chernikov 			counter_u64_zero(bd->bd_dcount);
2999b2b7ca49SAlexander V. Chernikov 			counter_u64_zero(bd->bd_fcount);
3000b2b7ca49SAlexander V. Chernikov 			counter_u64_zero(bd->bd_wcount);
3001b2b7ca49SAlexander V. Chernikov 			counter_u64_zero(bd->bd_wfcount);
3002b2b7ca49SAlexander V. Chernikov 			counter_u64_zero(bd->bd_zcopy);
30030e37f3e1SChristian S.J. Peron 		}
30040e37f3e1SChristian S.J. Peron 	}
3005e4b3229aSAlexander V. Chernikov 	BPF_UNLOCK();
30060e37f3e1SChristian S.J. Peron }
30070e37f3e1SChristian S.J. Peron 
30086c74ff0eSAlexander V. Chernikov /*
30096c74ff0eSAlexander V. Chernikov  * Fill filter statistics
30106c74ff0eSAlexander V. Chernikov  */
301169f7644bSChristian S.J. Peron static void
301269f7644bSChristian S.J. Peron bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
301369f7644bSChristian S.J. Peron {
301469f7644bSChristian S.J. Peron 
3015699281b5SAndrey V. Elsukov 	BPF_LOCK_ASSERT();
301669f7644bSChristian S.J. Peron 	bzero(d, sizeof(*d));
30174d621040SChristian S.J. Peron 	d->bd_structsize = sizeof(*d);
301869f7644bSChristian S.J. Peron 	d->bd_immediate = bd->bd_immediate;
301969f7644bSChristian S.J. Peron 	d->bd_promisc = bd->bd_promisc;
302069f7644bSChristian S.J. Peron 	d->bd_hdrcmplt = bd->bd_hdrcmplt;
3021560a54e1SJung-uk Kim 	d->bd_direction = bd->bd_direction;
3022560a54e1SJung-uk Kim 	d->bd_feedback = bd->bd_feedback;
302369f7644bSChristian S.J. Peron 	d->bd_async = bd->bd_async;
3024b2b7ca49SAlexander V. Chernikov 	d->bd_rcount = counter_u64_fetch(bd->bd_rcount);
3025b2b7ca49SAlexander V. Chernikov 	d->bd_dcount = counter_u64_fetch(bd->bd_dcount);
3026b2b7ca49SAlexander V. Chernikov 	d->bd_fcount = counter_u64_fetch(bd->bd_fcount);
302769f7644bSChristian S.J. Peron 	d->bd_sig = bd->bd_sig;
302869f7644bSChristian S.J. Peron 	d->bd_slen = bd->bd_slen;
302969f7644bSChristian S.J. Peron 	d->bd_hlen = bd->bd_hlen;
303069f7644bSChristian S.J. Peron 	d->bd_bufsize = bd->bd_bufsize;
303169f7644bSChristian S.J. Peron 	d->bd_pid = bd->bd_pid;
303269f7644bSChristian S.J. Peron 	strlcpy(d->bd_ifname,
303369f7644bSChristian S.J. Peron 	    bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
303493e39f0bSChristian S.J. Peron 	d->bd_locked = bd->bd_locked;
3035b2b7ca49SAlexander V. Chernikov 	d->bd_wcount = counter_u64_fetch(bd->bd_wcount);
3036b2b7ca49SAlexander V. Chernikov 	d->bd_wdcount = counter_u64_fetch(bd->bd_wdcount);
3037b2b7ca49SAlexander V. Chernikov 	d->bd_wfcount = counter_u64_fetch(bd->bd_wfcount);
3038b2b7ca49SAlexander V. Chernikov 	d->bd_zcopy = counter_u64_fetch(bd->bd_zcopy);
30394d621040SChristian S.J. Peron 	d->bd_bufmode = bd->bd_bufmode;
304069f7644bSChristian S.J. Peron }
304169f7644bSChristian S.J. Peron 
30426c74ff0eSAlexander V. Chernikov /*
30436c74ff0eSAlexander V. Chernikov  * Handle `netstat -B' stats request
30446c74ff0eSAlexander V. Chernikov  */
304569f7644bSChristian S.J. Peron static int
304669f7644bSChristian S.J. Peron bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
304769f7644bSChristian S.J. Peron {
30480e1152fcSHans Petter Selasky 	static const struct xbpf_d zerostats;
30490e1152fcSHans Petter Selasky 	struct xbpf_d *xbdbuf, *xbd, tempstats;
3050422a63daSChristian S.J. Peron 	int index, error;
305169f7644bSChristian S.J. Peron 	struct bpf_if *bp;
305269f7644bSChristian S.J. Peron 	struct bpf_d *bd;
305369f7644bSChristian S.J. Peron 
305469f7644bSChristian S.J. Peron 	/*
305569f7644bSChristian S.J. Peron 	 * XXX This is not technically correct. It is possible for non
305669f7644bSChristian S.J. Peron 	 * privileged users to open bpf devices. It would make sense
305769f7644bSChristian S.J. Peron 	 * if the users who opened the devices were able to retrieve
305869f7644bSChristian S.J. Peron 	 * the statistics for them, too.
305969f7644bSChristian S.J. Peron 	 */
3060acd3428bSRobert Watson 	error = priv_check(req->td, PRIV_NET_BPF);
306169f7644bSChristian S.J. Peron 	if (error)
306269f7644bSChristian S.J. Peron 		return (error);
30630e37f3e1SChristian S.J. Peron 	/*
30640e37f3e1SChristian S.J. Peron 	 * Check to see if the user is requesting that the counters be
30650e37f3e1SChristian S.J. Peron 	 * zeroed out.  Explicitly check that the supplied data is zeroed,
30660e37f3e1SChristian S.J. Peron 	 * as we aren't allowing the user to set the counters currently.
30670e37f3e1SChristian S.J. Peron 	 */
30680e37f3e1SChristian S.J. Peron 	if (req->newptr != NULL) {
30690e1152fcSHans Petter Selasky 		if (req->newlen != sizeof(tempstats))
30700e37f3e1SChristian S.J. Peron 			return (EINVAL);
30710e1152fcSHans Petter Selasky 		memset(&tempstats, 0, sizeof(tempstats));
30720e1152fcSHans Petter Selasky 		error = SYSCTL_IN(req, &tempstats, sizeof(tempstats));
30730e1152fcSHans Petter Selasky 		if (error)
30740e1152fcSHans Petter Selasky 			return (error);
30750e1152fcSHans Petter Selasky 		if (bcmp(&tempstats, &zerostats, sizeof(tempstats)) != 0)
30760e37f3e1SChristian S.J. Peron 			return (EINVAL);
30770e37f3e1SChristian S.J. Peron 		bpf_zero_counters();
30780e37f3e1SChristian S.J. Peron 		return (0);
30790e37f3e1SChristian S.J. Peron 	}
308069f7644bSChristian S.J. Peron 	if (req->oldptr == NULL)
3081422a63daSChristian S.J. Peron 		return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
308269f7644bSChristian S.J. Peron 	if (bpf_bpfd_cnt == 0)
308369f7644bSChristian S.J. Peron 		return (SYSCTL_OUT(req, 0, 0));
3084422a63daSChristian S.J. Peron 	xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
3085e4b3229aSAlexander V. Chernikov 	BPF_LOCK();
3086422a63daSChristian S.J. Peron 	if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
3087e4b3229aSAlexander V. Chernikov 		BPF_UNLOCK();
3088422a63daSChristian S.J. Peron 		free(xbdbuf, M_BPF);
3089422a63daSChristian S.J. Peron 		return (ENOMEM);
3090422a63daSChristian S.J. Peron 	}
3091422a63daSChristian S.J. Peron 	index = 0;
3092699281b5SAndrey V. Elsukov 	CK_LIST_FOREACH(bp, &bpf_iflist, bif_next) {
309351ec1eb7SAlexander V. Chernikov 		/* Send writers-only first */
3094699281b5SAndrey V. Elsukov 		CK_LIST_FOREACH(bd, &bp->bif_wlist, bd_next) {
309551ec1eb7SAlexander V. Chernikov 			xbd = &xbdbuf[index++];
309651ec1eb7SAlexander V. Chernikov 			bpfstats_fill_xbpf(xbd, bd);
309751ec1eb7SAlexander V. Chernikov 		}
3098699281b5SAndrey V. Elsukov 		CK_LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
3099422a63daSChristian S.J. Peron 			xbd = &xbdbuf[index++];
3100422a63daSChristian S.J. Peron 			bpfstats_fill_xbpf(xbd, bd);
310169f7644bSChristian S.J. Peron 		}
310269f7644bSChristian S.J. Peron 	}
3103e4b3229aSAlexander V. Chernikov 	BPF_UNLOCK();
3104422a63daSChristian S.J. Peron 	error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
3105422a63daSChristian S.J. Peron 	free(xbdbuf, M_BPF);
310669f7644bSChristian S.J. Peron 	return (error);
310769f7644bSChristian S.J. Peron }
310869f7644bSChristian S.J. Peron 
3109237fdd78SRobert Watson SYSINIT(bpfdev, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, bpf_drvinit, NULL);
311053ac6efbSJulian Elischer 
31115bb5f2c9SPeter Wemm #else /* !DEV_BPF && !NETGRAPH_BPF */
31128bd1f0cfSMark Johnston 
3113f8dc4716SMike Smith /*
3114f8dc4716SMike Smith  * NOP stubs to allow bpf-using drivers to load and function.
3115f8dc4716SMike Smith  *
3116f8dc4716SMike Smith  * A 'better' implementation would allow the core bpf functionality
3117f8dc4716SMike Smith  * to be loaded at runtime.
3118f8dc4716SMike Smith  */
3119f8dc4716SMike Smith 
3120f8dc4716SMike Smith void
312119ba8395SChristian S.J. Peron bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
3122f8dc4716SMike Smith {
3123f8dc4716SMike Smith }
3124f8dc4716SMike Smith 
3125f8dc4716SMike Smith void
31269df6eeabSZhenlei Huang bpf_tap_if(if_t ifp, u_char *pkt, u_int pktlen)
31279df6eeabSZhenlei Huang {
31289df6eeabSZhenlei Huang }
31299df6eeabSZhenlei Huang 
31309df6eeabSZhenlei Huang void
313119ba8395SChristian S.J. Peron bpf_mtap(struct bpf_if *bp, struct mbuf *m)
3132f8dc4716SMike Smith {
3133f8dc4716SMike Smith }
3134f8dc4716SMike Smith 
3135f8dc4716SMike Smith void
31369df6eeabSZhenlei Huang bpf_mtap_if(if_t ifp, struct mbuf *m)
31379df6eeabSZhenlei Huang {
31389df6eeabSZhenlei Huang }
31399df6eeabSZhenlei Huang 
31409df6eeabSZhenlei Huang void
314119ba8395SChristian S.J. Peron bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
3142437ffe18SSam Leffler {
3143437ffe18SSam Leffler }
3144437ffe18SSam Leffler 
3145437ffe18SSam Leffler void
31469df6eeabSZhenlei Huang bpf_mtap2_if(if_t ifp, void *data, u_int dlen, struct mbuf *m)
31479df6eeabSZhenlei Huang {
31489df6eeabSZhenlei Huang }
31499df6eeabSZhenlei Huang 
31509df6eeabSZhenlei Huang void
315119ba8395SChristian S.J. Peron bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
3152f8dc4716SMike Smith {
31537eae78a4SChristian S.J. Peron 
31547eae78a4SChristian S.J. Peron 	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
3155f8dc4716SMike Smith }
3156f8dc4716SMike Smith 
3157da626c17SBill Paul void
315819ba8395SChristian S.J. Peron bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
31595f7a7923SSam Leffler {
31607eae78a4SChristian S.J. Peron 
31617a974a64SZhenlei Huang 	*driverp = __DECONST(struct bpf_if *, &dead_bpf_if);
31625f7a7923SSam Leffler }
31635f7a7923SSam Leffler 
31645f7a7923SSam Leffler void
316519ba8395SChristian S.J. Peron bpfdetach(struct ifnet *ifp)
3166da626c17SBill Paul {
3167da626c17SBill Paul }
3168da626c17SBill Paul 
31698f31b879SJustin Hibbits bool
31708f31b879SJustin Hibbits bpf_peers_present_if(struct ifnet *ifp)
31718f31b879SJustin Hibbits {
31728f31b879SJustin Hibbits 	return (false);
31738f31b879SJustin Hibbits }
31748f31b879SJustin Hibbits 
3175f8dc4716SMike Smith u_int
317619ba8395SChristian S.J. Peron bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
3177f8dc4716SMike Smith {
3178*1baf6164SZhenlei Huang 	return (-1);	/* "no filter" behaviour */
3179f8dc4716SMike Smith }
3180f8dc4716SMike Smith 
31815bb5f2c9SPeter Wemm int
318219ba8395SChristian S.J. Peron bpf_validate(const struct bpf_insn *f, int len)
31835bb5f2c9SPeter Wemm {
3184*1baf6164SZhenlei Huang 	return (0);	/* false */
31855bb5f2c9SPeter Wemm }
31865bb5f2c9SPeter Wemm 
31875bb5f2c9SPeter Wemm #endif /* !DEV_BPF && !NETGRAPH_BPF */
318805fc4164SBjoern A. Zeeb 
318905fc4164SBjoern A. Zeeb #ifdef DDB
319005fc4164SBjoern A. Zeeb static void
319105fc4164SBjoern A. Zeeb bpf_show_bpf_if(struct bpf_if *bpf_if)
319205fc4164SBjoern A. Zeeb {
319305fc4164SBjoern A. Zeeb 
319405fc4164SBjoern A. Zeeb 	if (bpf_if == NULL)
319505fc4164SBjoern A. Zeeb 		return;
319605fc4164SBjoern A. Zeeb 	db_printf("%p:\n", bpf_if);
319705fc4164SBjoern A. Zeeb #define	BPF_DB_PRINTF(f, e)	db_printf("   %s = " f "\n", #e, bpf_if->e);
31988288117aSMateusz Guzik #define	BPF_DB_PRINTF_RAW(f, e)	db_printf("   %s = " f "\n", #e, e);
319905fc4164SBjoern A. Zeeb 	/* bif_ext.bif_next */
320005fc4164SBjoern A. Zeeb 	/* bif_ext.bif_dlist */
320105fc4164SBjoern A. Zeeb 	BPF_DB_PRINTF("%#x", bif_dlt);
320205fc4164SBjoern A. Zeeb 	BPF_DB_PRINTF("%u", bif_hdrlen);
320305fc4164SBjoern A. Zeeb 	/* bif_wlist */
3204699281b5SAndrey V. Elsukov 	BPF_DB_PRINTF("%p", bif_ifp);
3205699281b5SAndrey V. Elsukov 	BPF_DB_PRINTF("%p", bif_bpf);
32068288117aSMateusz Guzik 	BPF_DB_PRINTF_RAW("%u", refcount_load(&bpf_if->bif_refcnt));
320705fc4164SBjoern A. Zeeb }
320805fc4164SBjoern A. Zeeb 
320905fc4164SBjoern A. Zeeb DB_SHOW_COMMAND(bpf_if, db_show_bpf_if)
321005fc4164SBjoern A. Zeeb {
321105fc4164SBjoern A. Zeeb 
321205fc4164SBjoern A. Zeeb 	if (!have_addr) {
321305fc4164SBjoern A. Zeeb 		db_printf("usage: show bpf_if <struct bpf_if *>\n");
321405fc4164SBjoern A. Zeeb 		return;
321505fc4164SBjoern A. Zeeb 	}
321605fc4164SBjoern A. Zeeb 
321705fc4164SBjoern A. Zeeb 	bpf_show_bpf_if((struct bpf_if *)addr);
321805fc4164SBjoern A. Zeeb }
321905fc4164SBjoern A. Zeeb #endif
3220