xref: /freebsd/sys/netpfil/pf/if_pfsync.c (revision cc1a53bc1aea0675d64e9547cdca241612906592)
1 /*-
2  * SPDX-License-Identifier: (BSD-2-Clause-FreeBSD AND ISC)
3  *
4  * Copyright (c) 2002 Michael Shalayeff
5  * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT,
21  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23  * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
25  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
26  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
27  * THE POSSIBILITY OF SUCH DAMAGE.
28  */
29 
30 /*-
31  * Copyright (c) 2009 David Gwynne <dlg@openbsd.org>
32  *
33  * Permission to use, copy, modify, and distribute this software for any
34  * purpose with or without fee is hereby granted, provided that the above
35  * copyright notice and this permission notice appear in all copies.
36  *
37  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
38  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
39  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
40  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
41  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
42  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
43  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
44  */
45 
46 /*
47  * $OpenBSD: if_pfsync.c,v 1.110 2009/02/24 05:39:19 dlg Exp $
48  *
49  * Revisions picked from OpenBSD after revision 1.110 import:
50  * 1.119 - don't m_copydata() beyond the len of mbuf in pfsync_input()
51  * 1.118, 1.124, 1.148, 1.149, 1.151, 1.171 - fixes to bulk updates
52  * 1.120, 1.175 - use monotonic time_uptime
53  * 1.122 - reduce number of updates for non-TCP sessions
54  * 1.125, 1.127 - rewrite merge or stale processing
55  * 1.128 - cleanups
56  * 1.146 - bzero() mbuf before sparsely filling it with data
57  * 1.170 - SIOCSIFMTU checks
58  * 1.126, 1.142 - deferred packets processing
59  * 1.173 - correct expire time processing
60  */
61 
62 #include <sys/cdefs.h>
63 __FBSDID("$FreeBSD$");
64 
65 #include "opt_inet.h"
66 #include "opt_inet6.h"
67 #include "opt_pf.h"
68 
69 #include <sys/param.h>
70 #include <sys/bus.h>
71 #include <sys/endian.h>
72 #include <sys/interrupt.h>
73 #include <sys/kernel.h>
74 #include <sys/lock.h>
75 #include <sys/mbuf.h>
76 #include <sys/module.h>
77 #include <sys/mutex.h>
78 #include <sys/priv.h>
79 #include <sys/protosw.h>
80 #include <sys/smp.h>
81 #include <sys/socket.h>
82 #include <sys/sockio.h>
83 #include <sys/sysctl.h>
84 #include <sys/syslog.h>
85 
86 #include <net/bpf.h>
87 #include <net/if.h>
88 #include <net/if_var.h>
89 #include <net/if_clone.h>
90 #include <net/if_types.h>
91 #include <net/vnet.h>
92 #include <net/pfvar.h>
93 #include <net/if_pfsync.h>
94 
95 #include <netinet/if_ether.h>
96 #include <netinet/in.h>
97 #include <netinet/in_var.h>
98 #include <netinet/ip.h>
99 #include <netinet/ip_carp.h>
100 #include <netinet/ip_var.h>
101 #include <netinet/tcp.h>
102 #include <netinet/tcp_fsm.h>
103 #include <netinet/tcp_seq.h>
104 
105 #define PFSYNC_MINPKT ( \
106 	sizeof(struct ip) + \
107 	sizeof(struct pfsync_header) + \
108 	sizeof(struct pfsync_subheader) )
109 
110 struct pfsync_bucket;
111 
112 struct pfsync_pkt {
113 	struct ip *ip;
114 	struct in_addr src;
115 	u_int8_t flags;
116 };
117 
118 static int	pfsync_upd_tcp(struct pf_kstate *, struct pfsync_state_peer *,
119 		    struct pfsync_state_peer *);
120 static int	pfsync_in_clr(struct pfsync_pkt *, struct mbuf *, int, int);
121 static int	pfsync_in_ins(struct pfsync_pkt *, struct mbuf *, int, int);
122 static int	pfsync_in_iack(struct pfsync_pkt *, struct mbuf *, int, int);
123 static int	pfsync_in_upd(struct pfsync_pkt *, struct mbuf *, int, int);
124 static int	pfsync_in_upd_c(struct pfsync_pkt *, struct mbuf *, int, int);
125 static int	pfsync_in_ureq(struct pfsync_pkt *, struct mbuf *, int, int);
126 static int	pfsync_in_del(struct pfsync_pkt *, struct mbuf *, int, int);
127 static int	pfsync_in_del_c(struct pfsync_pkt *, struct mbuf *, int, int);
128 static int	pfsync_in_bus(struct pfsync_pkt *, struct mbuf *, int, int);
129 static int	pfsync_in_tdb(struct pfsync_pkt *, struct mbuf *, int, int);
130 static int	pfsync_in_eof(struct pfsync_pkt *, struct mbuf *, int, int);
131 static int	pfsync_in_error(struct pfsync_pkt *, struct mbuf *, int, int);
132 
133 static int (*pfsync_acts[])(struct pfsync_pkt *, struct mbuf *, int, int) = {
134 	pfsync_in_clr,			/* PFSYNC_ACT_CLR */
135 	pfsync_in_ins,			/* PFSYNC_ACT_INS */
136 	pfsync_in_iack,			/* PFSYNC_ACT_INS_ACK */
137 	pfsync_in_upd,			/* PFSYNC_ACT_UPD */
138 	pfsync_in_upd_c,		/* PFSYNC_ACT_UPD_C */
139 	pfsync_in_ureq,			/* PFSYNC_ACT_UPD_REQ */
140 	pfsync_in_del,			/* PFSYNC_ACT_DEL */
141 	pfsync_in_del_c,		/* PFSYNC_ACT_DEL_C */
142 	pfsync_in_error,		/* PFSYNC_ACT_INS_F */
143 	pfsync_in_error,		/* PFSYNC_ACT_DEL_F */
144 	pfsync_in_bus,			/* PFSYNC_ACT_BUS */
145 	pfsync_in_tdb,			/* PFSYNC_ACT_TDB */
146 	pfsync_in_eof			/* PFSYNC_ACT_EOF */
147 };
148 
149 struct pfsync_q {
150 	void		(*write)(struct pf_kstate *, void *);
151 	size_t		len;
152 	u_int8_t	action;
153 };
154 
155 /* we have one of these for every PFSYNC_S_ */
156 static void	pfsync_out_state(struct pf_kstate *, void *);
157 static void	pfsync_out_iack(struct pf_kstate *, void *);
158 static void	pfsync_out_upd_c(struct pf_kstate *, void *);
159 static void	pfsync_out_del(struct pf_kstate *, void *);
160 
161 static struct pfsync_q pfsync_qs[] = {
162 	{ pfsync_out_state, sizeof(struct pfsync_state),   PFSYNC_ACT_INS },
163 	{ pfsync_out_iack,  sizeof(struct pfsync_ins_ack), PFSYNC_ACT_INS_ACK },
164 	{ pfsync_out_state, sizeof(struct pfsync_state),   PFSYNC_ACT_UPD },
165 	{ pfsync_out_upd_c, sizeof(struct pfsync_upd_c),   PFSYNC_ACT_UPD_C },
166 	{ pfsync_out_del,   sizeof(struct pfsync_del_c),   PFSYNC_ACT_DEL_C }
167 };
168 
169 static void	pfsync_q_ins(struct pf_kstate *, int, bool);
170 static void	pfsync_q_del(struct pf_kstate *, bool, struct pfsync_bucket *);
171 
172 static void	pfsync_update_state(struct pf_kstate *);
173 
174 struct pfsync_upd_req_item {
175 	TAILQ_ENTRY(pfsync_upd_req_item)	ur_entry;
176 	struct pfsync_upd_req			ur_msg;
177 };
178 
179 struct pfsync_deferral {
180 	struct pfsync_softc		*pd_sc;
181 	TAILQ_ENTRY(pfsync_deferral)	pd_entry;
182 	u_int				pd_refs;
183 	struct callout			pd_tmo;
184 
185 	struct pf_kstate		*pd_st;
186 	struct mbuf			*pd_m;
187 };
188 
189 struct pfsync_sofct;
190 
191 struct pfsync_bucket
192 {
193 	int			b_id;
194 	struct pfsync_softc	*b_sc;
195 	struct mtx		b_mtx;
196 	struct callout		b_tmo;
197 	int			b_flags;
198 #define	PFSYNCF_BUCKET_PUSH	0x00000001
199 
200 	size_t			b_len;
201 	TAILQ_HEAD(, pf_kstate)			b_qs[PFSYNC_S_COUNT];
202 	TAILQ_HEAD(, pfsync_upd_req_item)	b_upd_req_list;
203 	TAILQ_HEAD(, pfsync_deferral)		b_deferrals;
204 	u_int			b_deferred;
205 	void			*b_plus;
206 	size_t			b_pluslen;
207 
208 	struct  ifaltq b_snd;
209 };
210 
211 struct pfsync_softc {
212 	/* Configuration */
213 	struct ifnet		*sc_ifp;
214 	struct ifnet		*sc_sync_if;
215 	struct ip_moptions	sc_imo;
216 	struct in_addr		sc_sync_peer;
217 	uint32_t		sc_flags;
218 	uint8_t			sc_maxupdates;
219 	struct ip		sc_template;
220 	struct mtx		sc_mtx;
221 
222 	/* Queued data */
223 	struct pfsync_bucket	*sc_buckets;
224 
225 	/* Bulk update info */
226 	struct mtx		sc_bulk_mtx;
227 	uint32_t		sc_ureq_sent;
228 	int			sc_bulk_tries;
229 	uint32_t		sc_ureq_received;
230 	int			sc_bulk_hashid;
231 	uint64_t		sc_bulk_stateid;
232 	uint32_t		sc_bulk_creatorid;
233 	struct callout		sc_bulk_tmo;
234 	struct callout		sc_bulkfail_tmo;
235 };
236 
237 #define	PFSYNC_LOCK(sc)		mtx_lock(&(sc)->sc_mtx)
238 #define	PFSYNC_UNLOCK(sc)	mtx_unlock(&(sc)->sc_mtx)
239 #define	PFSYNC_LOCK_ASSERT(sc)	mtx_assert(&(sc)->sc_mtx, MA_OWNED)
240 
241 #define PFSYNC_BUCKET_LOCK(b)		mtx_lock(&(b)->b_mtx)
242 #define PFSYNC_BUCKET_UNLOCK(b)		mtx_unlock(&(b)->b_mtx)
243 #define PFSYNC_BUCKET_LOCK_ASSERT(b)	mtx_assert(&(b)->b_mtx, MA_OWNED)
244 
245 #define	PFSYNC_BLOCK(sc)	mtx_lock(&(sc)->sc_bulk_mtx)
246 #define	PFSYNC_BUNLOCK(sc)	mtx_unlock(&(sc)->sc_bulk_mtx)
247 #define	PFSYNC_BLOCK_ASSERT(sc)	mtx_assert(&(sc)->sc_bulk_mtx, MA_OWNED)
248 
249 static const char pfsyncname[] = "pfsync";
250 static MALLOC_DEFINE(M_PFSYNC, pfsyncname, "pfsync(4) data");
251 VNET_DEFINE_STATIC(struct pfsync_softc	*, pfsyncif) = NULL;
252 #define	V_pfsyncif		VNET(pfsyncif)
253 VNET_DEFINE_STATIC(void *, pfsync_swi_cookie) = NULL;
254 #define	V_pfsync_swi_cookie	VNET(pfsync_swi_cookie)
255 VNET_DEFINE_STATIC(struct intr_event *, pfsync_swi_ie);
256 #define	V_pfsync_swi_ie		VNET(pfsync_swi_ie)
257 VNET_DEFINE_STATIC(struct pfsyncstats, pfsyncstats);
258 #define	V_pfsyncstats		VNET(pfsyncstats)
259 VNET_DEFINE_STATIC(int, pfsync_carp_adj) = CARP_MAXSKEW;
260 #define	V_pfsync_carp_adj	VNET(pfsync_carp_adj)
261 
262 static void	pfsync_timeout(void *);
263 static void	pfsync_push(struct pfsync_bucket *);
264 static void	pfsync_push_all(struct pfsync_softc *);
265 static void	pfsyncintr(void *);
266 static int	pfsync_multicast_setup(struct pfsync_softc *, struct ifnet *,
267 		    struct in_mfilter *imf);
268 static void	pfsync_multicast_cleanup(struct pfsync_softc *);
269 static void	pfsync_pointers_init(void);
270 static void	pfsync_pointers_uninit(void);
271 static int	pfsync_init(void);
272 static void	pfsync_uninit(void);
273 
274 static unsigned long pfsync_buckets;
275 
276 SYSCTL_NODE(_net, OID_AUTO, pfsync, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
277     "PFSYNC");
278 SYSCTL_STRUCT(_net_pfsync, OID_AUTO, stats, CTLFLAG_VNET | CTLFLAG_RW,
279     &VNET_NAME(pfsyncstats), pfsyncstats,
280     "PFSYNC statistics (struct pfsyncstats, net/if_pfsync.h)");
281 SYSCTL_INT(_net_pfsync, OID_AUTO, carp_demotion_factor, CTLFLAG_VNET | CTLFLAG_RW,
282     &VNET_NAME(pfsync_carp_adj), 0, "pfsync's CARP demotion factor adjustment");
283 SYSCTL_ULONG(_net_pfsync, OID_AUTO, pfsync_buckets, CTLFLAG_RDTUN,
284     &pfsync_buckets, 0, "Number of pfsync hash buckets");
285 
286 static int	pfsync_clone_create(struct if_clone *, int, caddr_t);
287 static void	pfsync_clone_destroy(struct ifnet *);
288 static int	pfsync_alloc_scrub_memory(struct pfsync_state_peer *,
289 		    struct pf_state_peer *);
290 static int	pfsyncoutput(struct ifnet *, struct mbuf *,
291 		    const struct sockaddr *, struct route *);
292 static int	pfsyncioctl(struct ifnet *, u_long, caddr_t);
293 
294 static int	pfsync_defer(struct pf_kstate *, struct mbuf *);
295 static void	pfsync_undefer(struct pfsync_deferral *, int);
296 static void	pfsync_undefer_state(struct pf_kstate *, int);
297 static void	pfsync_defer_tmo(void *);
298 
299 static void	pfsync_request_update(u_int32_t, u_int64_t);
300 static bool	pfsync_update_state_req(struct pf_kstate *);
301 
302 static void	pfsync_drop(struct pfsync_softc *);
303 static void	pfsync_sendout(int, int);
304 static void	pfsync_send_plus(void *, size_t);
305 
306 static void	pfsync_bulk_start(void);
307 static void	pfsync_bulk_status(u_int8_t);
308 static void	pfsync_bulk_update(void *);
309 static void	pfsync_bulk_fail(void *);
310 
311 static void	pfsync_detach_ifnet(struct ifnet *);
312 #ifdef IPSEC
313 static void	pfsync_update_net_tdb(struct pfsync_tdb *);
314 #endif
315 static struct pfsync_bucket	*pfsync_get_bucket(struct pfsync_softc *,
316 		    struct pf_kstate *);
317 
318 #define PFSYNC_MAX_BULKTRIES	12
319 #define PFSYNC_DEFER_TIMEOUT	((20 * hz) / 1000)
320 
321 VNET_DEFINE(struct if_clone *, pfsync_cloner);
322 #define	V_pfsync_cloner	VNET(pfsync_cloner)
323 
324 static int
325 pfsync_clone_create(struct if_clone *ifc, int unit, caddr_t param)
326 {
327 	struct pfsync_softc *sc;
328 	struct ifnet *ifp;
329 	struct pfsync_bucket *b;
330 	int c, q;
331 
332 	if (unit != 0)
333 		return (EINVAL);
334 
335 	if (! pfsync_buckets)
336 		pfsync_buckets = mp_ncpus * 2;
337 
338 	sc = malloc(sizeof(struct pfsync_softc), M_PFSYNC, M_WAITOK | M_ZERO);
339 	sc->sc_flags |= PFSYNCF_OK;
340 	sc->sc_maxupdates = 128;
341 
342 	ifp = sc->sc_ifp = if_alloc(IFT_PFSYNC);
343 	if (ifp == NULL) {
344 		free(sc, M_PFSYNC);
345 		return (ENOSPC);
346 	}
347 	if_initname(ifp, pfsyncname, unit);
348 	ifp->if_softc = sc;
349 	ifp->if_ioctl = pfsyncioctl;
350 	ifp->if_output = pfsyncoutput;
351 	ifp->if_type = IFT_PFSYNC;
352 	ifp->if_hdrlen = sizeof(struct pfsync_header);
353 	ifp->if_mtu = ETHERMTU;
354 	mtx_init(&sc->sc_mtx, pfsyncname, NULL, MTX_DEF);
355 	mtx_init(&sc->sc_bulk_mtx, "pfsync bulk", NULL, MTX_DEF);
356 	callout_init_mtx(&sc->sc_bulk_tmo, &sc->sc_bulk_mtx, 0);
357 	callout_init_mtx(&sc->sc_bulkfail_tmo, &sc->sc_bulk_mtx, 0);
358 
359 	if_attach(ifp);
360 
361 	bpfattach(ifp, DLT_PFSYNC, PFSYNC_HDRLEN);
362 
363 	sc->sc_buckets = mallocarray(pfsync_buckets, sizeof(*sc->sc_buckets),
364 	    M_PFSYNC, M_ZERO | M_WAITOK);
365 	for (c = 0; c < pfsync_buckets; c++) {
366 		b = &sc->sc_buckets[c];
367 		mtx_init(&b->b_mtx, "pfsync bucket", NULL, MTX_DEF);
368 
369 		b->b_id = c;
370 		b->b_sc = sc;
371 		b->b_len = PFSYNC_MINPKT;
372 
373 		for (q = 0; q < PFSYNC_S_COUNT; q++)
374 			TAILQ_INIT(&b->b_qs[q]);
375 
376 		TAILQ_INIT(&b->b_upd_req_list);
377 		TAILQ_INIT(&b->b_deferrals);
378 
379 		callout_init(&b->b_tmo, 1);
380 
381 		b->b_snd.ifq_maxlen = ifqmaxlen;
382 	}
383 
384 	V_pfsyncif = sc;
385 
386 	return (0);
387 }
388 
389 static void
390 pfsync_clone_destroy(struct ifnet *ifp)
391 {
392 	struct pfsync_softc *sc = ifp->if_softc;
393 	struct pfsync_bucket *b;
394 	int c;
395 
396 	for (c = 0; c < pfsync_buckets; c++) {
397 		b = &sc->sc_buckets[c];
398 		/*
399 		 * At this stage, everything should have already been
400 		 * cleared by pfsync_uninit(), and we have only to
401 		 * drain callouts.
402 		 */
403 		while (b->b_deferred > 0) {
404 			struct pfsync_deferral *pd =
405 			    TAILQ_FIRST(&b->b_deferrals);
406 
407 			TAILQ_REMOVE(&b->b_deferrals, pd, pd_entry);
408 			b->b_deferred--;
409 			if (callout_stop(&pd->pd_tmo) > 0) {
410 				pf_release_state(pd->pd_st);
411 				m_freem(pd->pd_m);
412 				free(pd, M_PFSYNC);
413 			} else {
414 				pd->pd_refs++;
415 				callout_drain(&pd->pd_tmo);
416 				free(pd, M_PFSYNC);
417 			}
418 		}
419 
420 		callout_drain(&b->b_tmo);
421 	}
422 
423 	callout_drain(&sc->sc_bulkfail_tmo);
424 	callout_drain(&sc->sc_bulk_tmo);
425 
426 	if (!(sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
427 		(*carp_demote_adj_p)(-V_pfsync_carp_adj, "pfsync destroy");
428 	bpfdetach(ifp);
429 	if_detach(ifp);
430 
431 	pfsync_drop(sc);
432 
433 	if_free(ifp);
434 	pfsync_multicast_cleanup(sc);
435 	mtx_destroy(&sc->sc_mtx);
436 	mtx_destroy(&sc->sc_bulk_mtx);
437 
438 	free(sc->sc_buckets, M_PFSYNC);
439 	free(sc, M_PFSYNC);
440 
441 	V_pfsyncif = NULL;
442 }
443 
444 static int
445 pfsync_alloc_scrub_memory(struct pfsync_state_peer *s,
446     struct pf_state_peer *d)
447 {
448 	if (s->scrub.scrub_flag && d->scrub == NULL) {
449 		d->scrub = uma_zalloc(V_pf_state_scrub_z, M_NOWAIT | M_ZERO);
450 		if (d->scrub == NULL)
451 			return (ENOMEM);
452 	}
453 
454 	return (0);
455 }
456 
457 static int
458 pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
459 {
460 	struct pfsync_softc *sc = V_pfsyncif;
461 #ifndef	__NO_STRICT_ALIGNMENT
462 	struct pfsync_state_key key[2];
463 #endif
464 	struct pfsync_state_key *kw, *ks;
465 	struct pf_kstate	*st = NULL;
466 	struct pf_state_key *skw = NULL, *sks = NULL;
467 	struct pf_krule *r = NULL;
468 	struct pfi_kkif	*kif;
469 	int error;
470 
471 	PF_RULES_RASSERT();
472 
473 	if (sp->creatorid == 0) {
474 		if (V_pf_status.debug >= PF_DEBUG_MISC)
475 			printf("%s: invalid creator id: %08x\n", __func__,
476 			    ntohl(sp->creatorid));
477 		return (EINVAL);
478 	}
479 
480 	if ((kif = pfi_kkif_find(sp->ifname)) == NULL) {
481 		if (V_pf_status.debug >= PF_DEBUG_MISC)
482 			printf("%s: unknown interface: %s\n", __func__,
483 			    sp->ifname);
484 		if (flags & PFSYNC_SI_IOCTL)
485 			return (EINVAL);
486 		return (0);	/* skip this state */
487 	}
488 
489 	/*
490 	 * If the ruleset checksums match or the state is coming from the ioctl,
491 	 * it's safe to associate the state with the rule of that number.
492 	 */
493 	if (sp->rule != htonl(-1) && sp->anchor == htonl(-1) &&
494 	    (flags & (PFSYNC_SI_IOCTL | PFSYNC_SI_CKSUM)) && ntohl(sp->rule) <
495 	    pf_main_ruleset.rules[PF_RULESET_FILTER].active.rcount)
496 		r = pf_main_ruleset.rules[
497 		    PF_RULESET_FILTER].active.ptr_array[ntohl(sp->rule)];
498 	else
499 		r = &V_pf_default_rule;
500 
501 	if ((r->max_states &&
502 	    counter_u64_fetch(r->states_cur) >= r->max_states))
503 		goto cleanup;
504 
505 	/*
506 	 * XXXGL: consider M_WAITOK in ioctl path after.
507 	 */
508 	st = pf_alloc_state(M_NOWAIT);
509 	if (__predict_false(st == NULL))
510 		goto cleanup;
511 
512 	if ((skw = uma_zalloc(V_pf_state_key_z, M_NOWAIT)) == NULL)
513 		goto cleanup;
514 
515 #ifndef	__NO_STRICT_ALIGNMENT
516 	bcopy(&sp->key, key, sizeof(struct pfsync_state_key) * 2);
517 	kw = &key[PF_SK_WIRE];
518 	ks = &key[PF_SK_STACK];
519 #else
520 	kw = &sp->key[PF_SK_WIRE];
521 	ks = &sp->key[PF_SK_STACK];
522 #endif
523 
524 	if (PF_ANEQ(&kw->addr[0], &ks->addr[0], sp->af) ||
525 	    PF_ANEQ(&kw->addr[1], &ks->addr[1], sp->af) ||
526 	    kw->port[0] != ks->port[0] ||
527 	    kw->port[1] != ks->port[1]) {
528 		sks = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
529 		if (sks == NULL)
530 			goto cleanup;
531 	} else
532 		sks = skw;
533 
534 	/* allocate memory for scrub info */
535 	if (pfsync_alloc_scrub_memory(&sp->src, &st->src) ||
536 	    pfsync_alloc_scrub_memory(&sp->dst, &st->dst))
537 		goto cleanup;
538 
539 	/* Copy to state key(s). */
540 	skw->addr[0] = kw->addr[0];
541 	skw->addr[1] = kw->addr[1];
542 	skw->port[0] = kw->port[0];
543 	skw->port[1] = kw->port[1];
544 	skw->proto = sp->proto;
545 	skw->af = sp->af;
546 	if (sks != skw) {
547 		sks->addr[0] = ks->addr[0];
548 		sks->addr[1] = ks->addr[1];
549 		sks->port[0] = ks->port[0];
550 		sks->port[1] = ks->port[1];
551 		sks->proto = sp->proto;
552 		sks->af = sp->af;
553 	}
554 
555 	/* copy to state */
556 	bcopy(&sp->rt_addr, &st->rt_addr, sizeof(st->rt_addr));
557 	st->creation = time_uptime - ntohl(sp->creation);
558 	st->expire = time_uptime;
559 	if (sp->expire) {
560 		uint32_t timeout;
561 
562 		timeout = r->timeout[sp->timeout];
563 		if (!timeout)
564 			timeout = V_pf_default_rule.timeout[sp->timeout];
565 
566 		/* sp->expire may have been adaptively scaled by export. */
567 		st->expire -= timeout - ntohl(sp->expire);
568 	}
569 
570 	st->direction = sp->direction;
571 	st->log = sp->log;
572 	st->timeout = sp->timeout;
573 	st->state_flags = sp->state_flags;
574 
575 	st->id = sp->id;
576 	st->creatorid = sp->creatorid;
577 	pf_state_peer_ntoh(&sp->src, &st->src);
578 	pf_state_peer_ntoh(&sp->dst, &st->dst);
579 
580 	st->rule.ptr = r;
581 	st->nat_rule.ptr = NULL;
582 	st->anchor.ptr = NULL;
583 	st->rt_kif = NULL;
584 
585 	st->pfsync_time = time_uptime;
586 	st->sync_state = PFSYNC_S_NONE;
587 
588 	if (!(flags & PFSYNC_SI_IOCTL))
589 		st->state_flags |= PFSTATE_NOSYNC;
590 
591 	if ((error = pf_state_insert(kif, kif, skw, sks, st)) != 0)
592 		goto cleanup_state;
593 
594 	/* XXX when we have nat_rule/anchors, use STATE_INC_COUNTERS */
595 	counter_u64_add(r->states_cur, 1);
596 	counter_u64_add(r->states_tot, 1);
597 
598 	if (!(flags & PFSYNC_SI_IOCTL)) {
599 		st->state_flags &= ~PFSTATE_NOSYNC;
600 		if (st->state_flags & PFSTATE_ACK) {
601 			pfsync_q_ins(st, PFSYNC_S_IACK, true);
602 			pfsync_push_all(sc);
603 		}
604 	}
605 	st->state_flags &= ~PFSTATE_ACK;
606 	PF_STATE_UNLOCK(st);
607 
608 	return (0);
609 
610 cleanup:
611 	error = ENOMEM;
612 	if (skw == sks)
613 		sks = NULL;
614 	if (skw != NULL)
615 		uma_zfree(V_pf_state_key_z, skw);
616 	if (sks != NULL)
617 		uma_zfree(V_pf_state_key_z, sks);
618 
619 cleanup_state:	/* pf_state_insert() frees the state keys. */
620 	if (st) {
621 		pf_free_state(st);
622 	}
623 	return (error);
624 }
625 
626 static int
627 pfsync_input(struct mbuf **mp, int *offp __unused, int proto __unused)
628 {
629 	struct pfsync_softc *sc = V_pfsyncif;
630 	struct pfsync_pkt pkt;
631 	struct mbuf *m = *mp;
632 	struct ip *ip = mtod(m, struct ip *);
633 	struct pfsync_header *ph;
634 	struct pfsync_subheader subh;
635 
636 	int offset, len;
637 	int rv;
638 	uint16_t count;
639 
640 	PF_RULES_RLOCK_TRACKER;
641 
642 	*mp = NULL;
643 	V_pfsyncstats.pfsyncs_ipackets++;
644 
645 	/* Verify that we have a sync interface configured. */
646 	if (!sc || !sc->sc_sync_if || !V_pf_status.running ||
647 	    (sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
648 		goto done;
649 
650 	/* verify that the packet came in on the right interface */
651 	if (sc->sc_sync_if != m->m_pkthdr.rcvif) {
652 		V_pfsyncstats.pfsyncs_badif++;
653 		goto done;
654 	}
655 
656 	if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
657 	if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len);
658 	/* verify that the IP TTL is 255. */
659 	if (ip->ip_ttl != PFSYNC_DFLTTL) {
660 		V_pfsyncstats.pfsyncs_badttl++;
661 		goto done;
662 	}
663 
664 	offset = ip->ip_hl << 2;
665 	if (m->m_pkthdr.len < offset + sizeof(*ph)) {
666 		V_pfsyncstats.pfsyncs_hdrops++;
667 		goto done;
668 	}
669 
670 	if (offset + sizeof(*ph) > m->m_len) {
671 		if (m_pullup(m, offset + sizeof(*ph)) == NULL) {
672 			V_pfsyncstats.pfsyncs_hdrops++;
673 			return (IPPROTO_DONE);
674 		}
675 		ip = mtod(m, struct ip *);
676 	}
677 	ph = (struct pfsync_header *)((char *)ip + offset);
678 
679 	/* verify the version */
680 	if (ph->version != PFSYNC_VERSION) {
681 		V_pfsyncstats.pfsyncs_badver++;
682 		goto done;
683 	}
684 
685 	len = ntohs(ph->len) + offset;
686 	if (m->m_pkthdr.len < len) {
687 		V_pfsyncstats.pfsyncs_badlen++;
688 		goto done;
689 	}
690 
691 	/* Cheaper to grab this now than having to mess with mbufs later */
692 	pkt.ip = ip;
693 	pkt.src = ip->ip_src;
694 	pkt.flags = 0;
695 
696 	/*
697 	 * Trusting pf_chksum during packet processing, as well as seeking
698 	 * in interface name tree, require holding PF_RULES_RLOCK().
699 	 */
700 	PF_RULES_RLOCK();
701 	if (!bcmp(&ph->pfcksum, &V_pf_status.pf_chksum, PF_MD5_DIGEST_LENGTH))
702 		pkt.flags |= PFSYNC_SI_CKSUM;
703 
704 	offset += sizeof(*ph);
705 	while (offset <= len - sizeof(subh)) {
706 		m_copydata(m, offset, sizeof(subh), (caddr_t)&subh);
707 		offset += sizeof(subh);
708 
709 		if (subh.action >= PFSYNC_ACT_MAX) {
710 			V_pfsyncstats.pfsyncs_badact++;
711 			PF_RULES_RUNLOCK();
712 			goto done;
713 		}
714 
715 		count = ntohs(subh.count);
716 		V_pfsyncstats.pfsyncs_iacts[subh.action] += count;
717 		rv = (*pfsync_acts[subh.action])(&pkt, m, offset, count);
718 		if (rv == -1) {
719 			PF_RULES_RUNLOCK();
720 			return (IPPROTO_DONE);
721 		}
722 
723 		offset += rv;
724 	}
725 	PF_RULES_RUNLOCK();
726 
727 done:
728 	m_freem(m);
729 	return (IPPROTO_DONE);
730 }
731 
732 static int
733 pfsync_in_clr(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
734 {
735 	struct pfsync_clr *clr;
736 	struct mbuf *mp;
737 	int len = sizeof(*clr) * count;
738 	int i, offp;
739 	u_int32_t creatorid;
740 
741 	mp = m_pulldown(m, offset, len, &offp);
742 	if (mp == NULL) {
743 		V_pfsyncstats.pfsyncs_badlen++;
744 		return (-1);
745 	}
746 	clr = (struct pfsync_clr *)(mp->m_data + offp);
747 
748 	for (i = 0; i < count; i++) {
749 		creatorid = clr[i].creatorid;
750 
751 		if (clr[i].ifname[0] != '\0' &&
752 		    pfi_kkif_find(clr[i].ifname) == NULL)
753 			continue;
754 
755 		for (int i = 0; i <= pf_hashmask; i++) {
756 			struct pf_idhash *ih = &V_pf_idhash[i];
757 			struct pf_kstate *s;
758 relock:
759 			PF_HASHROW_LOCK(ih);
760 			LIST_FOREACH(s, &ih->states, entry) {
761 				if (s->creatorid == creatorid) {
762 					s->state_flags |= PFSTATE_NOSYNC;
763 					pf_unlink_state(s);
764 					goto relock;
765 				}
766 			}
767 			PF_HASHROW_UNLOCK(ih);
768 		}
769 	}
770 
771 	return (len);
772 }
773 
774 static int
775 pfsync_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
776 {
777 	struct mbuf *mp;
778 	struct pfsync_state *sa, *sp;
779 	int len = sizeof(*sp) * count;
780 	int i, offp;
781 
782 	mp = m_pulldown(m, offset, len, &offp);
783 	if (mp == NULL) {
784 		V_pfsyncstats.pfsyncs_badlen++;
785 		return (-1);
786 	}
787 	sa = (struct pfsync_state *)(mp->m_data + offp);
788 
789 	for (i = 0; i < count; i++) {
790 		sp = &sa[i];
791 
792 		/* Check for invalid values. */
793 		if (sp->timeout >= PFTM_MAX ||
794 		    sp->src.state > PF_TCPS_PROXY_DST ||
795 		    sp->dst.state > PF_TCPS_PROXY_DST ||
796 		    sp->direction > PF_OUT ||
797 		    (sp->af != AF_INET && sp->af != AF_INET6)) {
798 			if (V_pf_status.debug >= PF_DEBUG_MISC)
799 				printf("%s: invalid value\n", __func__);
800 			V_pfsyncstats.pfsyncs_badval++;
801 			continue;
802 		}
803 
804 		if (pfsync_state_import(sp, pkt->flags) == ENOMEM)
805 			/* Drop out, but process the rest of the actions. */
806 			break;
807 	}
808 
809 	return (len);
810 }
811 
812 static int
813 pfsync_in_iack(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
814 {
815 	struct pfsync_ins_ack *ia, *iaa;
816 	struct pf_kstate *st;
817 
818 	struct mbuf *mp;
819 	int len = count * sizeof(*ia);
820 	int offp, i;
821 
822 	mp = m_pulldown(m, offset, len, &offp);
823 	if (mp == NULL) {
824 		V_pfsyncstats.pfsyncs_badlen++;
825 		return (-1);
826 	}
827 	iaa = (struct pfsync_ins_ack *)(mp->m_data + offp);
828 
829 	for (i = 0; i < count; i++) {
830 		ia = &iaa[i];
831 
832 		st = pf_find_state_byid(ia->id, ia->creatorid);
833 		if (st == NULL)
834 			continue;
835 
836 		if (st->state_flags & PFSTATE_ACK) {
837 			pfsync_undefer_state(st, 0);
838 		}
839 		PF_STATE_UNLOCK(st);
840 	}
841 	/*
842 	 * XXX this is not yet implemented, but we know the size of the
843 	 * message so we can skip it.
844 	 */
845 
846 	return (count * sizeof(struct pfsync_ins_ack));
847 }
848 
849 static int
850 pfsync_upd_tcp(struct pf_kstate *st, struct pfsync_state_peer *src,
851     struct pfsync_state_peer *dst)
852 {
853 	int sync = 0;
854 
855 	PF_STATE_LOCK_ASSERT(st);
856 
857 	/*
858 	 * The state should never go backwards except
859 	 * for syn-proxy states.  Neither should the
860 	 * sequence window slide backwards.
861 	 */
862 	if ((st->src.state > src->state &&
863 	    (st->src.state < PF_TCPS_PROXY_SRC ||
864 	    src->state >= PF_TCPS_PROXY_SRC)) ||
865 
866 	    (st->src.state == src->state &&
867 	    SEQ_GT(st->src.seqlo, ntohl(src->seqlo))))
868 		sync++;
869 	else
870 		pf_state_peer_ntoh(src, &st->src);
871 
872 	if ((st->dst.state > dst->state) ||
873 
874 	    (st->dst.state >= TCPS_SYN_SENT &&
875 	    SEQ_GT(st->dst.seqlo, ntohl(dst->seqlo))))
876 		sync++;
877 	else
878 		pf_state_peer_ntoh(dst, &st->dst);
879 
880 	return (sync);
881 }
882 
883 static int
884 pfsync_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
885 {
886 	struct pfsync_softc *sc = V_pfsyncif;
887 	struct pfsync_state *sa, *sp;
888 	struct pf_kstate *st;
889 	int sync;
890 
891 	struct mbuf *mp;
892 	int len = count * sizeof(*sp);
893 	int offp, i;
894 
895 	mp = m_pulldown(m, offset, len, &offp);
896 	if (mp == NULL) {
897 		V_pfsyncstats.pfsyncs_badlen++;
898 		return (-1);
899 	}
900 	sa = (struct pfsync_state *)(mp->m_data + offp);
901 
902 	for (i = 0; i < count; i++) {
903 		sp = &sa[i];
904 
905 		/* check for invalid values */
906 		if (sp->timeout >= PFTM_MAX ||
907 		    sp->src.state > PF_TCPS_PROXY_DST ||
908 		    sp->dst.state > PF_TCPS_PROXY_DST) {
909 			if (V_pf_status.debug >= PF_DEBUG_MISC) {
910 				printf("pfsync_input: PFSYNC_ACT_UPD: "
911 				    "invalid value\n");
912 			}
913 			V_pfsyncstats.pfsyncs_badval++;
914 			continue;
915 		}
916 
917 		st = pf_find_state_byid(sp->id, sp->creatorid);
918 		if (st == NULL) {
919 			/* insert the update */
920 			if (pfsync_state_import(sp, pkt->flags))
921 				V_pfsyncstats.pfsyncs_badstate++;
922 			continue;
923 		}
924 
925 		if (st->state_flags & PFSTATE_ACK) {
926 			pfsync_undefer_state(st, 1);
927 		}
928 
929 		if (st->key[PF_SK_WIRE]->proto == IPPROTO_TCP)
930 			sync = pfsync_upd_tcp(st, &sp->src, &sp->dst);
931 		else {
932 			sync = 0;
933 
934 			/*
935 			 * Non-TCP protocol state machine always go
936 			 * forwards
937 			 */
938 			if (st->src.state > sp->src.state)
939 				sync++;
940 			else
941 				pf_state_peer_ntoh(&sp->src, &st->src);
942 			if (st->dst.state > sp->dst.state)
943 				sync++;
944 			else
945 				pf_state_peer_ntoh(&sp->dst, &st->dst);
946 		}
947 		if (sync < 2) {
948 			pfsync_alloc_scrub_memory(&sp->dst, &st->dst);
949 			pf_state_peer_ntoh(&sp->dst, &st->dst);
950 			st->expire = time_uptime;
951 			st->timeout = sp->timeout;
952 		}
953 		st->pfsync_time = time_uptime;
954 
955 		if (sync) {
956 			V_pfsyncstats.pfsyncs_stale++;
957 
958 			pfsync_update_state(st);
959 			PF_STATE_UNLOCK(st);
960 			pfsync_push_all(sc);
961 			continue;
962 		}
963 		PF_STATE_UNLOCK(st);
964 	}
965 
966 	return (len);
967 }
968 
969 static int
970 pfsync_in_upd_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
971 {
972 	struct pfsync_softc *sc = V_pfsyncif;
973 	struct pfsync_upd_c *ua, *up;
974 	struct pf_kstate *st;
975 	int len = count * sizeof(*up);
976 	int sync;
977 	struct mbuf *mp;
978 	int offp, i;
979 
980 	mp = m_pulldown(m, offset, len, &offp);
981 	if (mp == NULL) {
982 		V_pfsyncstats.pfsyncs_badlen++;
983 		return (-1);
984 	}
985 	ua = (struct pfsync_upd_c *)(mp->m_data + offp);
986 
987 	for (i = 0; i < count; i++) {
988 		up = &ua[i];
989 
990 		/* check for invalid values */
991 		if (up->timeout >= PFTM_MAX ||
992 		    up->src.state > PF_TCPS_PROXY_DST ||
993 		    up->dst.state > PF_TCPS_PROXY_DST) {
994 			if (V_pf_status.debug >= PF_DEBUG_MISC) {
995 				printf("pfsync_input: "
996 				    "PFSYNC_ACT_UPD_C: "
997 				    "invalid value\n");
998 			}
999 			V_pfsyncstats.pfsyncs_badval++;
1000 			continue;
1001 		}
1002 
1003 		st = pf_find_state_byid(up->id, up->creatorid);
1004 		if (st == NULL) {
1005 			/* We don't have this state. Ask for it. */
1006 			PFSYNC_BUCKET_LOCK(&sc->sc_buckets[0]);
1007 			pfsync_request_update(up->creatorid, up->id);
1008 			PFSYNC_BUCKET_UNLOCK(&sc->sc_buckets[0]);
1009 			continue;
1010 		}
1011 
1012 		if (st->state_flags & PFSTATE_ACK) {
1013 			pfsync_undefer_state(st, 1);
1014 		}
1015 
1016 		if (st->key[PF_SK_WIRE]->proto == IPPROTO_TCP)
1017 			sync = pfsync_upd_tcp(st, &up->src, &up->dst);
1018 		else {
1019 			sync = 0;
1020 
1021 			/*
1022 			 * Non-TCP protocol state machine always go
1023 			 * forwards
1024 			 */
1025 			if (st->src.state > up->src.state)
1026 				sync++;
1027 			else
1028 				pf_state_peer_ntoh(&up->src, &st->src);
1029 			if (st->dst.state > up->dst.state)
1030 				sync++;
1031 			else
1032 				pf_state_peer_ntoh(&up->dst, &st->dst);
1033 		}
1034 		if (sync < 2) {
1035 			pfsync_alloc_scrub_memory(&up->dst, &st->dst);
1036 			pf_state_peer_ntoh(&up->dst, &st->dst);
1037 			st->expire = time_uptime;
1038 			st->timeout = up->timeout;
1039 		}
1040 		st->pfsync_time = time_uptime;
1041 
1042 		if (sync) {
1043 			V_pfsyncstats.pfsyncs_stale++;
1044 
1045 			pfsync_update_state(st);
1046 			PF_STATE_UNLOCK(st);
1047 			pfsync_push_all(sc);
1048 			continue;
1049 		}
1050 		PF_STATE_UNLOCK(st);
1051 	}
1052 
1053 	return (len);
1054 }
1055 
1056 static int
1057 pfsync_in_ureq(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1058 {
1059 	struct pfsync_upd_req *ur, *ura;
1060 	struct mbuf *mp;
1061 	int len = count * sizeof(*ur);
1062 	int i, offp;
1063 
1064 	struct pf_kstate *st;
1065 
1066 	mp = m_pulldown(m, offset, len, &offp);
1067 	if (mp == NULL) {
1068 		V_pfsyncstats.pfsyncs_badlen++;
1069 		return (-1);
1070 	}
1071 	ura = (struct pfsync_upd_req *)(mp->m_data + offp);
1072 
1073 	for (i = 0; i < count; i++) {
1074 		ur = &ura[i];
1075 
1076 		if (ur->id == 0 && ur->creatorid == 0)
1077 			pfsync_bulk_start();
1078 		else {
1079 			st = pf_find_state_byid(ur->id, ur->creatorid);
1080 			if (st == NULL) {
1081 				V_pfsyncstats.pfsyncs_badstate++;
1082 				continue;
1083 			}
1084 			if (st->state_flags & PFSTATE_NOSYNC) {
1085 				PF_STATE_UNLOCK(st);
1086 				continue;
1087 			}
1088 
1089 			pfsync_update_state_req(st);
1090 			PF_STATE_UNLOCK(st);
1091 		}
1092 	}
1093 
1094 	return (len);
1095 }
1096 
1097 static int
1098 pfsync_in_del(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1099 {
1100 	struct mbuf *mp;
1101 	struct pfsync_state *sa, *sp;
1102 	struct pf_kstate *st;
1103 	int len = count * sizeof(*sp);
1104 	int offp, i;
1105 
1106 	mp = m_pulldown(m, offset, len, &offp);
1107 	if (mp == NULL) {
1108 		V_pfsyncstats.pfsyncs_badlen++;
1109 		return (-1);
1110 	}
1111 	sa = (struct pfsync_state *)(mp->m_data + offp);
1112 
1113 	for (i = 0; i < count; i++) {
1114 		sp = &sa[i];
1115 
1116 		st = pf_find_state_byid(sp->id, sp->creatorid);
1117 		if (st == NULL) {
1118 			V_pfsyncstats.pfsyncs_badstate++;
1119 			continue;
1120 		}
1121 		st->state_flags |= PFSTATE_NOSYNC;
1122 		pf_unlink_state(st);
1123 	}
1124 
1125 	return (len);
1126 }
1127 
1128 static int
1129 pfsync_in_del_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1130 {
1131 	struct mbuf *mp;
1132 	struct pfsync_del_c *sa, *sp;
1133 	struct pf_kstate *st;
1134 	int len = count * sizeof(*sp);
1135 	int offp, i;
1136 
1137 	mp = m_pulldown(m, offset, len, &offp);
1138 	if (mp == NULL) {
1139 		V_pfsyncstats.pfsyncs_badlen++;
1140 		return (-1);
1141 	}
1142 	sa = (struct pfsync_del_c *)(mp->m_data + offp);
1143 
1144 	for (i = 0; i < count; i++) {
1145 		sp = &sa[i];
1146 
1147 		st = pf_find_state_byid(sp->id, sp->creatorid);
1148 		if (st == NULL) {
1149 			V_pfsyncstats.pfsyncs_badstate++;
1150 			continue;
1151 		}
1152 
1153 		st->state_flags |= PFSTATE_NOSYNC;
1154 		pf_unlink_state(st);
1155 	}
1156 
1157 	return (len);
1158 }
1159 
1160 static int
1161 pfsync_in_bus(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1162 {
1163 	struct pfsync_softc *sc = V_pfsyncif;
1164 	struct pfsync_bus *bus;
1165 	struct mbuf *mp;
1166 	int len = count * sizeof(*bus);
1167 	int offp;
1168 
1169 	PFSYNC_BLOCK(sc);
1170 
1171 	/* If we're not waiting for a bulk update, who cares. */
1172 	if (sc->sc_ureq_sent == 0) {
1173 		PFSYNC_BUNLOCK(sc);
1174 		return (len);
1175 	}
1176 
1177 	mp = m_pulldown(m, offset, len, &offp);
1178 	if (mp == NULL) {
1179 		PFSYNC_BUNLOCK(sc);
1180 		V_pfsyncstats.pfsyncs_badlen++;
1181 		return (-1);
1182 	}
1183 	bus = (struct pfsync_bus *)(mp->m_data + offp);
1184 
1185 	switch (bus->status) {
1186 	case PFSYNC_BUS_START:
1187 		callout_reset(&sc->sc_bulkfail_tmo, 4 * hz +
1188 		    V_pf_limits[PF_LIMIT_STATES].limit /
1189 		    ((sc->sc_ifp->if_mtu - PFSYNC_MINPKT) /
1190 		    sizeof(struct pfsync_state)),
1191 		    pfsync_bulk_fail, sc);
1192 		if (V_pf_status.debug >= PF_DEBUG_MISC)
1193 			printf("pfsync: received bulk update start\n");
1194 		break;
1195 
1196 	case PFSYNC_BUS_END:
1197 		if (time_uptime - ntohl(bus->endtime) >=
1198 		    sc->sc_ureq_sent) {
1199 			/* that's it, we're happy */
1200 			sc->sc_ureq_sent = 0;
1201 			sc->sc_bulk_tries = 0;
1202 			callout_stop(&sc->sc_bulkfail_tmo);
1203 			if (!(sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
1204 				(*carp_demote_adj_p)(-V_pfsync_carp_adj,
1205 				    "pfsync bulk done");
1206 			sc->sc_flags |= PFSYNCF_OK;
1207 			if (V_pf_status.debug >= PF_DEBUG_MISC)
1208 				printf("pfsync: received valid "
1209 				    "bulk update end\n");
1210 		} else {
1211 			if (V_pf_status.debug >= PF_DEBUG_MISC)
1212 				printf("pfsync: received invalid "
1213 				    "bulk update end: bad timestamp\n");
1214 		}
1215 		break;
1216 	}
1217 	PFSYNC_BUNLOCK(sc);
1218 
1219 	return (len);
1220 }
1221 
1222 static int
1223 pfsync_in_tdb(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1224 {
1225 	int len = count * sizeof(struct pfsync_tdb);
1226 
1227 #if defined(IPSEC)
1228 	struct pfsync_tdb *tp;
1229 	struct mbuf *mp;
1230 	int offp;
1231 	int i;
1232 	int s;
1233 
1234 	mp = m_pulldown(m, offset, len, &offp);
1235 	if (mp == NULL) {
1236 		V_pfsyncstats.pfsyncs_badlen++;
1237 		return (-1);
1238 	}
1239 	tp = (struct pfsync_tdb *)(mp->m_data + offp);
1240 
1241 	for (i = 0; i < count; i++)
1242 		pfsync_update_net_tdb(&tp[i]);
1243 #endif
1244 
1245 	return (len);
1246 }
1247 
1248 #if defined(IPSEC)
1249 /* Update an in-kernel tdb. Silently fail if no tdb is found. */
1250 static void
1251 pfsync_update_net_tdb(struct pfsync_tdb *pt)
1252 {
1253 	struct tdb		*tdb;
1254 	int			 s;
1255 
1256 	/* check for invalid values */
1257 	if (ntohl(pt->spi) <= SPI_RESERVED_MAX ||
1258 	    (pt->dst.sa.sa_family != AF_INET &&
1259 	    pt->dst.sa.sa_family != AF_INET6))
1260 		goto bad;
1261 
1262 	tdb = gettdb(pt->spi, &pt->dst, pt->sproto);
1263 	if (tdb) {
1264 		pt->rpl = ntohl(pt->rpl);
1265 		pt->cur_bytes = (unsigned long long)be64toh(pt->cur_bytes);
1266 
1267 		/* Neither replay nor byte counter should ever decrease. */
1268 		if (pt->rpl < tdb->tdb_rpl ||
1269 		    pt->cur_bytes < tdb->tdb_cur_bytes) {
1270 			goto bad;
1271 		}
1272 
1273 		tdb->tdb_rpl = pt->rpl;
1274 		tdb->tdb_cur_bytes = pt->cur_bytes;
1275 	}
1276 	return;
1277 
1278 bad:
1279 	if (V_pf_status.debug >= PF_DEBUG_MISC)
1280 		printf("pfsync_insert: PFSYNC_ACT_TDB_UPD: "
1281 		    "invalid value\n");
1282 	V_pfsyncstats.pfsyncs_badstate++;
1283 	return;
1284 }
1285 #endif
1286 
1287 static int
1288 pfsync_in_eof(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1289 {
1290 	/* check if we are at the right place in the packet */
1291 	if (offset != m->m_pkthdr.len)
1292 		V_pfsyncstats.pfsyncs_badlen++;
1293 
1294 	/* we're done. free and let the caller return */
1295 	m_freem(m);
1296 	return (-1);
1297 }
1298 
1299 static int
1300 pfsync_in_error(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
1301 {
1302 	V_pfsyncstats.pfsyncs_badact++;
1303 
1304 	m_freem(m);
1305 	return (-1);
1306 }
1307 
1308 static int
1309 pfsyncoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
1310 	struct route *rt)
1311 {
1312 	m_freem(m);
1313 	return (0);
1314 }
1315 
1316 /* ARGSUSED */
1317 static int
1318 pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
1319 {
1320 	struct pfsync_softc *sc = ifp->if_softc;
1321 	struct ifreq *ifr = (struct ifreq *)data;
1322 	struct pfsyncreq pfsyncr;
1323 	int error;
1324 	int c;
1325 
1326 	switch (cmd) {
1327 	case SIOCSIFFLAGS:
1328 		PFSYNC_LOCK(sc);
1329 		if (ifp->if_flags & IFF_UP) {
1330 			ifp->if_drv_flags |= IFF_DRV_RUNNING;
1331 			PFSYNC_UNLOCK(sc);
1332 			pfsync_pointers_init();
1333 		} else {
1334 			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1335 			PFSYNC_UNLOCK(sc);
1336 			pfsync_pointers_uninit();
1337 		}
1338 		break;
1339 	case SIOCSIFMTU:
1340 		if (!sc->sc_sync_if ||
1341 		    ifr->ifr_mtu <= PFSYNC_MINPKT ||
1342 		    ifr->ifr_mtu > sc->sc_sync_if->if_mtu)
1343 			return (EINVAL);
1344 		if (ifr->ifr_mtu < ifp->if_mtu) {
1345 			for (c = 0; c < pfsync_buckets; c++) {
1346 				PFSYNC_BUCKET_LOCK(&sc->sc_buckets[c]);
1347 				if (sc->sc_buckets[c].b_len > PFSYNC_MINPKT)
1348 					pfsync_sendout(1, c);
1349 				PFSYNC_BUCKET_UNLOCK(&sc->sc_buckets[c]);
1350 			}
1351 		}
1352 		ifp->if_mtu = ifr->ifr_mtu;
1353 		break;
1354 	case SIOCGETPFSYNC:
1355 		bzero(&pfsyncr, sizeof(pfsyncr));
1356 		PFSYNC_LOCK(sc);
1357 		if (sc->sc_sync_if) {
1358 			strlcpy(pfsyncr.pfsyncr_syncdev,
1359 			    sc->sc_sync_if->if_xname, IFNAMSIZ);
1360 		}
1361 		pfsyncr.pfsyncr_syncpeer = sc->sc_sync_peer;
1362 		pfsyncr.pfsyncr_maxupdates = sc->sc_maxupdates;
1363 		pfsyncr.pfsyncr_defer = sc->sc_flags;
1364 		PFSYNC_UNLOCK(sc);
1365 		return (copyout(&pfsyncr, ifr_data_get_ptr(ifr),
1366 		    sizeof(pfsyncr)));
1367 
1368 	case SIOCSETPFSYNC:
1369 	    {
1370 		struct in_mfilter *imf = NULL;
1371 		struct ifnet *sifp;
1372 		struct ip *ip;
1373 
1374 		if ((error = priv_check(curthread, PRIV_NETINET_PF)) != 0)
1375 			return (error);
1376 		if ((error = copyin(ifr_data_get_ptr(ifr), &pfsyncr,
1377 		    sizeof(pfsyncr))))
1378 			return (error);
1379 
1380 		if (pfsyncr.pfsyncr_maxupdates > 255)
1381 			return (EINVAL);
1382 
1383 		if (pfsyncr.pfsyncr_syncdev[0] == 0)
1384 			sifp = NULL;
1385 		else if ((sifp = ifunit_ref(pfsyncr.pfsyncr_syncdev)) == NULL)
1386 			return (EINVAL);
1387 
1388 		if (sifp != NULL && (
1389 		    pfsyncr.pfsyncr_syncpeer.s_addr == 0 ||
1390 		    pfsyncr.pfsyncr_syncpeer.s_addr ==
1391 		    htonl(INADDR_PFSYNC_GROUP)))
1392 			imf = ip_mfilter_alloc(M_WAITOK, 0, 0);
1393 
1394 		PFSYNC_LOCK(sc);
1395 		if (pfsyncr.pfsyncr_syncpeer.s_addr == 0)
1396 			sc->sc_sync_peer.s_addr = htonl(INADDR_PFSYNC_GROUP);
1397 		else
1398 			sc->sc_sync_peer.s_addr =
1399 			    pfsyncr.pfsyncr_syncpeer.s_addr;
1400 
1401 		sc->sc_maxupdates = pfsyncr.pfsyncr_maxupdates;
1402 		if (pfsyncr.pfsyncr_defer & PFSYNCF_DEFER) {
1403 			sc->sc_flags |= PFSYNCF_DEFER;
1404 			V_pfsync_defer_ptr = pfsync_defer;
1405 		} else {
1406 			sc->sc_flags &= ~PFSYNCF_DEFER;
1407 			V_pfsync_defer_ptr = NULL;
1408 		}
1409 
1410 		if (sifp == NULL) {
1411 			if (sc->sc_sync_if)
1412 				if_rele(sc->sc_sync_if);
1413 			sc->sc_sync_if = NULL;
1414 			pfsync_multicast_cleanup(sc);
1415 			PFSYNC_UNLOCK(sc);
1416 			break;
1417 		}
1418 
1419 		for (c = 0; c < pfsync_buckets; c++) {
1420 			PFSYNC_BUCKET_LOCK(&sc->sc_buckets[c]);
1421 			if (sc->sc_buckets[c].b_len > PFSYNC_MINPKT &&
1422 			    (sifp->if_mtu < sc->sc_ifp->if_mtu ||
1423 			    (sc->sc_sync_if != NULL &&
1424 			    sifp->if_mtu < sc->sc_sync_if->if_mtu) ||
1425 			    sifp->if_mtu < MCLBYTES - sizeof(struct ip)))
1426 				pfsync_sendout(1, c);
1427 			PFSYNC_BUCKET_UNLOCK(&sc->sc_buckets[c]);
1428 		}
1429 
1430 		pfsync_multicast_cleanup(sc);
1431 
1432 		if (sc->sc_sync_peer.s_addr == htonl(INADDR_PFSYNC_GROUP)) {
1433 			error = pfsync_multicast_setup(sc, sifp, imf);
1434 			if (error) {
1435 				if_rele(sifp);
1436 				ip_mfilter_free(imf);
1437 				PFSYNC_UNLOCK(sc);
1438 				return (error);
1439 			}
1440 		}
1441 		if (sc->sc_sync_if)
1442 			if_rele(sc->sc_sync_if);
1443 		sc->sc_sync_if = sifp;
1444 
1445 		ip = &sc->sc_template;
1446 		bzero(ip, sizeof(*ip));
1447 		ip->ip_v = IPVERSION;
1448 		ip->ip_hl = sizeof(sc->sc_template) >> 2;
1449 		ip->ip_tos = IPTOS_LOWDELAY;
1450 		/* len and id are set later. */
1451 		ip->ip_off = htons(IP_DF);
1452 		ip->ip_ttl = PFSYNC_DFLTTL;
1453 		ip->ip_p = IPPROTO_PFSYNC;
1454 		ip->ip_src.s_addr = INADDR_ANY;
1455 		ip->ip_dst.s_addr = sc->sc_sync_peer.s_addr;
1456 
1457 		/* Request a full state table update. */
1458 		if ((sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
1459 			(*carp_demote_adj_p)(V_pfsync_carp_adj,
1460 			    "pfsync bulk start");
1461 		sc->sc_flags &= ~PFSYNCF_OK;
1462 		if (V_pf_status.debug >= PF_DEBUG_MISC)
1463 			printf("pfsync: requesting bulk update\n");
1464 		PFSYNC_UNLOCK(sc);
1465 		PFSYNC_BUCKET_LOCK(&sc->sc_buckets[0]);
1466 		pfsync_request_update(0, 0);
1467 		PFSYNC_BUCKET_UNLOCK(&sc->sc_buckets[0]);
1468 		PFSYNC_BLOCK(sc);
1469 		sc->sc_ureq_sent = time_uptime;
1470 		callout_reset(&sc->sc_bulkfail_tmo, 5 * hz, pfsync_bulk_fail,
1471 		    sc);
1472 		PFSYNC_BUNLOCK(sc);
1473 
1474 		break;
1475 	    }
1476 	default:
1477 		return (ENOTTY);
1478 	}
1479 
1480 	return (0);
1481 }
1482 
1483 static void
1484 pfsync_out_state(struct pf_kstate *st, void *buf)
1485 {
1486 	struct pfsync_state *sp = buf;
1487 
1488 	pfsync_state_export(sp, st);
1489 }
1490 
1491 static void
1492 pfsync_out_iack(struct pf_kstate *st, void *buf)
1493 {
1494 	struct pfsync_ins_ack *iack = buf;
1495 
1496 	iack->id = st->id;
1497 	iack->creatorid = st->creatorid;
1498 }
1499 
1500 static void
1501 pfsync_out_upd_c(struct pf_kstate *st, void *buf)
1502 {
1503 	struct pfsync_upd_c *up = buf;
1504 
1505 	bzero(up, sizeof(*up));
1506 	up->id = st->id;
1507 	pf_state_peer_hton(&st->src, &up->src);
1508 	pf_state_peer_hton(&st->dst, &up->dst);
1509 	up->creatorid = st->creatorid;
1510 	up->timeout = st->timeout;
1511 }
1512 
1513 static void
1514 pfsync_out_del(struct pf_kstate *st, void *buf)
1515 {
1516 	struct pfsync_del_c *dp = buf;
1517 
1518 	dp->id = st->id;
1519 	dp->creatorid = st->creatorid;
1520 	st->state_flags |= PFSTATE_NOSYNC;
1521 }
1522 
1523 static void
1524 pfsync_drop(struct pfsync_softc *sc)
1525 {
1526 	struct pf_kstate *st, *next;
1527 	struct pfsync_upd_req_item *ur;
1528 	struct pfsync_bucket *b;
1529 	int c, q;
1530 
1531 	for (c = 0; c < pfsync_buckets; c++) {
1532 		b = &sc->sc_buckets[c];
1533 		for (q = 0; q < PFSYNC_S_COUNT; q++) {
1534 			if (TAILQ_EMPTY(&b->b_qs[q]))
1535 				continue;
1536 
1537 			TAILQ_FOREACH_SAFE(st, &b->b_qs[q], sync_list, next) {
1538 				KASSERT(st->sync_state == q,
1539 					("%s: st->sync_state == q",
1540 						__func__));
1541 				st->sync_state = PFSYNC_S_NONE;
1542 				pf_release_state(st);
1543 			}
1544 			TAILQ_INIT(&b->b_qs[q]);
1545 		}
1546 
1547 		while ((ur = TAILQ_FIRST(&b->b_upd_req_list)) != NULL) {
1548 			TAILQ_REMOVE(&b->b_upd_req_list, ur, ur_entry);
1549 			free(ur, M_PFSYNC);
1550 		}
1551 
1552 		b->b_len = PFSYNC_MINPKT;
1553 		b->b_plus = NULL;
1554 	}
1555 }
1556 
1557 static void
1558 pfsync_sendout(int schedswi, int c)
1559 {
1560 	struct pfsync_softc *sc = V_pfsyncif;
1561 	struct ifnet *ifp = sc->sc_ifp;
1562 	struct mbuf *m;
1563 	struct ip *ip;
1564 	struct pfsync_header *ph;
1565 	struct pfsync_subheader *subh;
1566 	struct pf_kstate *st, *st_next;
1567 	struct pfsync_upd_req_item *ur;
1568 	struct pfsync_bucket *b = &sc->sc_buckets[c];
1569 	int offset;
1570 	int q, count = 0;
1571 
1572 	KASSERT(sc != NULL, ("%s: null sc", __func__));
1573 	KASSERT(b->b_len > PFSYNC_MINPKT,
1574 	    ("%s: sc_len %zu", __func__, b->b_len));
1575 	PFSYNC_BUCKET_LOCK_ASSERT(b);
1576 
1577 	if (ifp->if_bpf == NULL && sc->sc_sync_if == NULL) {
1578 		pfsync_drop(sc);
1579 		return;
1580 	}
1581 
1582 	m = m_get2(max_linkhdr + b->b_len, M_NOWAIT, MT_DATA, M_PKTHDR);
1583 	if (m == NULL) {
1584 		if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1);
1585 		V_pfsyncstats.pfsyncs_onomem++;
1586 		return;
1587 	}
1588 	m->m_data += max_linkhdr;
1589 	m->m_len = m->m_pkthdr.len = b->b_len;
1590 
1591 	/* build the ip header */
1592 	ip = (struct ip *)m->m_data;
1593 	bcopy(&sc->sc_template, ip, sizeof(*ip));
1594 	offset = sizeof(*ip);
1595 
1596 	ip->ip_len = htons(m->m_pkthdr.len);
1597 	ip_fillid(ip);
1598 
1599 	/* build the pfsync header */
1600 	ph = (struct pfsync_header *)(m->m_data + offset);
1601 	bzero(ph, sizeof(*ph));
1602 	offset += sizeof(*ph);
1603 
1604 	ph->version = PFSYNC_VERSION;
1605 	ph->len = htons(b->b_len - sizeof(*ip));
1606 	bcopy(V_pf_status.pf_chksum, ph->pfcksum, PF_MD5_DIGEST_LENGTH);
1607 
1608 	/* walk the queues */
1609 	for (q = 0; q < PFSYNC_S_COUNT; q++) {
1610 		if (TAILQ_EMPTY(&b->b_qs[q]))
1611 			continue;
1612 
1613 		subh = (struct pfsync_subheader *)(m->m_data + offset);
1614 		offset += sizeof(*subh);
1615 
1616 		count = 0;
1617 		TAILQ_FOREACH_SAFE(st, &b->b_qs[q], sync_list, st_next) {
1618 			KASSERT(st->sync_state == q,
1619 				("%s: st->sync_state == q",
1620 					__func__));
1621 			/*
1622 			 * XXXGL: some of write methods do unlocked reads
1623 			 * of state data :(
1624 			 */
1625 			pfsync_qs[q].write(st, m->m_data + offset);
1626 			offset += pfsync_qs[q].len;
1627 			st->sync_state = PFSYNC_S_NONE;
1628 			pf_release_state(st);
1629 			count++;
1630 		}
1631 		TAILQ_INIT(&b->b_qs[q]);
1632 
1633 		bzero(subh, sizeof(*subh));
1634 		subh->action = pfsync_qs[q].action;
1635 		subh->count = htons(count);
1636 		V_pfsyncstats.pfsyncs_oacts[pfsync_qs[q].action] += count;
1637 	}
1638 
1639 	if (!TAILQ_EMPTY(&b->b_upd_req_list)) {
1640 		subh = (struct pfsync_subheader *)(m->m_data + offset);
1641 		offset += sizeof(*subh);
1642 
1643 		count = 0;
1644 		while ((ur = TAILQ_FIRST(&b->b_upd_req_list)) != NULL) {
1645 			TAILQ_REMOVE(&b->b_upd_req_list, ur, ur_entry);
1646 
1647 			bcopy(&ur->ur_msg, m->m_data + offset,
1648 			    sizeof(ur->ur_msg));
1649 			offset += sizeof(ur->ur_msg);
1650 			free(ur, M_PFSYNC);
1651 			count++;
1652 		}
1653 
1654 		bzero(subh, sizeof(*subh));
1655 		subh->action = PFSYNC_ACT_UPD_REQ;
1656 		subh->count = htons(count);
1657 		V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_UPD_REQ] += count;
1658 	}
1659 
1660 	/* has someone built a custom region for us to add? */
1661 	if (b->b_plus != NULL) {
1662 		bcopy(b->b_plus, m->m_data + offset, b->b_pluslen);
1663 		offset += b->b_pluslen;
1664 
1665 		b->b_plus = NULL;
1666 	}
1667 
1668 	subh = (struct pfsync_subheader *)(m->m_data + offset);
1669 	offset += sizeof(*subh);
1670 
1671 	bzero(subh, sizeof(*subh));
1672 	subh->action = PFSYNC_ACT_EOF;
1673 	subh->count = htons(1);
1674 	V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_EOF]++;
1675 
1676 	/* we're done, let's put it on the wire */
1677 	if (ifp->if_bpf) {
1678 		m->m_data += sizeof(*ip);
1679 		m->m_len = m->m_pkthdr.len = b->b_len - sizeof(*ip);
1680 		BPF_MTAP(ifp, m);
1681 		m->m_data -= sizeof(*ip);
1682 		m->m_len = m->m_pkthdr.len = b->b_len;
1683 	}
1684 
1685 	if (sc->sc_sync_if == NULL) {
1686 		b->b_len = PFSYNC_MINPKT;
1687 		m_freem(m);
1688 		return;
1689 	}
1690 
1691 	if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1);
1692 	if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len);
1693 	b->b_len = PFSYNC_MINPKT;
1694 
1695 	if (!_IF_QFULL(&b->b_snd))
1696 		_IF_ENQUEUE(&b->b_snd, m);
1697 	else {
1698 		m_freem(m);
1699 		if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1);
1700 	}
1701 	if (schedswi)
1702 		swi_sched(V_pfsync_swi_cookie, 0);
1703 }
1704 
1705 static void
1706 pfsync_insert_state(struct pf_kstate *st)
1707 {
1708 	struct pfsync_softc *sc = V_pfsyncif;
1709 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
1710 
1711 	if (st->state_flags & PFSTATE_NOSYNC)
1712 		return;
1713 
1714 	if ((st->rule.ptr->rule_flag & PFRULE_NOSYNC) ||
1715 	    st->key[PF_SK_WIRE]->proto == IPPROTO_PFSYNC) {
1716 		st->state_flags |= PFSTATE_NOSYNC;
1717 		return;
1718 	}
1719 
1720 	KASSERT(st->sync_state == PFSYNC_S_NONE,
1721 		("%s: st->sync_state %u", __func__, st->sync_state));
1722 
1723 	PFSYNC_BUCKET_LOCK(b);
1724 	if (b->b_len == PFSYNC_MINPKT)
1725 		callout_reset(&b->b_tmo, 1 * hz, pfsync_timeout, b);
1726 
1727 	pfsync_q_ins(st, PFSYNC_S_INS, true);
1728 	PFSYNC_BUCKET_UNLOCK(b);
1729 
1730 	st->sync_updates = 0;
1731 }
1732 
1733 static int
1734 pfsync_defer(struct pf_kstate *st, struct mbuf *m)
1735 {
1736 	struct pfsync_softc *sc = V_pfsyncif;
1737 	struct pfsync_deferral *pd;
1738 	struct pfsync_bucket *b;
1739 
1740 	if (m->m_flags & (M_BCAST|M_MCAST))
1741 		return (0);
1742 
1743 	if (sc == NULL)
1744 		return (0);
1745 
1746 	b = pfsync_get_bucket(sc, st);
1747 
1748 	PFSYNC_LOCK(sc);
1749 
1750 	if (!(sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) ||
1751 	    !(sc->sc_flags & PFSYNCF_DEFER)) {
1752 		PFSYNC_UNLOCK(sc);
1753 		return (0);
1754 	}
1755 
1756 	PFSYNC_BUCKET_LOCK(b);
1757 	PFSYNC_UNLOCK(sc);
1758 
1759 	if (b->b_deferred >= 128)
1760 		pfsync_undefer(TAILQ_FIRST(&b->b_deferrals), 0);
1761 
1762 	pd = malloc(sizeof(*pd), M_PFSYNC, M_NOWAIT);
1763 	if (pd == NULL) {
1764 		PFSYNC_BUCKET_UNLOCK(b);
1765 		return (0);
1766 	}
1767 	b->b_deferred++;
1768 
1769 	m->m_flags |= M_SKIP_FIREWALL;
1770 	st->state_flags |= PFSTATE_ACK;
1771 
1772 	pd->pd_sc = sc;
1773 	pd->pd_refs = 0;
1774 	pd->pd_st = st;
1775 	pf_ref_state(st);
1776 	pd->pd_m = m;
1777 
1778 	TAILQ_INSERT_TAIL(&b->b_deferrals, pd, pd_entry);
1779 	callout_init_mtx(&pd->pd_tmo, &b->b_mtx, CALLOUT_RETURNUNLOCKED);
1780 	callout_reset(&pd->pd_tmo, PFSYNC_DEFER_TIMEOUT, pfsync_defer_tmo, pd);
1781 
1782 	pfsync_push(b);
1783 	PFSYNC_BUCKET_UNLOCK(b);
1784 
1785 	return (1);
1786 }
1787 
1788 static void
1789 pfsync_undefer(struct pfsync_deferral *pd, int drop)
1790 {
1791 	struct pfsync_softc *sc = pd->pd_sc;
1792 	struct mbuf *m = pd->pd_m;
1793 	struct pf_kstate *st = pd->pd_st;
1794 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
1795 
1796 	PFSYNC_BUCKET_LOCK_ASSERT(b);
1797 
1798 	TAILQ_REMOVE(&b->b_deferrals, pd, pd_entry);
1799 	b->b_deferred--;
1800 	pd->pd_st->state_flags &= ~PFSTATE_ACK;	/* XXX: locking! */
1801 	free(pd, M_PFSYNC);
1802 	pf_release_state(st);
1803 
1804 	if (drop)
1805 		m_freem(m);
1806 	else {
1807 		_IF_ENQUEUE(&b->b_snd, m);
1808 		pfsync_push(b);
1809 	}
1810 }
1811 
1812 static void
1813 pfsync_defer_tmo(void *arg)
1814 {
1815 	struct epoch_tracker et;
1816 	struct pfsync_deferral *pd = arg;
1817 	struct pfsync_softc *sc = pd->pd_sc;
1818 	struct mbuf *m = pd->pd_m;
1819 	struct pf_kstate *st = pd->pd_st;
1820 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
1821 
1822 	PFSYNC_BUCKET_LOCK_ASSERT(b);
1823 
1824 	NET_EPOCH_ENTER(et);
1825 	CURVNET_SET(m->m_pkthdr.rcvif->if_vnet);
1826 
1827 	TAILQ_REMOVE(&b->b_deferrals, pd, pd_entry);
1828 	b->b_deferred--;
1829 	pd->pd_st->state_flags &= ~PFSTATE_ACK;	/* XXX: locking! */
1830 	if (pd->pd_refs == 0)
1831 		free(pd, M_PFSYNC);
1832 	PFSYNC_BUCKET_UNLOCK(b);
1833 
1834 	ip_output(m, NULL, NULL, 0, NULL, NULL);
1835 
1836 	pf_release_state(st);
1837 
1838 	CURVNET_RESTORE();
1839 	NET_EPOCH_EXIT(et);
1840 }
1841 
1842 static void
1843 pfsync_undefer_state(struct pf_kstate *st, int drop)
1844 {
1845 	struct pfsync_softc *sc = V_pfsyncif;
1846 	struct pfsync_deferral *pd;
1847 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
1848 
1849 	PFSYNC_BUCKET_LOCK(b);
1850 
1851 	TAILQ_FOREACH(pd, &b->b_deferrals, pd_entry) {
1852 		 if (pd->pd_st == st) {
1853 			if (callout_stop(&pd->pd_tmo) > 0)
1854 				pfsync_undefer(pd, drop);
1855 
1856 			PFSYNC_BUCKET_UNLOCK(b);
1857 			return;
1858 		}
1859 	}
1860 	PFSYNC_BUCKET_UNLOCK(b);
1861 
1862 	panic("%s: unable to find deferred state", __func__);
1863 }
1864 
1865 static struct pfsync_bucket*
1866 pfsync_get_bucket(struct pfsync_softc *sc, struct pf_kstate *st)
1867 {
1868 	int c = PF_IDHASH(st) % pfsync_buckets;
1869 	return &sc->sc_buckets[c];
1870 }
1871 
1872 static void
1873 pfsync_update_state(struct pf_kstate *st)
1874 {
1875 	struct pfsync_softc *sc = V_pfsyncif;
1876 	bool sync = false, ref = true;
1877 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
1878 
1879 	PF_STATE_LOCK_ASSERT(st);
1880 	PFSYNC_BUCKET_LOCK(b);
1881 
1882 	if (st->state_flags & PFSTATE_ACK)
1883 		pfsync_undefer_state(st, 0);
1884 	if (st->state_flags & PFSTATE_NOSYNC) {
1885 		if (st->sync_state != PFSYNC_S_NONE)
1886 			pfsync_q_del(st, true, b);
1887 		PFSYNC_BUCKET_UNLOCK(b);
1888 		return;
1889 	}
1890 
1891 	if (b->b_len == PFSYNC_MINPKT)
1892 		callout_reset(&b->b_tmo, 1 * hz, pfsync_timeout, b);
1893 
1894 	switch (st->sync_state) {
1895 	case PFSYNC_S_UPD_C:
1896 	case PFSYNC_S_UPD:
1897 	case PFSYNC_S_INS:
1898 		/* we're already handling it */
1899 
1900 		if (st->key[PF_SK_WIRE]->proto == IPPROTO_TCP) {
1901 			st->sync_updates++;
1902 			if (st->sync_updates >= sc->sc_maxupdates)
1903 				sync = true;
1904 		}
1905 		break;
1906 
1907 	case PFSYNC_S_IACK:
1908 		pfsync_q_del(st, false, b);
1909 		ref = false;
1910 		/* FALLTHROUGH */
1911 
1912 	case PFSYNC_S_NONE:
1913 		pfsync_q_ins(st, PFSYNC_S_UPD_C, ref);
1914 		st->sync_updates = 0;
1915 		break;
1916 
1917 	default:
1918 		panic("%s: unexpected sync state %d", __func__, st->sync_state);
1919 	}
1920 
1921 	if (sync || (time_uptime - st->pfsync_time) < 2)
1922 		pfsync_push(b);
1923 
1924 	PFSYNC_BUCKET_UNLOCK(b);
1925 }
1926 
1927 static void
1928 pfsync_request_update(u_int32_t creatorid, u_int64_t id)
1929 {
1930 	struct pfsync_softc *sc = V_pfsyncif;
1931 	struct pfsync_bucket *b = &sc->sc_buckets[0];
1932 	struct pfsync_upd_req_item *item;
1933 	size_t nlen = sizeof(struct pfsync_upd_req);
1934 
1935 	PFSYNC_BUCKET_LOCK_ASSERT(b);
1936 
1937 	/*
1938 	 * This code does a bit to prevent multiple update requests for the
1939 	 * same state being generated. It searches current subheader queue,
1940 	 * but it doesn't lookup into queue of already packed datagrams.
1941 	 */
1942 	TAILQ_FOREACH(item, &b->b_upd_req_list, ur_entry)
1943 		if (item->ur_msg.id == id &&
1944 		    item->ur_msg.creatorid == creatorid)
1945 			return;
1946 
1947 	item = malloc(sizeof(*item), M_PFSYNC, M_NOWAIT);
1948 	if (item == NULL)
1949 		return; /* XXX stats */
1950 
1951 	item->ur_msg.id = id;
1952 	item->ur_msg.creatorid = creatorid;
1953 
1954 	if (TAILQ_EMPTY(&b->b_upd_req_list))
1955 		nlen += sizeof(struct pfsync_subheader);
1956 
1957 	if (b->b_len + nlen > sc->sc_ifp->if_mtu) {
1958 		pfsync_sendout(0, 0);
1959 
1960 		nlen = sizeof(struct pfsync_subheader) +
1961 		    sizeof(struct pfsync_upd_req);
1962 	}
1963 
1964 	TAILQ_INSERT_TAIL(&b->b_upd_req_list, item, ur_entry);
1965 	b->b_len += nlen;
1966 
1967 	pfsync_push(b);
1968 }
1969 
1970 static bool
1971 pfsync_update_state_req(struct pf_kstate *st)
1972 {
1973 	struct pfsync_softc *sc = V_pfsyncif;
1974 	bool ref = true, full = false;
1975 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
1976 
1977 	PF_STATE_LOCK_ASSERT(st);
1978 	PFSYNC_BUCKET_LOCK(b);
1979 
1980 	if (st->state_flags & PFSTATE_NOSYNC) {
1981 		if (st->sync_state != PFSYNC_S_NONE)
1982 			pfsync_q_del(st, true, b);
1983 		PFSYNC_BUCKET_UNLOCK(b);
1984 		return (full);
1985 	}
1986 
1987 	switch (st->sync_state) {
1988 	case PFSYNC_S_UPD_C:
1989 	case PFSYNC_S_IACK:
1990 		pfsync_q_del(st, false, b);
1991 		ref = false;
1992 		/* FALLTHROUGH */
1993 
1994 	case PFSYNC_S_NONE:
1995 		pfsync_q_ins(st, PFSYNC_S_UPD, ref);
1996 		pfsync_push(b);
1997 		break;
1998 
1999 	case PFSYNC_S_INS:
2000 	case PFSYNC_S_UPD:
2001 	case PFSYNC_S_DEL:
2002 		/* we're already handling it */
2003 		break;
2004 
2005 	default:
2006 		panic("%s: unexpected sync state %d", __func__, st->sync_state);
2007 	}
2008 
2009 	if ((sc->sc_ifp->if_mtu - b->b_len) < sizeof(struct pfsync_state))
2010 		full = true;
2011 
2012 	PFSYNC_BUCKET_UNLOCK(b);
2013 
2014 	return (full);
2015 }
2016 
2017 static void
2018 pfsync_delete_state(struct pf_kstate *st)
2019 {
2020 	struct pfsync_softc *sc = V_pfsyncif;
2021 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
2022 	bool ref = true;
2023 
2024 	PFSYNC_BUCKET_LOCK(b);
2025 	if (st->state_flags & PFSTATE_ACK)
2026 		pfsync_undefer_state(st, 1);
2027 	if (st->state_flags & PFSTATE_NOSYNC) {
2028 		if (st->sync_state != PFSYNC_S_NONE)
2029 			pfsync_q_del(st, true, b);
2030 		PFSYNC_BUCKET_UNLOCK(b);
2031 		return;
2032 	}
2033 
2034 	if (b->b_len == PFSYNC_MINPKT)
2035 		callout_reset(&b->b_tmo, 1 * hz, pfsync_timeout, b);
2036 
2037 	switch (st->sync_state) {
2038 	case PFSYNC_S_INS:
2039 		/* We never got to tell the world so just forget about it. */
2040 		pfsync_q_del(st, true, b);
2041 		break;
2042 
2043 	case PFSYNC_S_UPD_C:
2044 	case PFSYNC_S_UPD:
2045 	case PFSYNC_S_IACK:
2046 		pfsync_q_del(st, false, b);
2047 		ref = false;
2048 		/* FALLTHROUGH */
2049 
2050 	case PFSYNC_S_NONE:
2051 		pfsync_q_ins(st, PFSYNC_S_DEL, ref);
2052 		break;
2053 
2054 	default:
2055 		panic("%s: unexpected sync state %d", __func__, st->sync_state);
2056 	}
2057 
2058 	PFSYNC_BUCKET_UNLOCK(b);
2059 }
2060 
2061 static void
2062 pfsync_clear_states(u_int32_t creatorid, const char *ifname)
2063 {
2064 	struct {
2065 		struct pfsync_subheader subh;
2066 		struct pfsync_clr clr;
2067 	} __packed r;
2068 
2069 	bzero(&r, sizeof(r));
2070 
2071 	r.subh.action = PFSYNC_ACT_CLR;
2072 	r.subh.count = htons(1);
2073 	V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_CLR]++;
2074 
2075 	strlcpy(r.clr.ifname, ifname, sizeof(r.clr.ifname));
2076 	r.clr.creatorid = creatorid;
2077 
2078 	pfsync_send_plus(&r, sizeof(r));
2079 }
2080 
2081 static void
2082 pfsync_q_ins(struct pf_kstate *st, int q, bool ref)
2083 {
2084 	struct pfsync_softc *sc = V_pfsyncif;
2085 	size_t nlen = pfsync_qs[q].len;
2086 	struct pfsync_bucket *b = pfsync_get_bucket(sc, st);
2087 
2088 	PFSYNC_BUCKET_LOCK_ASSERT(b);
2089 
2090 	KASSERT(st->sync_state == PFSYNC_S_NONE,
2091 		("%s: st->sync_state %u", __func__, st->sync_state));
2092 	KASSERT(b->b_len >= PFSYNC_MINPKT, ("pfsync pkt len is too low %zu",
2093 	    b->b_len));
2094 
2095 	if (TAILQ_EMPTY(&b->b_qs[q]))
2096 		nlen += sizeof(struct pfsync_subheader);
2097 
2098 	if (b->b_len + nlen > sc->sc_ifp->if_mtu) {
2099 		pfsync_sendout(1, b->b_id);
2100 
2101 		nlen = sizeof(struct pfsync_subheader) + pfsync_qs[q].len;
2102 	}
2103 
2104 	b->b_len += nlen;
2105 	TAILQ_INSERT_TAIL(&b->b_qs[q], st, sync_list);
2106 	st->sync_state = q;
2107 	if (ref)
2108 		pf_ref_state(st);
2109 }
2110 
2111 static void
2112 pfsync_q_del(struct pf_kstate *st, bool unref, struct pfsync_bucket *b)
2113 {
2114 	int q = st->sync_state;
2115 
2116 	PFSYNC_BUCKET_LOCK_ASSERT(b);
2117 	KASSERT(st->sync_state != PFSYNC_S_NONE,
2118 		("%s: st->sync_state != PFSYNC_S_NONE", __func__));
2119 
2120 	b->b_len -= pfsync_qs[q].len;
2121 	TAILQ_REMOVE(&b->b_qs[q], st, sync_list);
2122 	st->sync_state = PFSYNC_S_NONE;
2123 	if (unref)
2124 		pf_release_state(st);
2125 
2126 	if (TAILQ_EMPTY(&b->b_qs[q]))
2127 		b->b_len -= sizeof(struct pfsync_subheader);
2128 }
2129 
2130 static void
2131 pfsync_bulk_start(void)
2132 {
2133 	struct pfsync_softc *sc = V_pfsyncif;
2134 
2135 	if (V_pf_status.debug >= PF_DEBUG_MISC)
2136 		printf("pfsync: received bulk update request\n");
2137 
2138 	PFSYNC_BLOCK(sc);
2139 
2140 	sc->sc_ureq_received = time_uptime;
2141 	sc->sc_bulk_hashid = 0;
2142 	sc->sc_bulk_stateid = 0;
2143 	pfsync_bulk_status(PFSYNC_BUS_START);
2144 	callout_reset(&sc->sc_bulk_tmo, 1, pfsync_bulk_update, sc);
2145 	PFSYNC_BUNLOCK(sc);
2146 }
2147 
2148 static void
2149 pfsync_bulk_update(void *arg)
2150 {
2151 	struct pfsync_softc *sc = arg;
2152 	struct pf_kstate *s;
2153 	int i, sent = 0;
2154 
2155 	PFSYNC_BLOCK_ASSERT(sc);
2156 	CURVNET_SET(sc->sc_ifp->if_vnet);
2157 
2158 	/*
2159 	 * Start with last state from previous invocation.
2160 	 * It may had gone, in this case start from the
2161 	 * hash slot.
2162 	 */
2163 	s = pf_find_state_byid(sc->sc_bulk_stateid, sc->sc_bulk_creatorid);
2164 
2165 	if (s != NULL)
2166 		i = PF_IDHASH(s);
2167 	else
2168 		i = sc->sc_bulk_hashid;
2169 
2170 	for (; i <= pf_hashmask; i++) {
2171 		struct pf_idhash *ih = &V_pf_idhash[i];
2172 
2173 		if (s != NULL)
2174 			PF_HASHROW_ASSERT(ih);
2175 		else {
2176 			PF_HASHROW_LOCK(ih);
2177 			s = LIST_FIRST(&ih->states);
2178 		}
2179 
2180 		for (; s; s = LIST_NEXT(s, entry)) {
2181 			if (s->sync_state == PFSYNC_S_NONE &&
2182 			    s->timeout < PFTM_MAX &&
2183 			    s->pfsync_time <= sc->sc_ureq_received) {
2184 				if (pfsync_update_state_req(s)) {
2185 					/* We've filled a packet. */
2186 					sc->sc_bulk_hashid = i;
2187 					sc->sc_bulk_stateid = s->id;
2188 					sc->sc_bulk_creatorid = s->creatorid;
2189 					PF_HASHROW_UNLOCK(ih);
2190 					callout_reset(&sc->sc_bulk_tmo, 1,
2191 					    pfsync_bulk_update, sc);
2192 					goto full;
2193 				}
2194 				sent++;
2195 			}
2196 		}
2197 		PF_HASHROW_UNLOCK(ih);
2198 	}
2199 
2200 	/* We're done. */
2201 	pfsync_bulk_status(PFSYNC_BUS_END);
2202 full:
2203 	CURVNET_RESTORE();
2204 }
2205 
2206 static void
2207 pfsync_bulk_status(u_int8_t status)
2208 {
2209 	struct {
2210 		struct pfsync_subheader subh;
2211 		struct pfsync_bus bus;
2212 	} __packed r;
2213 
2214 	struct pfsync_softc *sc = V_pfsyncif;
2215 
2216 	bzero(&r, sizeof(r));
2217 
2218 	r.subh.action = PFSYNC_ACT_BUS;
2219 	r.subh.count = htons(1);
2220 	V_pfsyncstats.pfsyncs_oacts[PFSYNC_ACT_BUS]++;
2221 
2222 	r.bus.creatorid = V_pf_status.hostid;
2223 	r.bus.endtime = htonl(time_uptime - sc->sc_ureq_received);
2224 	r.bus.status = status;
2225 
2226 	pfsync_send_plus(&r, sizeof(r));
2227 }
2228 
2229 static void
2230 pfsync_bulk_fail(void *arg)
2231 {
2232 	struct pfsync_softc *sc = arg;
2233 	struct pfsync_bucket *b = &sc->sc_buckets[0];
2234 
2235 	CURVNET_SET(sc->sc_ifp->if_vnet);
2236 
2237 	PFSYNC_BLOCK_ASSERT(sc);
2238 
2239 	if (sc->sc_bulk_tries++ < PFSYNC_MAX_BULKTRIES) {
2240 		/* Try again */
2241 		callout_reset(&sc->sc_bulkfail_tmo, 5 * hz,
2242 		    pfsync_bulk_fail, V_pfsyncif);
2243 		PFSYNC_BUCKET_LOCK(b);
2244 		pfsync_request_update(0, 0);
2245 		PFSYNC_BUCKET_UNLOCK(b);
2246 	} else {
2247 		/* Pretend like the transfer was ok. */
2248 		sc->sc_ureq_sent = 0;
2249 		sc->sc_bulk_tries = 0;
2250 		PFSYNC_LOCK(sc);
2251 		if (!(sc->sc_flags & PFSYNCF_OK) && carp_demote_adj_p)
2252 			(*carp_demote_adj_p)(-V_pfsync_carp_adj,
2253 			    "pfsync bulk fail");
2254 		sc->sc_flags |= PFSYNCF_OK;
2255 		PFSYNC_UNLOCK(sc);
2256 		if (V_pf_status.debug >= PF_DEBUG_MISC)
2257 			printf("pfsync: failed to receive bulk update\n");
2258 	}
2259 
2260 	CURVNET_RESTORE();
2261 }
2262 
2263 static void
2264 pfsync_send_plus(void *plus, size_t pluslen)
2265 {
2266 	struct pfsync_softc *sc = V_pfsyncif;
2267 	struct pfsync_bucket *b = &sc->sc_buckets[0];
2268 
2269 	PFSYNC_BUCKET_LOCK(b);
2270 
2271 	if (b->b_len + pluslen > sc->sc_ifp->if_mtu)
2272 		pfsync_sendout(1, b->b_id);
2273 
2274 	b->b_plus = plus;
2275 	b->b_len += (b->b_pluslen = pluslen);
2276 
2277 	pfsync_sendout(1, b->b_id);
2278 	PFSYNC_BUCKET_UNLOCK(b);
2279 }
2280 
2281 static void
2282 pfsync_timeout(void *arg)
2283 {
2284 	struct pfsync_bucket *b = arg;
2285 
2286 	CURVNET_SET(b->b_sc->sc_ifp->if_vnet);
2287 	PFSYNC_BUCKET_LOCK(b);
2288 	pfsync_push(b);
2289 	PFSYNC_BUCKET_UNLOCK(b);
2290 	CURVNET_RESTORE();
2291 }
2292 
2293 static void
2294 pfsync_push(struct pfsync_bucket *b)
2295 {
2296 
2297 	PFSYNC_BUCKET_LOCK_ASSERT(b);
2298 
2299 	b->b_flags |= PFSYNCF_BUCKET_PUSH;
2300 	swi_sched(V_pfsync_swi_cookie, 0);
2301 }
2302 
2303 static void
2304 pfsync_push_all(struct pfsync_softc *sc)
2305 {
2306 	int c;
2307 	struct pfsync_bucket *b;
2308 
2309 	for (c = 0; c < pfsync_buckets; c++) {
2310 		b = &sc->sc_buckets[c];
2311 
2312 		PFSYNC_BUCKET_LOCK(b);
2313 		pfsync_push(b);
2314 		PFSYNC_BUCKET_UNLOCK(b);
2315 	}
2316 }
2317 
2318 static void
2319 pfsyncintr(void *arg)
2320 {
2321 	struct epoch_tracker et;
2322 	struct pfsync_softc *sc = arg;
2323 	struct pfsync_bucket *b;
2324 	struct mbuf *m, *n;
2325 	int c;
2326 
2327 	NET_EPOCH_ENTER(et);
2328 	CURVNET_SET(sc->sc_ifp->if_vnet);
2329 
2330 	for (c = 0; c < pfsync_buckets; c++) {
2331 		b = &sc->sc_buckets[c];
2332 
2333 		PFSYNC_BUCKET_LOCK(b);
2334 		if ((b->b_flags & PFSYNCF_BUCKET_PUSH) && b->b_len > PFSYNC_MINPKT) {
2335 			pfsync_sendout(0, b->b_id);
2336 			b->b_flags &= ~PFSYNCF_BUCKET_PUSH;
2337 		}
2338 		_IF_DEQUEUE_ALL(&b->b_snd, m);
2339 		PFSYNC_BUCKET_UNLOCK(b);
2340 
2341 		for (; m != NULL; m = n) {
2342 			n = m->m_nextpkt;
2343 			m->m_nextpkt = NULL;
2344 
2345 			/*
2346 			 * We distinguish between a deferral packet and our
2347 			 * own pfsync packet based on M_SKIP_FIREWALL
2348 			 * flag. This is XXX.
2349 			 */
2350 			if (m->m_flags & M_SKIP_FIREWALL)
2351 				ip_output(m, NULL, NULL, 0, NULL, NULL);
2352 			else if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo,
2353 			    NULL) == 0)
2354 				V_pfsyncstats.pfsyncs_opackets++;
2355 			else
2356 				V_pfsyncstats.pfsyncs_oerrors++;
2357 		}
2358 	}
2359 	CURVNET_RESTORE();
2360 	NET_EPOCH_EXIT(et);
2361 }
2362 
2363 static int
2364 pfsync_multicast_setup(struct pfsync_softc *sc, struct ifnet *ifp,
2365     struct in_mfilter *imf)
2366 {
2367 	struct ip_moptions *imo = &sc->sc_imo;
2368 	int error;
2369 
2370 	if (!(ifp->if_flags & IFF_MULTICAST))
2371 		return (EADDRNOTAVAIL);
2372 
2373 	imo->imo_multicast_vif = -1;
2374 
2375 	if ((error = in_joingroup(ifp, &sc->sc_sync_peer, NULL,
2376 	    &imf->imf_inm)) != 0)
2377 		return (error);
2378 
2379 	ip_mfilter_init(&imo->imo_head);
2380 	ip_mfilter_insert(&imo->imo_head, imf);
2381 	imo->imo_multicast_ifp = ifp;
2382 	imo->imo_multicast_ttl = PFSYNC_DFLTTL;
2383 	imo->imo_multicast_loop = 0;
2384 
2385 	return (0);
2386 }
2387 
2388 static void
2389 pfsync_multicast_cleanup(struct pfsync_softc *sc)
2390 {
2391 	struct ip_moptions *imo = &sc->sc_imo;
2392 	struct in_mfilter *imf;
2393 
2394 	while ((imf = ip_mfilter_first(&imo->imo_head)) != NULL) {
2395 		ip_mfilter_remove(&imo->imo_head, imf);
2396 		in_leavegroup(imf->imf_inm, NULL);
2397 		ip_mfilter_free(imf);
2398 	}
2399 	imo->imo_multicast_ifp = NULL;
2400 }
2401 
2402 void
2403 pfsync_detach_ifnet(struct ifnet *ifp)
2404 {
2405 	struct pfsync_softc *sc = V_pfsyncif;
2406 
2407 	if (sc == NULL)
2408 		return;
2409 
2410 	PFSYNC_LOCK(sc);
2411 
2412 	if (sc->sc_sync_if == ifp) {
2413 		/* We don't need mutlicast cleanup here, because the interface
2414 		 * is going away. We do need to ensure we don't try to do
2415 		 * cleanup later.
2416 		 */
2417 		ip_mfilter_init(&sc->sc_imo.imo_head);
2418 		sc->sc_imo.imo_multicast_ifp = NULL;
2419 		sc->sc_sync_if = NULL;
2420 	}
2421 
2422 	PFSYNC_UNLOCK(sc);
2423 }
2424 
2425 #ifdef INET
2426 extern  struct domain inetdomain;
2427 static struct protosw in_pfsync_protosw = {
2428 	.pr_type =		SOCK_RAW,
2429 	.pr_domain =		&inetdomain,
2430 	.pr_protocol =		IPPROTO_PFSYNC,
2431 	.pr_flags =		PR_ATOMIC|PR_ADDR,
2432 	.pr_input =		pfsync_input,
2433 	.pr_output =		rip_output,
2434 	.pr_ctloutput =		rip_ctloutput,
2435 	.pr_usrreqs =		&rip_usrreqs
2436 };
2437 #endif
2438 
2439 static void
2440 pfsync_pointers_init()
2441 {
2442 
2443 	PF_RULES_WLOCK();
2444 	V_pfsync_state_import_ptr = pfsync_state_import;
2445 	V_pfsync_insert_state_ptr = pfsync_insert_state;
2446 	V_pfsync_update_state_ptr = pfsync_update_state;
2447 	V_pfsync_delete_state_ptr = pfsync_delete_state;
2448 	V_pfsync_clear_states_ptr = pfsync_clear_states;
2449 	V_pfsync_defer_ptr = pfsync_defer;
2450 	PF_RULES_WUNLOCK();
2451 }
2452 
2453 static void
2454 pfsync_pointers_uninit()
2455 {
2456 
2457 	PF_RULES_WLOCK();
2458 	V_pfsync_state_import_ptr = NULL;
2459 	V_pfsync_insert_state_ptr = NULL;
2460 	V_pfsync_update_state_ptr = NULL;
2461 	V_pfsync_delete_state_ptr = NULL;
2462 	V_pfsync_clear_states_ptr = NULL;
2463 	V_pfsync_defer_ptr = NULL;
2464 	PF_RULES_WUNLOCK();
2465 }
2466 
2467 static void
2468 vnet_pfsync_init(const void *unused __unused)
2469 {
2470 	int error;
2471 
2472 	V_pfsync_cloner = if_clone_simple(pfsyncname,
2473 	    pfsync_clone_create, pfsync_clone_destroy, 1);
2474 	error = swi_add(&V_pfsync_swi_ie, pfsyncname, pfsyncintr, V_pfsyncif,
2475 	    SWI_NET, INTR_MPSAFE, &V_pfsync_swi_cookie);
2476 	if (error) {
2477 		if_clone_detach(V_pfsync_cloner);
2478 		log(LOG_INFO, "swi_add() failed in %s\n", __func__);
2479 	}
2480 
2481 	pfsync_pointers_init();
2482 }
2483 VNET_SYSINIT(vnet_pfsync_init, SI_SUB_PROTO_FIREWALL, SI_ORDER_ANY,
2484     vnet_pfsync_init, NULL);
2485 
2486 static void
2487 vnet_pfsync_uninit(const void *unused __unused)
2488 {
2489 	int ret __diagused;
2490 
2491 	pfsync_pointers_uninit();
2492 
2493 	if_clone_detach(V_pfsync_cloner);
2494 	ret = swi_remove(V_pfsync_swi_cookie);
2495 	MPASS(ret == 0);
2496 	ret = intr_event_destroy(V_pfsync_swi_ie);
2497 	MPASS(ret == 0);
2498 }
2499 
2500 VNET_SYSUNINIT(vnet_pfsync_uninit, SI_SUB_PROTO_FIREWALL, SI_ORDER_FOURTH,
2501     vnet_pfsync_uninit, NULL);
2502 
2503 static int
2504 pfsync_init()
2505 {
2506 #ifdef INET
2507 	int error;
2508 
2509 	pfsync_detach_ifnet_ptr = pfsync_detach_ifnet;
2510 
2511 	error = pf_proto_register(PF_INET, &in_pfsync_protosw);
2512 	if (error)
2513 		return (error);
2514 	error = ipproto_register(IPPROTO_PFSYNC);
2515 	if (error) {
2516 		pf_proto_unregister(PF_INET, IPPROTO_PFSYNC, SOCK_RAW);
2517 		return (error);
2518 	}
2519 #endif
2520 
2521 	return (0);
2522 }
2523 
2524 static void
2525 pfsync_uninit()
2526 {
2527 	pfsync_detach_ifnet_ptr = NULL;
2528 
2529 #ifdef INET
2530 	ipproto_unregister(IPPROTO_PFSYNC);
2531 	pf_proto_unregister(PF_INET, IPPROTO_PFSYNC, SOCK_RAW);
2532 #endif
2533 }
2534 
2535 static int
2536 pfsync_modevent(module_t mod, int type, void *data)
2537 {
2538 	int error = 0;
2539 
2540 	switch (type) {
2541 	case MOD_LOAD:
2542 		error = pfsync_init();
2543 		break;
2544 	case MOD_UNLOAD:
2545 		pfsync_uninit();
2546 		break;
2547 	default:
2548 		error = EINVAL;
2549 		break;
2550 	}
2551 
2552 	return (error);
2553 }
2554 
2555 static moduledata_t pfsync_mod = {
2556 	pfsyncname,
2557 	pfsync_modevent,
2558 	0
2559 };
2560 
2561 #define PFSYNC_MODVER 1
2562 
2563 /* Stay on FIREWALL as we depend on pf being initialized and on inetdomain. */
2564 DECLARE_MODULE(pfsync, pfsync_mod, SI_SUB_PROTO_FIREWALL, SI_ORDER_ANY);
2565 MODULE_VERSION(pfsync, PFSYNC_MODVER);
2566 MODULE_DEPEND(pfsync, pf, PF_MODVER, PF_MODVER, PF_MODVER);
2567