xref: /freebsd/sys/netpfil/pf/pf.c (revision 944223076fbcb681a8eb3e118ddafab79b21ac3d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2001 Daniel Hartmeier
5  * Copyright (c) 2002 - 2008 Henning Brauer
6  * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  *    - Redistributions of source code must retain the above copyright
14  *      notice, this list of conditions and the following disclaimer.
15  *    - Redistributions in binary form must reproduce the above
16  *      copyright notice, this list of conditions and the following
17  *      disclaimer in the documentation and/or other materials provided
18  *      with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGE.
32  *
33  * Effort sponsored in part by the Defense Advanced Research Projects
34  * Agency (DARPA) and Air Force Research Laboratory, Air Force
35  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
36  *
37  *	$OpenBSD: pf.c,v 1.634 2009/02/27 12:37:45 henning Exp $
38  */
39 
40 #include <sys/cdefs.h>
41 #include "opt_bpf.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_pf.h"
45 #include "opt_sctp.h"
46 
47 #include <sys/param.h>
48 #include <sys/bus.h>
49 #include <sys/endian.h>
50 #include <sys/gsb_crc32.h>
51 #include <sys/hash.h>
52 #include <sys/interrupt.h>
53 #include <sys/kernel.h>
54 #include <sys/kthread.h>
55 #include <sys/limits.h>
56 #include <sys/mbuf.h>
57 #include <sys/md5.h>
58 #include <sys/random.h>
59 #include <sys/refcount.h>
60 #include <sys/sdt.h>
61 #include <sys/socket.h>
62 #include <sys/sysctl.h>
63 #include <sys/taskqueue.h>
64 #include <sys/ucred.h>
65 
66 #include <net/if.h>
67 #include <net/if_var.h>
68 #include <net/if_private.h>
69 #include <net/if_types.h>
70 #include <net/if_vlan_var.h>
71 #include <net/route.h>
72 #include <net/route/nhop.h>
73 #include <net/vnet.h>
74 
75 #include <net/pfil.h>
76 #include <net/pfvar.h>
77 #include <net/if_pflog.h>
78 #include <net/if_pfsync.h>
79 
80 #include <netinet/in_pcb.h>
81 #include <netinet/in_var.h>
82 #include <netinet/in_fib.h>
83 #include <netinet/ip.h>
84 #include <netinet/ip_fw.h>
85 #include <netinet/ip_icmp.h>
86 #include <netinet/icmp_var.h>
87 #include <netinet/ip_var.h>
88 #include <netinet/tcp.h>
89 #include <netinet/tcp_fsm.h>
90 #include <netinet/tcp_seq.h>
91 #include <netinet/tcp_timer.h>
92 #include <netinet/tcp_var.h>
93 #include <netinet/udp.h>
94 #include <netinet/udp_var.h>
95 
96 /* dummynet */
97 #include <netinet/ip_dummynet.h>
98 #include <netinet/ip_fw.h>
99 #include <netpfil/ipfw/dn_heap.h>
100 #include <netpfil/ipfw/ip_fw_private.h>
101 #include <netpfil/ipfw/ip_dn_private.h>
102 
103 #ifdef INET6
104 #include <netinet/ip6.h>
105 #include <netinet/icmp6.h>
106 #include <netinet6/nd6.h>
107 #include <netinet6/ip6_var.h>
108 #include <netinet6/in6_pcb.h>
109 #include <netinet6/in6_fib.h>
110 #include <netinet6/scope6_var.h>
111 #endif /* INET6 */
112 
113 #include <netinet/sctp_header.h>
114 #include <netinet/sctp_crc32.h>
115 
116 #include <machine/in_cksum.h>
117 #include <security/mac/mac_framework.h>
118 
119 #define	DPFPRINTF(n, x)	if (V_pf_status.debug >= (n)) printf x
120 
121 SDT_PROVIDER_DEFINE(pf);
122 SDT_PROBE_DEFINE4(pf, ip, test, done, "int", "int", "struct pf_krule *",
123     "struct pf_kstate *");
124 SDT_PROBE_DEFINE4(pf, ip, test6, done, "int", "int", "struct pf_krule *",
125     "struct pf_kstate *");
126 SDT_PROBE_DEFINE5(pf, ip, state, lookup, "struct pfi_kkif *",
127     "struct pf_state_key_cmp *", "int", "struct pf_pdesc *",
128     "struct pf_kstate *");
129 SDT_PROBE_DEFINE4(pf, sctp, multihome, test, "struct pfi_kkif *",
130     "struct pf_krule *", "struct mbuf *", "int");
131 
132 SDT_PROBE_DEFINE3(pf, eth, test_rule, entry, "int", "struct ifnet *",
133     "struct mbuf *");
134 SDT_PROBE_DEFINE2(pf, eth, test_rule, test, "int", "struct pf_keth_rule *");
135 SDT_PROBE_DEFINE3(pf, eth, test_rule, mismatch,
136     "int", "struct pf_keth_rule *", "char *");
137 SDT_PROBE_DEFINE2(pf, eth, test_rule, match, "int", "struct pf_keth_rule *");
138 SDT_PROBE_DEFINE2(pf, eth, test_rule, final_match,
139     "int", "struct pf_keth_rule *");
140 
141 /*
142  * Global variables
143  */
144 
145 /* state tables */
146 VNET_DEFINE(struct pf_altqqueue,	 pf_altqs[4]);
147 VNET_DEFINE(struct pf_kpalist,		 pf_pabuf);
148 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_active);
149 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_active);
150 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_inactive);
151 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_inactive);
152 VNET_DEFINE(struct pf_kstatus,		 pf_status);
153 
154 VNET_DEFINE(u_int32_t,			 ticket_altqs_active);
155 VNET_DEFINE(u_int32_t,			 ticket_altqs_inactive);
156 VNET_DEFINE(int,			 altqs_inactive_open);
157 VNET_DEFINE(u_int32_t,			 ticket_pabuf);
158 
159 VNET_DEFINE(MD5_CTX,			 pf_tcp_secret_ctx);
160 #define	V_pf_tcp_secret_ctx		 VNET(pf_tcp_secret_ctx)
161 VNET_DEFINE(u_char,			 pf_tcp_secret[16]);
162 #define	V_pf_tcp_secret			 VNET(pf_tcp_secret)
163 VNET_DEFINE(int,			 pf_tcp_secret_init);
164 #define	V_pf_tcp_secret_init		 VNET(pf_tcp_secret_init)
165 VNET_DEFINE(int,			 pf_tcp_iss_off);
166 #define	V_pf_tcp_iss_off		 VNET(pf_tcp_iss_off)
167 VNET_DECLARE(int,			 pf_vnet_active);
168 #define	V_pf_vnet_active		 VNET(pf_vnet_active)
169 
170 VNET_DEFINE_STATIC(uint32_t, pf_purge_idx);
171 #define V_pf_purge_idx	VNET(pf_purge_idx)
172 
173 #ifdef PF_WANT_32_TO_64_COUNTER
174 VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter);
175 #define	V_pf_counter_periodic_iter	VNET(pf_counter_periodic_iter)
176 
177 VNET_DEFINE(struct allrulelist_head, pf_allrulelist);
178 VNET_DEFINE(size_t, pf_allrulecount);
179 VNET_DEFINE(struct pf_krule *, pf_rulemarker);
180 #endif
181 
182 /*
183  * Queue for pf_intr() sends.
184  */
185 static MALLOC_DEFINE(M_PFTEMP, "pf_temp", "pf(4) temporary allocations");
186 struct pf_send_entry {
187 	STAILQ_ENTRY(pf_send_entry)	pfse_next;
188 	struct mbuf			*pfse_m;
189 	enum {
190 		PFSE_IP,
191 		PFSE_IP6,
192 		PFSE_ICMP,
193 		PFSE_ICMP6,
194 	}				pfse_type;
195 	struct {
196 		int		type;
197 		int		code;
198 		int		mtu;
199 	} icmpopts;
200 };
201 
202 STAILQ_HEAD(pf_send_head, pf_send_entry);
203 VNET_DEFINE_STATIC(struct pf_send_head, pf_sendqueue);
204 #define	V_pf_sendqueue	VNET(pf_sendqueue)
205 
206 static struct mtx_padalign pf_sendqueue_mtx;
207 MTX_SYSINIT(pf_sendqueue_mtx, &pf_sendqueue_mtx, "pf send queue", MTX_DEF);
208 #define	PF_SENDQ_LOCK()		mtx_lock(&pf_sendqueue_mtx)
209 #define	PF_SENDQ_UNLOCK()	mtx_unlock(&pf_sendqueue_mtx)
210 
211 /*
212  * Queue for pf_overload_task() tasks.
213  */
214 struct pf_overload_entry {
215 	SLIST_ENTRY(pf_overload_entry)	next;
216 	struct pf_addr  		addr;
217 	sa_family_t			af;
218 	uint8_t				dir;
219 	struct pf_krule  		*rule;
220 };
221 
222 SLIST_HEAD(pf_overload_head, pf_overload_entry);
223 VNET_DEFINE_STATIC(struct pf_overload_head, pf_overloadqueue);
224 #define V_pf_overloadqueue	VNET(pf_overloadqueue)
225 VNET_DEFINE_STATIC(struct task, pf_overloadtask);
226 #define	V_pf_overloadtask	VNET(pf_overloadtask)
227 
228 static struct mtx_padalign pf_overloadqueue_mtx;
229 MTX_SYSINIT(pf_overloadqueue_mtx, &pf_overloadqueue_mtx,
230     "pf overload/flush queue", MTX_DEF);
231 #define	PF_OVERLOADQ_LOCK()	mtx_lock(&pf_overloadqueue_mtx)
232 #define	PF_OVERLOADQ_UNLOCK()	mtx_unlock(&pf_overloadqueue_mtx)
233 
234 VNET_DEFINE(struct pf_krulequeue, pf_unlinked_rules);
235 struct mtx_padalign pf_unlnkdrules_mtx;
236 MTX_SYSINIT(pf_unlnkdrules_mtx, &pf_unlnkdrules_mtx, "pf unlinked rules",
237     MTX_DEF);
238 
239 struct sx pf_config_lock;
240 SX_SYSINIT(pf_config_lock, &pf_config_lock, "pf config");
241 
242 struct mtx_padalign pf_table_stats_lock;
243 MTX_SYSINIT(pf_table_stats_lock, &pf_table_stats_lock, "pf table stats",
244     MTX_DEF);
245 
246 VNET_DEFINE_STATIC(uma_zone_t,	pf_sources_z);
247 #define	V_pf_sources_z	VNET(pf_sources_z)
248 uma_zone_t		pf_mtag_z;
249 VNET_DEFINE(uma_zone_t,	 pf_state_z);
250 VNET_DEFINE(uma_zone_t,	 pf_state_key_z);
251 
252 VNET_DEFINE(struct unrhdr64, pf_stateid);
253 
254 static void		 pf_src_tree_remove_state(struct pf_kstate *);
255 static void		 pf_init_threshold(struct pf_threshold *, u_int32_t,
256 			    u_int32_t);
257 static void		 pf_add_threshold(struct pf_threshold *);
258 static int		 pf_check_threshold(struct pf_threshold *);
259 
260 static void		 pf_change_ap(struct mbuf *, struct pf_addr *, u_int16_t *,
261 			    u_int16_t *, u_int16_t *, struct pf_addr *,
262 			    u_int16_t, u_int8_t, sa_family_t);
263 static int		 pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *,
264 			    struct tcphdr *, struct pf_state_peer *);
265 static void		 pf_change_icmp(struct pf_addr *, u_int16_t *,
266 			    struct pf_addr *, struct pf_addr *, u_int16_t,
267 			    u_int16_t *, u_int16_t *, u_int16_t *,
268 			    u_int16_t *, u_int8_t, sa_family_t);
269 static void		 pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t,
270 			    sa_family_t, struct pf_krule *, int);
271 static void		 pf_detach_state(struct pf_kstate *);
272 static int		 pf_state_key_attach(struct pf_state_key *,
273 			    struct pf_state_key *, struct pf_kstate *);
274 static void		 pf_state_key_detach(struct pf_kstate *, int);
275 static int		 pf_state_key_ctor(void *, int, void *, int);
276 static u_int32_t	 pf_tcp_iss(struct pf_pdesc *);
277 static int		 pf_dummynet(struct pf_pdesc *, struct pf_kstate *,
278 			    struct pf_krule *, struct mbuf **);
279 static int		 pf_dummynet_route(struct pf_pdesc *,
280 			    struct pf_kstate *, struct pf_krule *,
281 			    struct ifnet *, struct sockaddr *, struct mbuf **);
282 static int		 pf_test_eth_rule(int, struct pfi_kkif *,
283 			    struct mbuf **);
284 static int		 pf_test_rule(struct pf_krule **, struct pf_kstate **,
285 			    struct pfi_kkif *, struct mbuf *, int,
286 			    struct pf_pdesc *, struct pf_krule **,
287 			    struct pf_kruleset **, struct inpcb *);
288 static int		 pf_create_state(struct pf_krule *, struct pf_krule *,
289 			    struct pf_krule *, struct pf_pdesc *,
290 			    struct pf_ksrc_node *, struct pf_state_key *,
291 			    struct pf_state_key *, struct mbuf *, int,
292 			    u_int16_t, u_int16_t, int *, struct pfi_kkif *,
293 			    struct pf_kstate **, int, u_int16_t, u_int16_t,
294 			    int, struct pf_krule_slist *);
295 static int		 pf_test_fragment(struct pf_krule **, struct pfi_kkif *,
296 			    struct mbuf *, void *, struct pf_pdesc *,
297 			    struct pf_krule **, struct pf_kruleset **);
298 static int		 pf_tcp_track_full(struct pf_kstate **,
299 			    struct pfi_kkif *, struct mbuf *, int,
300 			    struct pf_pdesc *, u_short *, int *);
301 static int		 pf_tcp_track_sloppy(struct pf_kstate **,
302 			    struct pf_pdesc *, u_short *);
303 static int		 pf_test_state_tcp(struct pf_kstate **,
304 			    struct pfi_kkif *, struct mbuf *, int,
305 			    void *, struct pf_pdesc *, u_short *);
306 static int		 pf_test_state_udp(struct pf_kstate **,
307 			    struct pfi_kkif *, struct mbuf *, int,
308 			    void *, struct pf_pdesc *);
309 static int		 pf_test_state_icmp(struct pf_kstate **,
310 			    struct pfi_kkif *, struct mbuf *, int,
311 			    void *, struct pf_pdesc *, u_short *);
312 static void		 pf_sctp_multihome_delayed(struct pf_pdesc *, int,
313 			    struct pfi_kkif *, struct pf_kstate *, int);
314 static int		 pf_test_state_sctp(struct pf_kstate **,
315 			    struct pfi_kkif *, struct mbuf *, int,
316 			    void *, struct pf_pdesc *, u_short *);
317 static int		 pf_test_state_other(struct pf_kstate **,
318 			    struct pfi_kkif *, struct mbuf *, struct pf_pdesc *);
319 static u_int16_t	 pf_calc_mss(struct pf_addr *, sa_family_t,
320 				int, u_int16_t);
321 static int		 pf_check_proto_cksum(struct mbuf *, int, int,
322 			    u_int8_t, sa_family_t);
323 static void		 pf_print_state_parts(struct pf_kstate *,
324 			    struct pf_state_key *, struct pf_state_key *);
325 static void		 pf_patch_8(struct mbuf *, u_int16_t *, u_int8_t *, u_int8_t,
326 			    bool, u_int8_t);
327 static struct pf_kstate	*pf_find_state(struct pfi_kkif *,
328 			    struct pf_state_key_cmp *, u_int);
329 static int		 pf_src_connlimit(struct pf_kstate **);
330 static void		 pf_overload_task(void *v, int pending);
331 static u_short		 pf_insert_src_node(struct pf_ksrc_node **,
332 			    struct pf_krule *, struct pf_addr *, sa_family_t);
333 static u_int		 pf_purge_expired_states(u_int, int);
334 static void		 pf_purge_unlinked_rules(void);
335 static int		 pf_mtag_uminit(void *, int, int);
336 static void		 pf_mtag_free(struct m_tag *);
337 static void		 pf_packet_rework_nat(struct mbuf *, struct pf_pdesc *,
338 			    int, struct pf_state_key *);
339 #ifdef INET
340 static void		 pf_route(struct mbuf **, struct pf_krule *,
341 			    struct ifnet *, struct pf_kstate *,
342 			    struct pf_pdesc *, struct inpcb *);
343 #endif /* INET */
344 #ifdef INET6
345 static void		 pf_change_a6(struct pf_addr *, u_int16_t *,
346 			    struct pf_addr *, u_int8_t);
347 static void		 pf_route6(struct mbuf **, struct pf_krule *,
348 			    struct ifnet *, struct pf_kstate *,
349 			    struct pf_pdesc *, struct inpcb *);
350 #endif /* INET6 */
351 static __inline void pf_set_protostate(struct pf_kstate *, int, u_int8_t);
352 
353 int in4_cksum(struct mbuf *m, u_int8_t nxt, int off, int len);
354 
355 extern int pf_end_threads;
356 extern struct proc *pf_purge_proc;
357 
358 VNET_DEFINE(struct pf_limit, pf_limits[PF_LIMIT_MAX]);
359 
360 #define	PACKET_UNDO_NAT(_m, _pd, _off, _s)		\
361 	do {								\
362 		struct pf_state_key *nk;				\
363 		if ((pd->dir) == PF_OUT)					\
364 			nk = (_s)->key[PF_SK_STACK];			\
365 		else							\
366 			nk = (_s)->key[PF_SK_WIRE];			\
367 		pf_packet_rework_nat(_m, _pd, _off, nk);		\
368 	} while (0)
369 
370 #define	PACKET_LOOPED(pd)	((pd)->pf_mtag &&			\
371 				 (pd)->pf_mtag->flags & PF_MTAG_FLAG_PACKET_LOOPED)
372 
373 #define	STATE_LOOKUP(i, k, s, pd)					\
374 	do {								\
375 		(s) = pf_find_state((i), (k), (pd->dir));			\
376 		SDT_PROBE5(pf, ip, state, lookup, i, k, (pd->dir), pd, (s));	\
377 		if ((s) == NULL)					\
378 			return (PF_DROP);				\
379 		if (PACKET_LOOPED(pd))					\
380 			return (PF_PASS);				\
381 	} while (0)
382 
383 #define	BOUND_IFACE(r, k) \
384 	((r)->rule_flag & PFRULE_IFBOUND) ? (k) : V_pfi_all
385 
386 #define	STATE_INC_COUNTERS(s)						\
387 	do {								\
388 		struct pf_krule_item *mrm;				\
389 		counter_u64_add(s->rule.ptr->states_cur, 1);		\
390 		counter_u64_add(s->rule.ptr->states_tot, 1);		\
391 		if (s->anchor.ptr != NULL) {				\
392 			counter_u64_add(s->anchor.ptr->states_cur, 1);	\
393 			counter_u64_add(s->anchor.ptr->states_tot, 1);	\
394 		}							\
395 		if (s->nat_rule.ptr != NULL) {				\
396 			counter_u64_add(s->nat_rule.ptr->states_cur, 1);\
397 			counter_u64_add(s->nat_rule.ptr->states_tot, 1);\
398 		}							\
399 		SLIST_FOREACH(mrm, &s->match_rules, entry) {		\
400 			counter_u64_add(mrm->r->states_cur, 1);		\
401 			counter_u64_add(mrm->r->states_tot, 1);		\
402 		}							\
403 	} while (0)
404 
405 #define	STATE_DEC_COUNTERS(s)						\
406 	do {								\
407 		struct pf_krule_item *mrm;				\
408 		if (s->nat_rule.ptr != NULL)				\
409 			counter_u64_add(s->nat_rule.ptr->states_cur, -1);\
410 		if (s->anchor.ptr != NULL)				\
411 			counter_u64_add(s->anchor.ptr->states_cur, -1);	\
412 		counter_u64_add(s->rule.ptr->states_cur, -1);		\
413 		SLIST_FOREACH(mrm, &s->match_rules, entry)		\
414 			counter_u64_add(mrm->r->states_cur, -1);	\
415 	} while (0)
416 
417 MALLOC_DEFINE(M_PFHASH, "pf_hash", "pf(4) hash header structures");
418 MALLOC_DEFINE(M_PF_RULE_ITEM, "pf_krule_item", "pf(4) rule items");
419 VNET_DEFINE(struct pf_keyhash *, pf_keyhash);
420 VNET_DEFINE(struct pf_idhash *, pf_idhash);
421 VNET_DEFINE(struct pf_srchash *, pf_srchash);
422 
423 SYSCTL_NODE(_net, OID_AUTO, pf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
424     "pf(4)");
425 
426 u_long	pf_hashmask;
427 u_long	pf_srchashmask;
428 static u_long	pf_hashsize;
429 static u_long	pf_srchashsize;
430 u_long	pf_ioctl_maxcount = 65535;
431 
432 SYSCTL_ULONG(_net_pf, OID_AUTO, states_hashsize, CTLFLAG_RDTUN,
433     &pf_hashsize, 0, "Size of pf(4) states hashtable");
434 SYSCTL_ULONG(_net_pf, OID_AUTO, source_nodes_hashsize, CTLFLAG_RDTUN,
435     &pf_srchashsize, 0, "Size of pf(4) source nodes hashtable");
436 SYSCTL_ULONG(_net_pf, OID_AUTO, request_maxcount, CTLFLAG_RWTUN,
437     &pf_ioctl_maxcount, 0, "Maximum number of tables, addresses, ... in a single ioctl() call");
438 
439 VNET_DEFINE(void *, pf_swi_cookie);
440 VNET_DEFINE(struct intr_event *, pf_swi_ie);
441 
442 VNET_DEFINE(uint32_t, pf_hashseed);
443 #define	V_pf_hashseed	VNET(pf_hashseed)
444 
445 static void
446 pf_sctp_checksum(struct mbuf *m, int off)
447 {
448 	uint32_t sum = 0;
449 
450 	/* Zero out the checksum, to enable recalculation. */
451 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
452 	    sizeof(sum), (caddr_t)&sum);
453 
454 	sum = sctp_calculate_cksum(m, off);
455 
456 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
457 	    sizeof(sum), (caddr_t)&sum);
458 }
459 
460 int
461 pf_addr_cmp(struct pf_addr *a, struct pf_addr *b, sa_family_t af)
462 {
463 
464 	switch (af) {
465 #ifdef INET
466 	case AF_INET:
467 		if (a->addr32[0] > b->addr32[0])
468 			return (1);
469 		if (a->addr32[0] < b->addr32[0])
470 			return (-1);
471 		break;
472 #endif /* INET */
473 #ifdef INET6
474 	case AF_INET6:
475 		if (a->addr32[3] > b->addr32[3])
476 			return (1);
477 		if (a->addr32[3] < b->addr32[3])
478 			return (-1);
479 		if (a->addr32[2] > b->addr32[2])
480 			return (1);
481 		if (a->addr32[2] < b->addr32[2])
482 			return (-1);
483 		if (a->addr32[1] > b->addr32[1])
484 			return (1);
485 		if (a->addr32[1] < b->addr32[1])
486 			return (-1);
487 		if (a->addr32[0] > b->addr32[0])
488 			return (1);
489 		if (a->addr32[0] < b->addr32[0])
490 			return (-1);
491 		break;
492 #endif /* INET6 */
493 	default:
494 		panic("%s: unknown address family %u", __func__, af);
495 	}
496 	return (0);
497 }
498 
499 static void
500 pf_packet_rework_nat(struct mbuf *m, struct pf_pdesc *pd, int off,
501 	struct pf_state_key *nk)
502 {
503 
504 	switch (pd->proto) {
505 	case IPPROTO_TCP: {
506 		struct tcphdr *th = &pd->hdr.tcp;
507 
508 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
509 			pf_change_ap(m, pd->src, &th->th_sport, pd->ip_sum,
510 			    &th->th_sum, &nk->addr[pd->sidx],
511 			    nk->port[pd->sidx], 0, pd->af);
512 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
513 			pf_change_ap(m, pd->dst, &th->th_dport, pd->ip_sum,
514 			    &th->th_sum, &nk->addr[pd->didx],
515 			    nk->port[pd->didx], 0, pd->af);
516 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
517 		break;
518 	}
519 	case IPPROTO_UDP: {
520 		struct udphdr *uh = &pd->hdr.udp;
521 
522 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
523 			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
524 			    &uh->uh_sum, &nk->addr[pd->sidx],
525 			    nk->port[pd->sidx], 1, pd->af);
526 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
527 			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
528 			    &uh->uh_sum, &nk->addr[pd->didx],
529 			    nk->port[pd->didx], 1, pd->af);
530 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
531 		break;
532 	}
533 	case IPPROTO_SCTP: {
534 		struct sctphdr *sh = &pd->hdr.sctp;
535 		uint16_t checksum = 0;
536 
537 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
538 			pf_change_ap(m, pd->src, &sh->src_port, pd->ip_sum,
539 			    &checksum, &nk->addr[pd->sidx],
540 			    nk->port[pd->sidx], 1, pd->af);
541 		}
542 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
543 			pf_change_ap(m, pd->dst, &sh->dest_port, pd->ip_sum,
544 			    &checksum, &nk->addr[pd->didx],
545 			    nk->port[pd->didx], 1, pd->af);
546 		}
547 
548 		break;
549 	}
550 	case IPPROTO_ICMP: {
551 		struct icmp *ih = &pd->hdr.icmp;
552 
553 		if (nk->port[pd->sidx] != ih->icmp_id) {
554 			pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
555 			    ih->icmp_cksum, ih->icmp_id,
556 			    nk->port[pd->sidx], 0);
557 			ih->icmp_id = nk->port[pd->sidx];
558 			pd->sport = &ih->icmp_id;
559 
560 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)ih);
561 		}
562 		/* FALLTHROUGH */
563 	}
564 	default:
565 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
566 			switch (pd->af) {
567 			case AF_INET:
568 				pf_change_a(&pd->src->v4.s_addr,
569 				    pd->ip_sum, nk->addr[pd->sidx].v4.s_addr,
570 				    0);
571 				break;
572 			case AF_INET6:
573 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
574 				break;
575 			}
576 		}
577 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
578 			switch (pd->af) {
579 			case AF_INET:
580 				pf_change_a(&pd->dst->v4.s_addr,
581 				    pd->ip_sum, nk->addr[pd->didx].v4.s_addr,
582 				    0);
583 				break;
584 			case AF_INET6:
585 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
586 				break;
587 			}
588 		}
589 		break;
590 	}
591 }
592 
593 static __inline uint32_t
594 pf_hashkey(struct pf_state_key *sk)
595 {
596 	uint32_t h;
597 
598 	h = murmur3_32_hash32((uint32_t *)sk,
599 	    sizeof(struct pf_state_key_cmp)/sizeof(uint32_t),
600 	    V_pf_hashseed);
601 
602 	return (h & pf_hashmask);
603 }
604 
605 static __inline uint32_t
606 pf_hashsrc(struct pf_addr *addr, sa_family_t af)
607 {
608 	uint32_t h;
609 
610 	switch (af) {
611 	case AF_INET:
612 		h = murmur3_32_hash32((uint32_t *)&addr->v4,
613 		    sizeof(addr->v4)/sizeof(uint32_t), V_pf_hashseed);
614 		break;
615 	case AF_INET6:
616 		h = murmur3_32_hash32((uint32_t *)&addr->v6,
617 		    sizeof(addr->v6)/sizeof(uint32_t), V_pf_hashseed);
618 		break;
619 	default:
620 		panic("%s: unknown address family %u", __func__, af);
621 	}
622 
623 	return (h & pf_srchashmask);
624 }
625 
626 #ifdef ALTQ
627 static int
628 pf_state_hash(struct pf_kstate *s)
629 {
630 	u_int32_t hv = (intptr_t)s / sizeof(*s);
631 
632 	hv ^= crc32(&s->src, sizeof(s->src));
633 	hv ^= crc32(&s->dst, sizeof(s->dst));
634 	if (hv == 0)
635 		hv = 1;
636 	return (hv);
637 }
638 #endif
639 
640 static __inline void
641 pf_set_protostate(struct pf_kstate *s, int which, u_int8_t newstate)
642 {
643 	if (which == PF_PEER_DST || which == PF_PEER_BOTH)
644 		s->dst.state = newstate;
645 	if (which == PF_PEER_DST)
646 		return;
647 	if (s->src.state == newstate)
648 		return;
649 	if (s->creatorid == V_pf_status.hostid &&
650 	    s->key[PF_SK_STACK] != NULL &&
651 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP &&
652 	    !(TCPS_HAVEESTABLISHED(s->src.state) ||
653 	    s->src.state == TCPS_CLOSED) &&
654 	    (TCPS_HAVEESTABLISHED(newstate) || newstate == TCPS_CLOSED))
655 		atomic_add_32(&V_pf_status.states_halfopen, -1);
656 
657 	s->src.state = newstate;
658 }
659 
660 #ifdef INET6
661 void
662 pf_addrcpy(struct pf_addr *dst, struct pf_addr *src, sa_family_t af)
663 {
664 	switch (af) {
665 #ifdef INET
666 	case AF_INET:
667 		dst->addr32[0] = src->addr32[0];
668 		break;
669 #endif /* INET */
670 	case AF_INET6:
671 		dst->addr32[0] = src->addr32[0];
672 		dst->addr32[1] = src->addr32[1];
673 		dst->addr32[2] = src->addr32[2];
674 		dst->addr32[3] = src->addr32[3];
675 		break;
676 	}
677 }
678 #endif /* INET6 */
679 
680 static void
681 pf_init_threshold(struct pf_threshold *threshold,
682     u_int32_t limit, u_int32_t seconds)
683 {
684 	threshold->limit = limit * PF_THRESHOLD_MULT;
685 	threshold->seconds = seconds;
686 	threshold->count = 0;
687 	threshold->last = time_uptime;
688 }
689 
690 static void
691 pf_add_threshold(struct pf_threshold *threshold)
692 {
693 	u_int32_t t = time_uptime, diff = t - threshold->last;
694 
695 	if (diff >= threshold->seconds)
696 		threshold->count = 0;
697 	else
698 		threshold->count -= threshold->count * diff /
699 		    threshold->seconds;
700 	threshold->count += PF_THRESHOLD_MULT;
701 	threshold->last = t;
702 }
703 
704 static int
705 pf_check_threshold(struct pf_threshold *threshold)
706 {
707 	return (threshold->count > threshold->limit);
708 }
709 
710 static int
711 pf_src_connlimit(struct pf_kstate **state)
712 {
713 	struct pf_overload_entry *pfoe;
714 	int bad = 0;
715 
716 	PF_STATE_LOCK_ASSERT(*state);
717 	/*
718 	 * XXXKS: The src node is accessed unlocked!
719 	 * PF_SRC_NODE_LOCK_ASSERT((*state)->src_node);
720 	 */
721 
722 	(*state)->src_node->conn++;
723 	(*state)->src.tcp_est = 1;
724 	pf_add_threshold(&(*state)->src_node->conn_rate);
725 
726 	if ((*state)->rule.ptr->max_src_conn &&
727 	    (*state)->rule.ptr->max_src_conn <
728 	    (*state)->src_node->conn) {
729 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONN], 1);
730 		bad++;
731 	}
732 
733 	if ((*state)->rule.ptr->max_src_conn_rate.limit &&
734 	    pf_check_threshold(&(*state)->src_node->conn_rate)) {
735 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONNRATE], 1);
736 		bad++;
737 	}
738 
739 	if (!bad)
740 		return (0);
741 
742 	/* Kill this state. */
743 	(*state)->timeout = PFTM_PURGE;
744 	pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
745 
746 	if ((*state)->rule.ptr->overload_tbl == NULL)
747 		return (1);
748 
749 	/* Schedule overloading and flushing task. */
750 	pfoe = malloc(sizeof(*pfoe), M_PFTEMP, M_NOWAIT);
751 	if (pfoe == NULL)
752 		return (1);	/* too bad :( */
753 
754 	bcopy(&(*state)->src_node->addr, &pfoe->addr, sizeof(pfoe->addr));
755 	pfoe->af = (*state)->key[PF_SK_WIRE]->af;
756 	pfoe->rule = (*state)->rule.ptr;
757 	pfoe->dir = (*state)->direction;
758 	PF_OVERLOADQ_LOCK();
759 	SLIST_INSERT_HEAD(&V_pf_overloadqueue, pfoe, next);
760 	PF_OVERLOADQ_UNLOCK();
761 	taskqueue_enqueue(taskqueue_swi, &V_pf_overloadtask);
762 
763 	return (1);
764 }
765 
766 static void
767 pf_overload_task(void *v, int pending)
768 {
769 	struct pf_overload_head queue;
770 	struct pfr_addr p;
771 	struct pf_overload_entry *pfoe, *pfoe1;
772 	uint32_t killed = 0;
773 
774 	CURVNET_SET((struct vnet *)v);
775 
776 	PF_OVERLOADQ_LOCK();
777 	queue = V_pf_overloadqueue;
778 	SLIST_INIT(&V_pf_overloadqueue);
779 	PF_OVERLOADQ_UNLOCK();
780 
781 	bzero(&p, sizeof(p));
782 	SLIST_FOREACH(pfoe, &queue, next) {
783 		counter_u64_add(V_pf_status.lcounters[LCNT_OVERLOAD_TABLE], 1);
784 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
785 			printf("%s: blocking address ", __func__);
786 			pf_print_host(&pfoe->addr, 0, pfoe->af);
787 			printf("\n");
788 		}
789 
790 		p.pfra_af = pfoe->af;
791 		switch (pfoe->af) {
792 #ifdef INET
793 		case AF_INET:
794 			p.pfra_net = 32;
795 			p.pfra_ip4addr = pfoe->addr.v4;
796 			break;
797 #endif
798 #ifdef INET6
799 		case AF_INET6:
800 			p.pfra_net = 128;
801 			p.pfra_ip6addr = pfoe->addr.v6;
802 			break;
803 #endif
804 		}
805 
806 		PF_RULES_WLOCK();
807 		pfr_insert_kentry(pfoe->rule->overload_tbl, &p, time_second);
808 		PF_RULES_WUNLOCK();
809 	}
810 
811 	/*
812 	 * Remove those entries, that don't need flushing.
813 	 */
814 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
815 		if (pfoe->rule->flush == 0) {
816 			SLIST_REMOVE(&queue, pfoe, pf_overload_entry, next);
817 			free(pfoe, M_PFTEMP);
818 		} else
819 			counter_u64_add(
820 			    V_pf_status.lcounters[LCNT_OVERLOAD_FLUSH], 1);
821 
822 	/* If nothing to flush, return. */
823 	if (SLIST_EMPTY(&queue)) {
824 		CURVNET_RESTORE();
825 		return;
826 	}
827 
828 	for (int i = 0; i <= pf_hashmask; i++) {
829 		struct pf_idhash *ih = &V_pf_idhash[i];
830 		struct pf_state_key *sk;
831 		struct pf_kstate *s;
832 
833 		PF_HASHROW_LOCK(ih);
834 		LIST_FOREACH(s, &ih->states, entry) {
835 		    sk = s->key[PF_SK_WIRE];
836 		    SLIST_FOREACH(pfoe, &queue, next)
837 			if (sk->af == pfoe->af &&
838 			    ((pfoe->rule->flush & PF_FLUSH_GLOBAL) ||
839 			    pfoe->rule == s->rule.ptr) &&
840 			    ((pfoe->dir == PF_OUT &&
841 			    PF_AEQ(&pfoe->addr, &sk->addr[1], sk->af)) ||
842 			    (pfoe->dir == PF_IN &&
843 			    PF_AEQ(&pfoe->addr, &sk->addr[0], sk->af)))) {
844 				s->timeout = PFTM_PURGE;
845 				pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
846 				killed++;
847 			}
848 		}
849 		PF_HASHROW_UNLOCK(ih);
850 	}
851 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
852 		free(pfoe, M_PFTEMP);
853 	if (V_pf_status.debug >= PF_DEBUG_MISC)
854 		printf("%s: %u states killed", __func__, killed);
855 
856 	CURVNET_RESTORE();
857 }
858 
859 /*
860  * Can return locked on failure, so that we can consistently
861  * allocate and insert a new one.
862  */
863 struct pf_ksrc_node *
864 pf_find_src_node(struct pf_addr *src, struct pf_krule *rule, sa_family_t af,
865 	struct pf_srchash **sh, bool returnlocked)
866 {
867 	struct pf_ksrc_node *n;
868 
869 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_SEARCH], 1);
870 
871 	*sh = &V_pf_srchash[pf_hashsrc(src, af)];
872 	PF_HASHROW_LOCK(*sh);
873 	LIST_FOREACH(n, &(*sh)->nodes, entry)
874 		if (n->rule.ptr == rule && n->af == af &&
875 		    ((af == AF_INET && n->addr.v4.s_addr == src->v4.s_addr) ||
876 		    (af == AF_INET6 && bcmp(&n->addr, src, sizeof(*src)) == 0)))
877 			break;
878 
879 	if (n != NULL) {
880 		n->states++;
881 		PF_HASHROW_UNLOCK(*sh);
882 	} else if (returnlocked == false)
883 		PF_HASHROW_UNLOCK(*sh);
884 
885 	return (n);
886 }
887 
888 static void
889 pf_free_src_node(struct pf_ksrc_node *sn)
890 {
891 
892 	for (int i = 0; i < 2; i++) {
893 		counter_u64_free(sn->bytes[i]);
894 		counter_u64_free(sn->packets[i]);
895 	}
896 	uma_zfree(V_pf_sources_z, sn);
897 }
898 
899 static u_short
900 pf_insert_src_node(struct pf_ksrc_node **sn, struct pf_krule *rule,
901     struct pf_addr *src, sa_family_t af)
902 {
903 	u_short			 reason = 0;
904 	struct pf_srchash	*sh = NULL;
905 
906 	KASSERT((rule->rule_flag & PFRULE_SRCTRACK ||
907 	    rule->rpool.opts & PF_POOL_STICKYADDR),
908 	    ("%s for non-tracking rule %p", __func__, rule));
909 
910 	if (*sn == NULL)
911 		*sn = pf_find_src_node(src, rule, af, &sh, true);
912 
913 	if (*sn == NULL) {
914 		PF_HASHROW_ASSERT(sh);
915 
916 		if (rule->max_src_nodes &&
917 		    counter_u64_fetch(rule->src_nodes) >= rule->max_src_nodes) {
918 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCNODES], 1);
919 			PF_HASHROW_UNLOCK(sh);
920 			reason = PFRES_SRCLIMIT;
921 			goto done;
922 		}
923 
924 		(*sn) = uma_zalloc(V_pf_sources_z, M_NOWAIT | M_ZERO);
925 		if ((*sn) == NULL) {
926 			PF_HASHROW_UNLOCK(sh);
927 			reason = PFRES_MEMORY;
928 			goto done;
929 		}
930 
931 		for (int i = 0; i < 2; i++) {
932 			(*sn)->bytes[i] = counter_u64_alloc(M_NOWAIT);
933 			(*sn)->packets[i] = counter_u64_alloc(M_NOWAIT);
934 
935 			if ((*sn)->bytes[i] == NULL || (*sn)->packets[i] == NULL) {
936 				pf_free_src_node(*sn);
937 				PF_HASHROW_UNLOCK(sh);
938 				reason = PFRES_MEMORY;
939 				goto done;
940 			}
941 		}
942 
943 		pf_init_threshold(&(*sn)->conn_rate,
944 		    rule->max_src_conn_rate.limit,
945 		    rule->max_src_conn_rate.seconds);
946 
947 		MPASS((*sn)->lock == NULL);
948 		(*sn)->lock = &sh->lock;
949 
950 		(*sn)->af = af;
951 		(*sn)->rule.ptr = rule;
952 		PF_ACPY(&(*sn)->addr, src, af);
953 		LIST_INSERT_HEAD(&sh->nodes, *sn, entry);
954 		(*sn)->creation = time_uptime;
955 		(*sn)->ruletype = rule->action;
956 		(*sn)->states = 1;
957 		if ((*sn)->rule.ptr != NULL)
958 			counter_u64_add((*sn)->rule.ptr->src_nodes, 1);
959 		PF_HASHROW_UNLOCK(sh);
960 		counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_INSERT], 1);
961 	} else {
962 		if (rule->max_src_states &&
963 		    (*sn)->states >= rule->max_src_states) {
964 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCSTATES],
965 			    1);
966 			reason = PFRES_SRCLIMIT;
967 			goto done;
968 		}
969 	}
970 done:
971 	return (reason);
972 }
973 
974 void
975 pf_unlink_src_node(struct pf_ksrc_node *src)
976 {
977 	PF_SRC_NODE_LOCK_ASSERT(src);
978 
979 	LIST_REMOVE(src, entry);
980 	if (src->rule.ptr)
981 		counter_u64_add(src->rule.ptr->src_nodes, -1);
982 }
983 
984 u_int
985 pf_free_src_nodes(struct pf_ksrc_node_list *head)
986 {
987 	struct pf_ksrc_node *sn, *tmp;
988 	u_int count = 0;
989 
990 	LIST_FOREACH_SAFE(sn, head, entry, tmp) {
991 		pf_free_src_node(sn);
992 		count++;
993 	}
994 
995 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], count);
996 
997 	return (count);
998 }
999 
1000 void
1001 pf_mtag_initialize(void)
1002 {
1003 
1004 	pf_mtag_z = uma_zcreate("pf mtags", sizeof(struct m_tag) +
1005 	    sizeof(struct pf_mtag), NULL, NULL, pf_mtag_uminit, NULL,
1006 	    UMA_ALIGN_PTR, 0);
1007 }
1008 
1009 /* Per-vnet data storage structures initialization. */
1010 void
1011 pf_initialize(void)
1012 {
1013 	struct pf_keyhash	*kh;
1014 	struct pf_idhash	*ih;
1015 	struct pf_srchash	*sh;
1016 	u_int i;
1017 
1018 	if (pf_hashsize == 0 || !powerof2(pf_hashsize))
1019 		pf_hashsize = PF_HASHSIZ;
1020 	if (pf_srchashsize == 0 || !powerof2(pf_srchashsize))
1021 		pf_srchashsize = PF_SRCHASHSIZ;
1022 
1023 	V_pf_hashseed = arc4random();
1024 
1025 	/* States and state keys storage. */
1026 	V_pf_state_z = uma_zcreate("pf states", sizeof(struct pf_kstate),
1027 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1028 	V_pf_limits[PF_LIMIT_STATES].zone = V_pf_state_z;
1029 	uma_zone_set_max(V_pf_state_z, PFSTATE_HIWAT);
1030 	uma_zone_set_warning(V_pf_state_z, "PF states limit reached");
1031 
1032 	V_pf_state_key_z = uma_zcreate("pf state keys",
1033 	    sizeof(struct pf_state_key), pf_state_key_ctor, NULL, NULL, NULL,
1034 	    UMA_ALIGN_PTR, 0);
1035 
1036 	V_pf_keyhash = mallocarray(pf_hashsize, sizeof(struct pf_keyhash),
1037 	    M_PFHASH, M_NOWAIT | M_ZERO);
1038 	V_pf_idhash = mallocarray(pf_hashsize, sizeof(struct pf_idhash),
1039 	    M_PFHASH, M_NOWAIT | M_ZERO);
1040 	if (V_pf_keyhash == NULL || V_pf_idhash == NULL) {
1041 		printf("pf: Unable to allocate memory for "
1042 		    "state_hashsize %lu.\n", pf_hashsize);
1043 
1044 		free(V_pf_keyhash, M_PFHASH);
1045 		free(V_pf_idhash, M_PFHASH);
1046 
1047 		pf_hashsize = PF_HASHSIZ;
1048 		V_pf_keyhash = mallocarray(pf_hashsize,
1049 		    sizeof(struct pf_keyhash), M_PFHASH, M_WAITOK | M_ZERO);
1050 		V_pf_idhash = mallocarray(pf_hashsize,
1051 		    sizeof(struct pf_idhash), M_PFHASH, M_WAITOK | M_ZERO);
1052 	}
1053 
1054 	pf_hashmask = pf_hashsize - 1;
1055 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= pf_hashmask;
1056 	    i++, kh++, ih++) {
1057 		mtx_init(&kh->lock, "pf_keyhash", NULL, MTX_DEF | MTX_DUPOK);
1058 		mtx_init(&ih->lock, "pf_idhash", NULL, MTX_DEF);
1059 	}
1060 
1061 	/* Source nodes. */
1062 	V_pf_sources_z = uma_zcreate("pf source nodes",
1063 	    sizeof(struct pf_ksrc_node), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
1064 	    0);
1065 	V_pf_limits[PF_LIMIT_SRC_NODES].zone = V_pf_sources_z;
1066 	uma_zone_set_max(V_pf_sources_z, PFSNODE_HIWAT);
1067 	uma_zone_set_warning(V_pf_sources_z, "PF source nodes limit reached");
1068 
1069 	V_pf_srchash = mallocarray(pf_srchashsize,
1070 	    sizeof(struct pf_srchash), M_PFHASH, M_NOWAIT | M_ZERO);
1071 	if (V_pf_srchash == NULL) {
1072 		printf("pf: Unable to allocate memory for "
1073 		    "source_hashsize %lu.\n", pf_srchashsize);
1074 
1075 		pf_srchashsize = PF_SRCHASHSIZ;
1076 		V_pf_srchash = mallocarray(pf_srchashsize,
1077 		    sizeof(struct pf_srchash), M_PFHASH, M_WAITOK | M_ZERO);
1078 	}
1079 
1080 	pf_srchashmask = pf_srchashsize - 1;
1081 	for (i = 0, sh = V_pf_srchash; i <= pf_srchashmask; i++, sh++)
1082 		mtx_init(&sh->lock, "pf_srchash", NULL, MTX_DEF);
1083 
1084 	/* ALTQ */
1085 	TAILQ_INIT(&V_pf_altqs[0]);
1086 	TAILQ_INIT(&V_pf_altqs[1]);
1087 	TAILQ_INIT(&V_pf_altqs[2]);
1088 	TAILQ_INIT(&V_pf_altqs[3]);
1089 	TAILQ_INIT(&V_pf_pabuf);
1090 	V_pf_altqs_active = &V_pf_altqs[0];
1091 	V_pf_altq_ifs_active = &V_pf_altqs[1];
1092 	V_pf_altqs_inactive = &V_pf_altqs[2];
1093 	V_pf_altq_ifs_inactive = &V_pf_altqs[3];
1094 
1095 	/* Send & overload+flush queues. */
1096 	STAILQ_INIT(&V_pf_sendqueue);
1097 	SLIST_INIT(&V_pf_overloadqueue);
1098 	TASK_INIT(&V_pf_overloadtask, 0, pf_overload_task, curvnet);
1099 
1100 	/* Unlinked, but may be referenced rules. */
1101 	TAILQ_INIT(&V_pf_unlinked_rules);
1102 }
1103 
1104 void
1105 pf_mtag_cleanup(void)
1106 {
1107 
1108 	uma_zdestroy(pf_mtag_z);
1109 }
1110 
1111 void
1112 pf_cleanup(void)
1113 {
1114 	struct pf_keyhash	*kh;
1115 	struct pf_idhash	*ih;
1116 	struct pf_srchash	*sh;
1117 	struct pf_send_entry	*pfse, *next;
1118 	u_int i;
1119 
1120 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= pf_hashmask;
1121 	    i++, kh++, ih++) {
1122 		KASSERT(LIST_EMPTY(&kh->keys), ("%s: key hash not empty",
1123 		    __func__));
1124 		KASSERT(LIST_EMPTY(&ih->states), ("%s: id hash not empty",
1125 		    __func__));
1126 		mtx_destroy(&kh->lock);
1127 		mtx_destroy(&ih->lock);
1128 	}
1129 	free(V_pf_keyhash, M_PFHASH);
1130 	free(V_pf_idhash, M_PFHASH);
1131 
1132 	for (i = 0, sh = V_pf_srchash; i <= pf_srchashmask; i++, sh++) {
1133 		KASSERT(LIST_EMPTY(&sh->nodes),
1134 		    ("%s: source node hash not empty", __func__));
1135 		mtx_destroy(&sh->lock);
1136 	}
1137 	free(V_pf_srchash, M_PFHASH);
1138 
1139 	STAILQ_FOREACH_SAFE(pfse, &V_pf_sendqueue, pfse_next, next) {
1140 		m_freem(pfse->pfse_m);
1141 		free(pfse, M_PFTEMP);
1142 	}
1143 
1144 	uma_zdestroy(V_pf_sources_z);
1145 	uma_zdestroy(V_pf_state_z);
1146 	uma_zdestroy(V_pf_state_key_z);
1147 }
1148 
1149 static int
1150 pf_mtag_uminit(void *mem, int size, int how)
1151 {
1152 	struct m_tag *t;
1153 
1154 	t = (struct m_tag *)mem;
1155 	t->m_tag_cookie = MTAG_ABI_COMPAT;
1156 	t->m_tag_id = PACKET_TAG_PF;
1157 	t->m_tag_len = sizeof(struct pf_mtag);
1158 	t->m_tag_free = pf_mtag_free;
1159 
1160 	return (0);
1161 }
1162 
1163 static void
1164 pf_mtag_free(struct m_tag *t)
1165 {
1166 
1167 	uma_zfree(pf_mtag_z, t);
1168 }
1169 
1170 struct pf_mtag *
1171 pf_get_mtag(struct mbuf *m)
1172 {
1173 	struct m_tag *mtag;
1174 
1175 	if ((mtag = m_tag_find(m, PACKET_TAG_PF, NULL)) != NULL)
1176 		return ((struct pf_mtag *)(mtag + 1));
1177 
1178 	mtag = uma_zalloc(pf_mtag_z, M_NOWAIT);
1179 	if (mtag == NULL)
1180 		return (NULL);
1181 	bzero(mtag + 1, sizeof(struct pf_mtag));
1182 	m_tag_prepend(m, mtag);
1183 
1184 	return ((struct pf_mtag *)(mtag + 1));
1185 }
1186 
1187 static int
1188 pf_state_key_attach(struct pf_state_key *skw, struct pf_state_key *sks,
1189     struct pf_kstate *s)
1190 {
1191 	struct pf_keyhash	*khs, *khw, *kh;
1192 	struct pf_state_key	*sk, *cur;
1193 	struct pf_kstate	*si, *olds = NULL;
1194 	int idx;
1195 
1196 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1197 	KASSERT(s->key[PF_SK_WIRE] == NULL, ("%s: state has key", __func__));
1198 	KASSERT(s->key[PF_SK_STACK] == NULL, ("%s: state has key", __func__));
1199 
1200 	/*
1201 	 * We need to lock hash slots of both keys. To avoid deadlock
1202 	 * we always lock the slot with lower address first. Unlock order
1203 	 * isn't important.
1204 	 *
1205 	 * We also need to lock ID hash slot before dropping key
1206 	 * locks. On success we return with ID hash slot locked.
1207 	 */
1208 
1209 	if (skw == sks) {
1210 		khs = khw = &V_pf_keyhash[pf_hashkey(skw)];
1211 		PF_HASHROW_LOCK(khs);
1212 	} else {
1213 		khs = &V_pf_keyhash[pf_hashkey(sks)];
1214 		khw = &V_pf_keyhash[pf_hashkey(skw)];
1215 		if (khs == khw) {
1216 			PF_HASHROW_LOCK(khs);
1217 		} else if (khs < khw) {
1218 			PF_HASHROW_LOCK(khs);
1219 			PF_HASHROW_LOCK(khw);
1220 		} else {
1221 			PF_HASHROW_LOCK(khw);
1222 			PF_HASHROW_LOCK(khs);
1223 		}
1224 	}
1225 
1226 #define	KEYS_UNLOCK()	do {			\
1227 	if (khs != khw) {			\
1228 		PF_HASHROW_UNLOCK(khs);		\
1229 		PF_HASHROW_UNLOCK(khw);		\
1230 	} else					\
1231 		PF_HASHROW_UNLOCK(khs);		\
1232 } while (0)
1233 
1234 	/*
1235 	 * First run: start with wire key.
1236 	 */
1237 	sk = skw;
1238 	kh = khw;
1239 	idx = PF_SK_WIRE;
1240 
1241 	MPASS(s->lock == NULL);
1242 	s->lock = &V_pf_idhash[PF_IDHASH(s)].lock;
1243 
1244 keyattach:
1245 	LIST_FOREACH(cur, &kh->keys, entry)
1246 		if (bcmp(cur, sk, sizeof(struct pf_state_key_cmp)) == 0)
1247 			break;
1248 
1249 	if (cur != NULL) {
1250 		/* Key exists. Check for same kif, if none, add to key. */
1251 		TAILQ_FOREACH(si, &cur->states[idx], key_list[idx]) {
1252 			struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(si)];
1253 
1254 			PF_HASHROW_LOCK(ih);
1255 			if (si->kif == s->kif &&
1256 			    si->direction == s->direction) {
1257 				if (sk->proto == IPPROTO_TCP &&
1258 				    si->src.state >= TCPS_FIN_WAIT_2 &&
1259 				    si->dst.state >= TCPS_FIN_WAIT_2) {
1260 					/*
1261 					 * New state matches an old >FIN_WAIT_2
1262 					 * state. We can't drop key hash locks,
1263 					 * thus we can't unlink it properly.
1264 					 *
1265 					 * As a workaround we drop it into
1266 					 * TCPS_CLOSED state, schedule purge
1267 					 * ASAP and push it into the very end
1268 					 * of the slot TAILQ, so that it won't
1269 					 * conflict with our new state.
1270 					 */
1271 					pf_set_protostate(si, PF_PEER_BOTH,
1272 					    TCPS_CLOSED);
1273 					si->timeout = PFTM_PURGE;
1274 					olds = si;
1275 				} else {
1276 					if (V_pf_status.debug >= PF_DEBUG_MISC) {
1277 						printf("pf: %s key attach "
1278 						    "failed on %s: ",
1279 						    (idx == PF_SK_WIRE) ?
1280 						    "wire" : "stack",
1281 						    s->kif->pfik_name);
1282 						pf_print_state_parts(s,
1283 						    (idx == PF_SK_WIRE) ?
1284 						    sk : NULL,
1285 						    (idx == PF_SK_STACK) ?
1286 						    sk : NULL);
1287 						printf(", existing: ");
1288 						pf_print_state_parts(si,
1289 						    (idx == PF_SK_WIRE) ?
1290 						    sk : NULL,
1291 						    (idx == PF_SK_STACK) ?
1292 						    sk : NULL);
1293 						printf("\n");
1294 					}
1295 					PF_HASHROW_UNLOCK(ih);
1296 					KEYS_UNLOCK();
1297 					uma_zfree(V_pf_state_key_z, sk);
1298 					if (idx == PF_SK_STACK)
1299 						pf_detach_state(s);
1300 					return (EEXIST); /* collision! */
1301 				}
1302 			}
1303 			PF_HASHROW_UNLOCK(ih);
1304 		}
1305 		uma_zfree(V_pf_state_key_z, sk);
1306 		s->key[idx] = cur;
1307 	} else {
1308 		LIST_INSERT_HEAD(&kh->keys, sk, entry);
1309 		s->key[idx] = sk;
1310 	}
1311 
1312 stateattach:
1313 	/* List is sorted, if-bound states before floating. */
1314 	if (s->kif == V_pfi_all)
1315 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], s, key_list[idx]);
1316 	else
1317 		TAILQ_INSERT_HEAD(&s->key[idx]->states[idx], s, key_list[idx]);
1318 
1319 	if (olds) {
1320 		TAILQ_REMOVE(&s->key[idx]->states[idx], olds, key_list[idx]);
1321 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], olds,
1322 		    key_list[idx]);
1323 		olds = NULL;
1324 	}
1325 
1326 	/*
1327 	 * Attach done. See how should we (or should not?)
1328 	 * attach a second key.
1329 	 */
1330 	if (sks == skw) {
1331 		s->key[PF_SK_STACK] = s->key[PF_SK_WIRE];
1332 		idx = PF_SK_STACK;
1333 		sks = NULL;
1334 		goto stateattach;
1335 	} else if (sks != NULL) {
1336 		/*
1337 		 * Continue attaching with stack key.
1338 		 */
1339 		sk = sks;
1340 		kh = khs;
1341 		idx = PF_SK_STACK;
1342 		sks = NULL;
1343 		goto keyattach;
1344 	}
1345 
1346 	PF_STATE_LOCK(s);
1347 	KEYS_UNLOCK();
1348 
1349 	KASSERT(s->key[PF_SK_WIRE] != NULL && s->key[PF_SK_STACK] != NULL,
1350 	    ("%s failure", __func__));
1351 
1352 	return (0);
1353 #undef	KEYS_UNLOCK
1354 }
1355 
1356 static void
1357 pf_detach_state(struct pf_kstate *s)
1358 {
1359 	struct pf_state_key *sks = s->key[PF_SK_STACK];
1360 	struct pf_keyhash *kh;
1361 
1362 	if (sks != NULL) {
1363 		kh = &V_pf_keyhash[pf_hashkey(sks)];
1364 		PF_HASHROW_LOCK(kh);
1365 		if (s->key[PF_SK_STACK] != NULL)
1366 			pf_state_key_detach(s, PF_SK_STACK);
1367 		/*
1368 		 * If both point to same key, then we are done.
1369 		 */
1370 		if (sks == s->key[PF_SK_WIRE]) {
1371 			pf_state_key_detach(s, PF_SK_WIRE);
1372 			PF_HASHROW_UNLOCK(kh);
1373 			return;
1374 		}
1375 		PF_HASHROW_UNLOCK(kh);
1376 	}
1377 
1378 	if (s->key[PF_SK_WIRE] != NULL) {
1379 		kh = &V_pf_keyhash[pf_hashkey(s->key[PF_SK_WIRE])];
1380 		PF_HASHROW_LOCK(kh);
1381 		if (s->key[PF_SK_WIRE] != NULL)
1382 			pf_state_key_detach(s, PF_SK_WIRE);
1383 		PF_HASHROW_UNLOCK(kh);
1384 	}
1385 }
1386 
1387 static void
1388 pf_state_key_detach(struct pf_kstate *s, int idx)
1389 {
1390 	struct pf_state_key *sk = s->key[idx];
1391 #ifdef INVARIANTS
1392 	struct pf_keyhash *kh = &V_pf_keyhash[pf_hashkey(sk)];
1393 
1394 	PF_HASHROW_ASSERT(kh);
1395 #endif
1396 	TAILQ_REMOVE(&sk->states[idx], s, key_list[idx]);
1397 	s->key[idx] = NULL;
1398 
1399 	if (TAILQ_EMPTY(&sk->states[0]) && TAILQ_EMPTY(&sk->states[1])) {
1400 		LIST_REMOVE(sk, entry);
1401 		uma_zfree(V_pf_state_key_z, sk);
1402 	}
1403 }
1404 
1405 static int
1406 pf_state_key_ctor(void *mem, int size, void *arg, int flags)
1407 {
1408 	struct pf_state_key *sk = mem;
1409 
1410 	bzero(sk, sizeof(struct pf_state_key_cmp));
1411 	TAILQ_INIT(&sk->states[PF_SK_WIRE]);
1412 	TAILQ_INIT(&sk->states[PF_SK_STACK]);
1413 
1414 	return (0);
1415 }
1416 
1417 struct pf_state_key *
1418 pf_state_key_setup(struct pf_pdesc *pd, struct pf_addr *saddr,
1419 	struct pf_addr *daddr, u_int16_t sport, u_int16_t dport)
1420 {
1421 	struct pf_state_key *sk;
1422 
1423 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1424 	if (sk == NULL)
1425 		return (NULL);
1426 
1427 	PF_ACPY(&sk->addr[pd->sidx], saddr, pd->af);
1428 	PF_ACPY(&sk->addr[pd->didx], daddr, pd->af);
1429 	sk->port[pd->sidx] = sport;
1430 	sk->port[pd->didx] = dport;
1431 	sk->proto = pd->proto;
1432 	sk->af = pd->af;
1433 
1434 	return (sk);
1435 }
1436 
1437 struct pf_state_key *
1438 pf_state_key_clone(struct pf_state_key *orig)
1439 {
1440 	struct pf_state_key *sk;
1441 
1442 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1443 	if (sk == NULL)
1444 		return (NULL);
1445 
1446 	bcopy(orig, sk, sizeof(struct pf_state_key_cmp));
1447 
1448 	return (sk);
1449 }
1450 
1451 int
1452 pf_state_insert(struct pfi_kkif *kif, struct pfi_kkif *orig_kif,
1453     struct pf_state_key *skw, struct pf_state_key *sks, struct pf_kstate *s)
1454 {
1455 	struct pf_idhash *ih;
1456 	struct pf_kstate *cur;
1457 	int error;
1458 
1459 	KASSERT(TAILQ_EMPTY(&sks->states[0]) && TAILQ_EMPTY(&sks->states[1]),
1460 	    ("%s: sks not pristine", __func__));
1461 	KASSERT(TAILQ_EMPTY(&skw->states[0]) && TAILQ_EMPTY(&skw->states[1]),
1462 	    ("%s: skw not pristine", __func__));
1463 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1464 
1465 	s->kif = kif;
1466 	s->orig_kif = orig_kif;
1467 
1468 	if (s->id == 0 && s->creatorid == 0) {
1469 		s->id = alloc_unr64(&V_pf_stateid);
1470 		s->id = htobe64(s->id);
1471 		s->creatorid = V_pf_status.hostid;
1472 	}
1473 
1474 	/* Returns with ID locked on success. */
1475 	if ((error = pf_state_key_attach(skw, sks, s)) != 0)
1476 		return (error);
1477 
1478 	ih = &V_pf_idhash[PF_IDHASH(s)];
1479 	PF_HASHROW_ASSERT(ih);
1480 	LIST_FOREACH(cur, &ih->states, entry)
1481 		if (cur->id == s->id && cur->creatorid == s->creatorid)
1482 			break;
1483 
1484 	if (cur != NULL) {
1485 		PF_HASHROW_UNLOCK(ih);
1486 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
1487 			printf("pf: state ID collision: "
1488 			    "id: %016llx creatorid: %08x\n",
1489 			    (unsigned long long)be64toh(s->id),
1490 			    ntohl(s->creatorid));
1491 		}
1492 		pf_detach_state(s);
1493 		return (EEXIST);
1494 	}
1495 	LIST_INSERT_HEAD(&ih->states, s, entry);
1496 	/* One for keys, one for ID hash. */
1497 	refcount_init(&s->refs, 2);
1498 
1499 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_INSERT], 1);
1500 	if (V_pfsync_insert_state_ptr != NULL)
1501 		V_pfsync_insert_state_ptr(s);
1502 
1503 	/* Returns locked. */
1504 	return (0);
1505 }
1506 
1507 /*
1508  * Find state by ID: returns with locked row on success.
1509  */
1510 struct pf_kstate *
1511 pf_find_state_byid(uint64_t id, uint32_t creatorid)
1512 {
1513 	struct pf_idhash *ih;
1514 	struct pf_kstate *s;
1515 
1516 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1517 
1518 	ih = &V_pf_idhash[(be64toh(id) % (pf_hashmask + 1))];
1519 
1520 	PF_HASHROW_LOCK(ih);
1521 	LIST_FOREACH(s, &ih->states, entry)
1522 		if (s->id == id && s->creatorid == creatorid)
1523 			break;
1524 
1525 	if (s == NULL)
1526 		PF_HASHROW_UNLOCK(ih);
1527 
1528 	return (s);
1529 }
1530 
1531 /*
1532  * Find state by key.
1533  * Returns with ID hash slot locked on success.
1534  */
1535 static struct pf_kstate *
1536 pf_find_state(struct pfi_kkif *kif, struct pf_state_key_cmp *key, u_int dir)
1537 {
1538 	struct pf_keyhash	*kh;
1539 	struct pf_state_key	*sk;
1540 	struct pf_kstate	*s;
1541 	int idx;
1542 
1543 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1544 
1545 	kh = &V_pf_keyhash[pf_hashkey((struct pf_state_key *)key)];
1546 
1547 	PF_HASHROW_LOCK(kh);
1548 	LIST_FOREACH(sk, &kh->keys, entry)
1549 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1550 			break;
1551 	if (sk == NULL) {
1552 		PF_HASHROW_UNLOCK(kh);
1553 		return (NULL);
1554 	}
1555 
1556 	idx = (dir == PF_IN ? PF_SK_WIRE : PF_SK_STACK);
1557 
1558 	/* List is sorted, if-bound states before floating ones. */
1559 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx])
1560 		if (s->kif == V_pfi_all || s->kif == kif) {
1561 			PF_STATE_LOCK(s);
1562 			PF_HASHROW_UNLOCK(kh);
1563 			if (__predict_false(s->timeout >= PFTM_MAX)) {
1564 				/*
1565 				 * State is either being processed by
1566 				 * pf_unlink_state() in an other thread, or
1567 				 * is scheduled for immediate expiry.
1568 				 */
1569 				PF_STATE_UNLOCK(s);
1570 				return (NULL);
1571 			}
1572 			return (s);
1573 		}
1574 	PF_HASHROW_UNLOCK(kh);
1575 
1576 	return (NULL);
1577 }
1578 
1579 /*
1580  * Returns with ID hash slot locked on success.
1581  */
1582 struct pf_kstate *
1583 pf_find_state_all(struct pf_state_key_cmp *key, u_int dir, int *more)
1584 {
1585 	struct pf_keyhash	*kh;
1586 	struct pf_state_key	*sk;
1587 	struct pf_kstate	*s, *ret = NULL;
1588 	int			 idx, inout = 0;
1589 
1590 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1591 
1592 	kh = &V_pf_keyhash[pf_hashkey((struct pf_state_key *)key)];
1593 
1594 	PF_HASHROW_LOCK(kh);
1595 	LIST_FOREACH(sk, &kh->keys, entry)
1596 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1597 			break;
1598 	if (sk == NULL) {
1599 		PF_HASHROW_UNLOCK(kh);
1600 		return (NULL);
1601 	}
1602 	switch (dir) {
1603 	case PF_IN:
1604 		idx = PF_SK_WIRE;
1605 		break;
1606 	case PF_OUT:
1607 		idx = PF_SK_STACK;
1608 		break;
1609 	case PF_INOUT:
1610 		idx = PF_SK_WIRE;
1611 		inout = 1;
1612 		break;
1613 	default:
1614 		panic("%s: dir %u", __func__, dir);
1615 	}
1616 second_run:
1617 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx]) {
1618 		if (more == NULL) {
1619 			PF_STATE_LOCK(s);
1620 			PF_HASHROW_UNLOCK(kh);
1621 			return (s);
1622 		}
1623 
1624 		if (ret)
1625 			(*more)++;
1626 		else {
1627 			ret = s;
1628 			PF_STATE_LOCK(s);
1629 		}
1630 	}
1631 	if (inout == 1) {
1632 		inout = 0;
1633 		idx = PF_SK_STACK;
1634 		goto second_run;
1635 	}
1636 	PF_HASHROW_UNLOCK(kh);
1637 
1638 	return (ret);
1639 }
1640 
1641 /*
1642  * FIXME
1643  * This routine is inefficient -- locks the state only to unlock immediately on
1644  * return.
1645  * It is racy -- after the state is unlocked nothing stops other threads from
1646  * removing it.
1647  */
1648 bool
1649 pf_find_state_all_exists(struct pf_state_key_cmp *key, u_int dir)
1650 {
1651 	struct pf_kstate *s;
1652 
1653 	s = pf_find_state_all(key, dir, NULL);
1654 	if (s != NULL) {
1655 		PF_STATE_UNLOCK(s);
1656 		return (true);
1657 	}
1658 	return (false);
1659 }
1660 
1661 /* END state table stuff */
1662 
1663 static void
1664 pf_send(struct pf_send_entry *pfse)
1665 {
1666 
1667 	PF_SENDQ_LOCK();
1668 	STAILQ_INSERT_TAIL(&V_pf_sendqueue, pfse, pfse_next);
1669 	PF_SENDQ_UNLOCK();
1670 	swi_sched(V_pf_swi_cookie, 0);
1671 }
1672 
1673 static bool
1674 pf_isforlocal(struct mbuf *m, int af)
1675 {
1676 	switch (af) {
1677 #ifdef INET
1678 	case AF_INET: {
1679 		struct ip *ip = mtod(m, struct ip *);
1680 
1681 		return (in_localip(ip->ip_dst));
1682 	}
1683 #endif
1684 #ifdef INET6
1685 	case AF_INET6: {
1686 		struct ip6_hdr *ip6;
1687 		struct in6_ifaddr *ia;
1688 		ip6 = mtod(m, struct ip6_hdr *);
1689 		ia = in6ifa_ifwithaddr(&ip6->ip6_dst, 0 /* XXX */, false);
1690 		if (ia == NULL)
1691 			return (false);
1692 		return (! (ia->ia6_flags & IN6_IFF_NOTREADY));
1693 	}
1694 #endif
1695 	default:
1696 		panic("Unsupported af %d", af);
1697 	}
1698 
1699 	return (false);
1700 }
1701 
1702 void
1703 pf_intr(void *v)
1704 {
1705 	struct epoch_tracker et;
1706 	struct pf_send_head queue;
1707 	struct pf_send_entry *pfse, *next;
1708 
1709 	CURVNET_SET((struct vnet *)v);
1710 
1711 	PF_SENDQ_LOCK();
1712 	queue = V_pf_sendqueue;
1713 	STAILQ_INIT(&V_pf_sendqueue);
1714 	PF_SENDQ_UNLOCK();
1715 
1716 	NET_EPOCH_ENTER(et);
1717 
1718 	STAILQ_FOREACH_SAFE(pfse, &queue, pfse_next, next) {
1719 		switch (pfse->pfse_type) {
1720 #ifdef INET
1721 		case PFSE_IP: {
1722 			if (pf_isforlocal(pfse->pfse_m, AF_INET)) {
1723 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
1724 				pfse->pfse_m->m_pkthdr.csum_flags |=
1725 				    CSUM_IP_VALID | CSUM_IP_CHECKED;
1726 				ip_input(pfse->pfse_m);
1727 			} else {
1728 				ip_output(pfse->pfse_m, NULL, NULL, 0, NULL,
1729 				    NULL);
1730 			}
1731 			break;
1732 		}
1733 		case PFSE_ICMP:
1734 			icmp_error(pfse->pfse_m, pfse->icmpopts.type,
1735 			    pfse->icmpopts.code, 0, pfse->icmpopts.mtu);
1736 			break;
1737 #endif /* INET */
1738 #ifdef INET6
1739 		case PFSE_IP6:
1740 			if (pf_isforlocal(pfse->pfse_m, AF_INET6)) {
1741 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
1742 				ip6_input(pfse->pfse_m);
1743 			} else {
1744 				ip6_output(pfse->pfse_m, NULL, NULL, 0, NULL,
1745 				    NULL, NULL);
1746 			}
1747 			break;
1748 		case PFSE_ICMP6:
1749 			icmp6_error(pfse->pfse_m, pfse->icmpopts.type,
1750 			    pfse->icmpopts.code, pfse->icmpopts.mtu);
1751 			break;
1752 #endif /* INET6 */
1753 		default:
1754 			panic("%s: unknown type", __func__);
1755 		}
1756 		free(pfse, M_PFTEMP);
1757 	}
1758 	NET_EPOCH_EXIT(et);
1759 	CURVNET_RESTORE();
1760 }
1761 
1762 #define	pf_purge_thread_period	(hz / 10)
1763 
1764 #ifdef PF_WANT_32_TO_64_COUNTER
1765 static void
1766 pf_status_counter_u64_periodic(void)
1767 {
1768 
1769 	PF_RULES_RASSERT();
1770 
1771 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 60)) != 0) {
1772 		return;
1773 	}
1774 
1775 	for (int i = 0; i < FCNT_MAX; i++) {
1776 		pf_counter_u64_periodic(&V_pf_status.fcounters[i]);
1777 	}
1778 }
1779 
1780 static void
1781 pf_kif_counter_u64_periodic(void)
1782 {
1783 	struct pfi_kkif *kif;
1784 	size_t r, run;
1785 
1786 	PF_RULES_RASSERT();
1787 
1788 	if (__predict_false(V_pf_allkifcount == 0)) {
1789 		return;
1790 	}
1791 
1792 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
1793 		return;
1794 	}
1795 
1796 	run = V_pf_allkifcount / 10;
1797 	if (run < 5)
1798 		run = 5;
1799 
1800 	for (r = 0; r < run; r++) {
1801 		kif = LIST_NEXT(V_pf_kifmarker, pfik_allkiflist);
1802 		if (kif == NULL) {
1803 			LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
1804 			LIST_INSERT_HEAD(&V_pf_allkiflist, V_pf_kifmarker, pfik_allkiflist);
1805 			break;
1806 		}
1807 
1808 		LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
1809 		LIST_INSERT_AFTER(kif, V_pf_kifmarker, pfik_allkiflist);
1810 
1811 		for (int i = 0; i < 2; i++) {
1812 			for (int j = 0; j < 2; j++) {
1813 				for (int k = 0; k < 2; k++) {
1814 					pf_counter_u64_periodic(&kif->pfik_packets[i][j][k]);
1815 					pf_counter_u64_periodic(&kif->pfik_bytes[i][j][k]);
1816 				}
1817 			}
1818 		}
1819 	}
1820 }
1821 
1822 static void
1823 pf_rule_counter_u64_periodic(void)
1824 {
1825 	struct pf_krule *rule;
1826 	size_t r, run;
1827 
1828 	PF_RULES_RASSERT();
1829 
1830 	if (__predict_false(V_pf_allrulecount == 0)) {
1831 		return;
1832 	}
1833 
1834 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
1835 		return;
1836 	}
1837 
1838 	run = V_pf_allrulecount / 10;
1839 	if (run < 5)
1840 		run = 5;
1841 
1842 	for (r = 0; r < run; r++) {
1843 		rule = LIST_NEXT(V_pf_rulemarker, allrulelist);
1844 		if (rule == NULL) {
1845 			LIST_REMOVE(V_pf_rulemarker, allrulelist);
1846 			LIST_INSERT_HEAD(&V_pf_allrulelist, V_pf_rulemarker, allrulelist);
1847 			break;
1848 		}
1849 
1850 		LIST_REMOVE(V_pf_rulemarker, allrulelist);
1851 		LIST_INSERT_AFTER(rule, V_pf_rulemarker, allrulelist);
1852 
1853 		pf_counter_u64_periodic(&rule->evaluations);
1854 		for (int i = 0; i < 2; i++) {
1855 			pf_counter_u64_periodic(&rule->packets[i]);
1856 			pf_counter_u64_periodic(&rule->bytes[i]);
1857 		}
1858 	}
1859 }
1860 
1861 static void
1862 pf_counter_u64_periodic_main(void)
1863 {
1864 	PF_RULES_RLOCK_TRACKER;
1865 
1866 	V_pf_counter_periodic_iter++;
1867 
1868 	PF_RULES_RLOCK();
1869 	pf_counter_u64_critical_enter();
1870 	pf_status_counter_u64_periodic();
1871 	pf_kif_counter_u64_periodic();
1872 	pf_rule_counter_u64_periodic();
1873 	pf_counter_u64_critical_exit();
1874 	PF_RULES_RUNLOCK();
1875 }
1876 #else
1877 #define	pf_counter_u64_periodic_main()	do { } while (0)
1878 #endif
1879 
1880 void
1881 pf_purge_thread(void *unused __unused)
1882 {
1883 	VNET_ITERATOR_DECL(vnet_iter);
1884 
1885 	sx_xlock(&pf_end_lock);
1886 	while (pf_end_threads == 0) {
1887 		sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
1888 
1889 		VNET_LIST_RLOCK();
1890 		VNET_FOREACH(vnet_iter) {
1891 			CURVNET_SET(vnet_iter);
1892 
1893 			/* Wait until V_pf_default_rule is initialized. */
1894 			if (V_pf_vnet_active == 0) {
1895 				CURVNET_RESTORE();
1896 				continue;
1897 			}
1898 
1899 			pf_counter_u64_periodic_main();
1900 
1901 			/*
1902 			 *  Process 1/interval fraction of the state
1903 			 * table every run.
1904 			 */
1905 			V_pf_purge_idx =
1906 			    pf_purge_expired_states(V_pf_purge_idx, pf_hashmask /
1907 			    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
1908 
1909 			/*
1910 			 * Purge other expired types every
1911 			 * PFTM_INTERVAL seconds.
1912 			 */
1913 			if (V_pf_purge_idx == 0) {
1914 				/*
1915 				 * Order is important:
1916 				 * - states and src nodes reference rules
1917 				 * - states and rules reference kifs
1918 				 */
1919 				pf_purge_expired_fragments();
1920 				pf_purge_expired_src_nodes();
1921 				pf_purge_unlinked_rules();
1922 				pfi_kkif_purge();
1923 			}
1924 			CURVNET_RESTORE();
1925 		}
1926 		VNET_LIST_RUNLOCK();
1927 	}
1928 
1929 	pf_end_threads++;
1930 	sx_xunlock(&pf_end_lock);
1931 	kproc_exit(0);
1932 }
1933 
1934 void
1935 pf_unload_vnet_purge(void)
1936 {
1937 
1938 	/*
1939 	 * To cleanse up all kifs and rules we need
1940 	 * two runs: first one clears reference flags,
1941 	 * then pf_purge_expired_states() doesn't
1942 	 * raise them, and then second run frees.
1943 	 */
1944 	pf_purge_unlinked_rules();
1945 	pfi_kkif_purge();
1946 
1947 	/*
1948 	 * Now purge everything.
1949 	 */
1950 	pf_purge_expired_states(0, pf_hashmask);
1951 	pf_purge_fragments(UINT_MAX);
1952 	pf_purge_expired_src_nodes();
1953 
1954 	/*
1955 	 * Now all kifs & rules should be unreferenced,
1956 	 * thus should be successfully freed.
1957 	 */
1958 	pf_purge_unlinked_rules();
1959 	pfi_kkif_purge();
1960 }
1961 
1962 u_int32_t
1963 pf_state_expires(const struct pf_kstate *state)
1964 {
1965 	u_int32_t	timeout;
1966 	u_int32_t	start;
1967 	u_int32_t	end;
1968 	u_int32_t	states;
1969 
1970 	/* handle all PFTM_* > PFTM_MAX here */
1971 	if (state->timeout == PFTM_PURGE)
1972 		return (time_uptime);
1973 	KASSERT(state->timeout != PFTM_UNLINKED,
1974 	    ("pf_state_expires: timeout == PFTM_UNLINKED"));
1975 	KASSERT((state->timeout < PFTM_MAX),
1976 	    ("pf_state_expires: timeout > PFTM_MAX"));
1977 	timeout = state->rule.ptr->timeout[state->timeout];
1978 	if (!timeout)
1979 		timeout = V_pf_default_rule.timeout[state->timeout];
1980 	start = state->rule.ptr->timeout[PFTM_ADAPTIVE_START];
1981 	if (start && state->rule.ptr != &V_pf_default_rule) {
1982 		end = state->rule.ptr->timeout[PFTM_ADAPTIVE_END];
1983 		states = counter_u64_fetch(state->rule.ptr->states_cur);
1984 	} else {
1985 		start = V_pf_default_rule.timeout[PFTM_ADAPTIVE_START];
1986 		end = V_pf_default_rule.timeout[PFTM_ADAPTIVE_END];
1987 		states = V_pf_status.states;
1988 	}
1989 	if (end && states > start && start < end) {
1990 		if (states < end) {
1991 			timeout = (u_int64_t)timeout * (end - states) /
1992 			    (end - start);
1993 			return (state->expire + timeout);
1994 		}
1995 		else
1996 			return (time_uptime);
1997 	}
1998 	return (state->expire + timeout);
1999 }
2000 
2001 void
2002 pf_purge_expired_src_nodes(void)
2003 {
2004 	struct pf_ksrc_node_list	 freelist;
2005 	struct pf_srchash	*sh;
2006 	struct pf_ksrc_node	*cur, *next;
2007 	int i;
2008 
2009 	LIST_INIT(&freelist);
2010 	for (i = 0, sh = V_pf_srchash; i <= pf_srchashmask; i++, sh++) {
2011 	    PF_HASHROW_LOCK(sh);
2012 	    LIST_FOREACH_SAFE(cur, &sh->nodes, entry, next)
2013 		if (cur->states == 0 && cur->expire <= time_uptime) {
2014 			pf_unlink_src_node(cur);
2015 			LIST_INSERT_HEAD(&freelist, cur, entry);
2016 		} else if (cur->rule.ptr != NULL)
2017 			cur->rule.ptr->rule_ref |= PFRULE_REFS;
2018 	    PF_HASHROW_UNLOCK(sh);
2019 	}
2020 
2021 	pf_free_src_nodes(&freelist);
2022 
2023 	V_pf_status.src_nodes = uma_zone_get_cur(V_pf_sources_z);
2024 }
2025 
2026 static void
2027 pf_src_tree_remove_state(struct pf_kstate *s)
2028 {
2029 	struct pf_ksrc_node *sn;
2030 	uint32_t timeout;
2031 
2032 	timeout = s->rule.ptr->timeout[PFTM_SRC_NODE] ?
2033 	    s->rule.ptr->timeout[PFTM_SRC_NODE] :
2034 	    V_pf_default_rule.timeout[PFTM_SRC_NODE];
2035 
2036 	if (s->src_node != NULL) {
2037 		sn = s->src_node;
2038 		PF_SRC_NODE_LOCK(sn);
2039 		if (s->src.tcp_est)
2040 			--sn->conn;
2041 		if (--sn->states == 0)
2042 			sn->expire = time_uptime + timeout;
2043 		PF_SRC_NODE_UNLOCK(sn);
2044 	}
2045 	if (s->nat_src_node != s->src_node && s->nat_src_node != NULL) {
2046 		sn = s->nat_src_node;
2047 		PF_SRC_NODE_LOCK(sn);
2048 		if (--sn->states == 0)
2049 			sn->expire = time_uptime + timeout;
2050 		PF_SRC_NODE_UNLOCK(sn);
2051 	}
2052 	s->src_node = s->nat_src_node = NULL;
2053 }
2054 
2055 /*
2056  * Unlink and potentilly free a state. Function may be
2057  * called with ID hash row locked, but always returns
2058  * unlocked, since it needs to go through key hash locking.
2059  */
2060 int
2061 pf_unlink_state(struct pf_kstate *s)
2062 {
2063 	struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)];
2064 
2065 	PF_HASHROW_ASSERT(ih);
2066 
2067 	if (s->timeout == PFTM_UNLINKED) {
2068 		/*
2069 		 * State is being processed
2070 		 * by pf_unlink_state() in
2071 		 * an other thread.
2072 		 */
2073 		PF_HASHROW_UNLOCK(ih);
2074 		return (0);	/* XXXGL: undefined actually */
2075 	}
2076 
2077 	if (s->src.state == PF_TCPS_PROXY_DST) {
2078 		/* XXX wire key the right one? */
2079 		pf_send_tcp(s->rule.ptr, s->key[PF_SK_WIRE]->af,
2080 		    &s->key[PF_SK_WIRE]->addr[1],
2081 		    &s->key[PF_SK_WIRE]->addr[0],
2082 		    s->key[PF_SK_WIRE]->port[1],
2083 		    s->key[PF_SK_WIRE]->port[0],
2084 		    s->src.seqhi, s->src.seqlo + 1,
2085 		    TH_RST|TH_ACK, 0, 0, 0, true, s->tag, 0, s->act.rtableid);
2086 	}
2087 
2088 	LIST_REMOVE(s, entry);
2089 	pf_src_tree_remove_state(s);
2090 
2091 	if (V_pfsync_delete_state_ptr != NULL)
2092 		V_pfsync_delete_state_ptr(s);
2093 
2094 	STATE_DEC_COUNTERS(s);
2095 
2096 	s->timeout = PFTM_UNLINKED;
2097 
2098 	/* Ensure we remove it from the list of halfopen states, if needed. */
2099 	if (s->key[PF_SK_STACK] != NULL &&
2100 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP)
2101 		pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
2102 
2103 	PF_HASHROW_UNLOCK(ih);
2104 
2105 	pf_detach_state(s);
2106 	/* pf_state_insert() initialises refs to 2 */
2107 	return (pf_release_staten(s, 2));
2108 }
2109 
2110 struct pf_kstate *
2111 pf_alloc_state(int flags)
2112 {
2113 
2114 	return (uma_zalloc(V_pf_state_z, flags | M_ZERO));
2115 }
2116 
2117 void
2118 pf_free_state(struct pf_kstate *cur)
2119 {
2120 	struct pf_krule_item *ri;
2121 
2122 	KASSERT(cur->refs == 0, ("%s: %p has refs", __func__, cur));
2123 	KASSERT(cur->timeout == PFTM_UNLINKED, ("%s: timeout %u", __func__,
2124 	    cur->timeout));
2125 
2126 	while ((ri = SLIST_FIRST(&cur->match_rules))) {
2127 		SLIST_REMOVE_HEAD(&cur->match_rules, entry);
2128 		free(ri, M_PF_RULE_ITEM);
2129 	}
2130 
2131 	pf_normalize_tcp_cleanup(cur);
2132 	uma_zfree(V_pf_state_z, cur);
2133 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_REMOVALS], 1);
2134 }
2135 
2136 /*
2137  * Called only from pf_purge_thread(), thus serialized.
2138  */
2139 static u_int
2140 pf_purge_expired_states(u_int i, int maxcheck)
2141 {
2142 	struct pf_idhash *ih;
2143 	struct pf_kstate *s;
2144 	struct pf_krule_item *mrm;
2145 
2146 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2147 
2148 	/*
2149 	 * Go through hash and unlink states that expire now.
2150 	 */
2151 	while (maxcheck > 0) {
2152 		ih = &V_pf_idhash[i];
2153 
2154 		/* only take the lock if we expect to do work */
2155 		if (!LIST_EMPTY(&ih->states)) {
2156 relock:
2157 			PF_HASHROW_LOCK(ih);
2158 			LIST_FOREACH(s, &ih->states, entry) {
2159 				if (pf_state_expires(s) <= time_uptime) {
2160 					V_pf_status.states -=
2161 					    pf_unlink_state(s);
2162 					goto relock;
2163 				}
2164 				s->rule.ptr->rule_ref |= PFRULE_REFS;
2165 				if (s->nat_rule.ptr != NULL)
2166 					s->nat_rule.ptr->rule_ref |= PFRULE_REFS;
2167 				if (s->anchor.ptr != NULL)
2168 					s->anchor.ptr->rule_ref |= PFRULE_REFS;
2169 				s->kif->pfik_flags |= PFI_IFLAG_REFS;
2170 				SLIST_FOREACH(mrm, &s->match_rules, entry)
2171 					mrm->r->rule_ref |= PFRULE_REFS;
2172 				if (s->rt_kif)
2173 					s->rt_kif->pfik_flags |= PFI_IFLAG_REFS;
2174 			}
2175 			PF_HASHROW_UNLOCK(ih);
2176 		}
2177 
2178 		/* Return when we hit end of hash. */
2179 		if (++i > pf_hashmask) {
2180 			V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2181 			return (0);
2182 		}
2183 
2184 		maxcheck--;
2185 	}
2186 
2187 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2188 
2189 	return (i);
2190 }
2191 
2192 static void
2193 pf_purge_unlinked_rules(void)
2194 {
2195 	struct pf_krulequeue tmpq;
2196 	struct pf_krule *r, *r1;
2197 
2198 	/*
2199 	 * If we have overloading task pending, then we'd
2200 	 * better skip purging this time. There is a tiny
2201 	 * probability that overloading task references
2202 	 * an already unlinked rule.
2203 	 */
2204 	PF_OVERLOADQ_LOCK();
2205 	if (!SLIST_EMPTY(&V_pf_overloadqueue)) {
2206 		PF_OVERLOADQ_UNLOCK();
2207 		return;
2208 	}
2209 	PF_OVERLOADQ_UNLOCK();
2210 
2211 	/*
2212 	 * Do naive mark-and-sweep garbage collecting of old rules.
2213 	 * Reference flag is raised by pf_purge_expired_states()
2214 	 * and pf_purge_expired_src_nodes().
2215 	 *
2216 	 * To avoid LOR between PF_UNLNKDRULES_LOCK/PF_RULES_WLOCK,
2217 	 * use a temporary queue.
2218 	 */
2219 	TAILQ_INIT(&tmpq);
2220 	PF_UNLNKDRULES_LOCK();
2221 	TAILQ_FOREACH_SAFE(r, &V_pf_unlinked_rules, entries, r1) {
2222 		if (!(r->rule_ref & PFRULE_REFS)) {
2223 			TAILQ_REMOVE(&V_pf_unlinked_rules, r, entries);
2224 			TAILQ_INSERT_TAIL(&tmpq, r, entries);
2225 		} else
2226 			r->rule_ref &= ~PFRULE_REFS;
2227 	}
2228 	PF_UNLNKDRULES_UNLOCK();
2229 
2230 	if (!TAILQ_EMPTY(&tmpq)) {
2231 		PF_CONFIG_LOCK();
2232 		PF_RULES_WLOCK();
2233 		TAILQ_FOREACH_SAFE(r, &tmpq, entries, r1) {
2234 			TAILQ_REMOVE(&tmpq, r, entries);
2235 			pf_free_rule(r);
2236 		}
2237 		PF_RULES_WUNLOCK();
2238 		PF_CONFIG_UNLOCK();
2239 	}
2240 }
2241 
2242 void
2243 pf_print_host(struct pf_addr *addr, u_int16_t p, sa_family_t af)
2244 {
2245 	switch (af) {
2246 #ifdef INET
2247 	case AF_INET: {
2248 		u_int32_t a = ntohl(addr->addr32[0]);
2249 		printf("%u.%u.%u.%u", (a>>24)&255, (a>>16)&255,
2250 		    (a>>8)&255, a&255);
2251 		if (p) {
2252 			p = ntohs(p);
2253 			printf(":%u", p);
2254 		}
2255 		break;
2256 	}
2257 #endif /* INET */
2258 #ifdef INET6
2259 	case AF_INET6: {
2260 		u_int16_t b;
2261 		u_int8_t i, curstart, curend, maxstart, maxend;
2262 		curstart = curend = maxstart = maxend = 255;
2263 		for (i = 0; i < 8; i++) {
2264 			if (!addr->addr16[i]) {
2265 				if (curstart == 255)
2266 					curstart = i;
2267 				curend = i;
2268 			} else {
2269 				if ((curend - curstart) >
2270 				    (maxend - maxstart)) {
2271 					maxstart = curstart;
2272 					maxend = curend;
2273 				}
2274 				curstart = curend = 255;
2275 			}
2276 		}
2277 		if ((curend - curstart) >
2278 		    (maxend - maxstart)) {
2279 			maxstart = curstart;
2280 			maxend = curend;
2281 		}
2282 		for (i = 0; i < 8; i++) {
2283 			if (i >= maxstart && i <= maxend) {
2284 				if (i == 0)
2285 					printf(":");
2286 				if (i == maxend)
2287 					printf(":");
2288 			} else {
2289 				b = ntohs(addr->addr16[i]);
2290 				printf("%x", b);
2291 				if (i < 7)
2292 					printf(":");
2293 			}
2294 		}
2295 		if (p) {
2296 			p = ntohs(p);
2297 			printf("[%u]", p);
2298 		}
2299 		break;
2300 	}
2301 #endif /* INET6 */
2302 	}
2303 }
2304 
2305 void
2306 pf_print_state(struct pf_kstate *s)
2307 {
2308 	pf_print_state_parts(s, NULL, NULL);
2309 }
2310 
2311 static void
2312 pf_print_state_parts(struct pf_kstate *s,
2313     struct pf_state_key *skwp, struct pf_state_key *sksp)
2314 {
2315 	struct pf_state_key *skw, *sks;
2316 	u_int8_t proto, dir;
2317 
2318 	/* Do our best to fill these, but they're skipped if NULL */
2319 	skw = skwp ? skwp : (s ? s->key[PF_SK_WIRE] : NULL);
2320 	sks = sksp ? sksp : (s ? s->key[PF_SK_STACK] : NULL);
2321 	proto = skw ? skw->proto : (sks ? sks->proto : 0);
2322 	dir = s ? s->direction : 0;
2323 
2324 	switch (proto) {
2325 	case IPPROTO_IPV4:
2326 		printf("IPv4");
2327 		break;
2328 	case IPPROTO_IPV6:
2329 		printf("IPv6");
2330 		break;
2331 	case IPPROTO_TCP:
2332 		printf("TCP");
2333 		break;
2334 	case IPPROTO_UDP:
2335 		printf("UDP");
2336 		break;
2337 	case IPPROTO_ICMP:
2338 		printf("ICMP");
2339 		break;
2340 	case IPPROTO_ICMPV6:
2341 		printf("ICMPv6");
2342 		break;
2343 	default:
2344 		printf("%u", proto);
2345 		break;
2346 	}
2347 	switch (dir) {
2348 	case PF_IN:
2349 		printf(" in");
2350 		break;
2351 	case PF_OUT:
2352 		printf(" out");
2353 		break;
2354 	}
2355 	if (skw) {
2356 		printf(" wire: ");
2357 		pf_print_host(&skw->addr[0], skw->port[0], skw->af);
2358 		printf(" ");
2359 		pf_print_host(&skw->addr[1], skw->port[1], skw->af);
2360 	}
2361 	if (sks) {
2362 		printf(" stack: ");
2363 		if (sks != skw) {
2364 			pf_print_host(&sks->addr[0], sks->port[0], sks->af);
2365 			printf(" ");
2366 			pf_print_host(&sks->addr[1], sks->port[1], sks->af);
2367 		} else
2368 			printf("-");
2369 	}
2370 	if (s) {
2371 		if (proto == IPPROTO_TCP) {
2372 			printf(" [lo=%u high=%u win=%u modulator=%u",
2373 			    s->src.seqlo, s->src.seqhi,
2374 			    s->src.max_win, s->src.seqdiff);
2375 			if (s->src.wscale && s->dst.wscale)
2376 				printf(" wscale=%u",
2377 				    s->src.wscale & PF_WSCALE_MASK);
2378 			printf("]");
2379 			printf(" [lo=%u high=%u win=%u modulator=%u",
2380 			    s->dst.seqlo, s->dst.seqhi,
2381 			    s->dst.max_win, s->dst.seqdiff);
2382 			if (s->src.wscale && s->dst.wscale)
2383 				printf(" wscale=%u",
2384 				s->dst.wscale & PF_WSCALE_MASK);
2385 			printf("]");
2386 		}
2387 		printf(" %u:%u", s->src.state, s->dst.state);
2388 	}
2389 }
2390 
2391 void
2392 pf_print_flags(u_int8_t f)
2393 {
2394 	if (f)
2395 		printf(" ");
2396 	if (f & TH_FIN)
2397 		printf("F");
2398 	if (f & TH_SYN)
2399 		printf("S");
2400 	if (f & TH_RST)
2401 		printf("R");
2402 	if (f & TH_PUSH)
2403 		printf("P");
2404 	if (f & TH_ACK)
2405 		printf("A");
2406 	if (f & TH_URG)
2407 		printf("U");
2408 	if (f & TH_ECE)
2409 		printf("E");
2410 	if (f & TH_CWR)
2411 		printf("W");
2412 }
2413 
2414 #define	PF_SET_SKIP_STEPS(i)					\
2415 	do {							\
2416 		while (head[i] != cur) {			\
2417 			head[i]->skip[i].ptr = cur;		\
2418 			head[i] = TAILQ_NEXT(head[i], entries);	\
2419 		}						\
2420 	} while (0)
2421 
2422 void
2423 pf_calc_skip_steps(struct pf_krulequeue *rules)
2424 {
2425 	struct pf_krule *cur, *prev, *head[PF_SKIP_COUNT];
2426 	int i;
2427 
2428 	cur = TAILQ_FIRST(rules);
2429 	prev = cur;
2430 	for (i = 0; i < PF_SKIP_COUNT; ++i)
2431 		head[i] = cur;
2432 	while (cur != NULL) {
2433 		if (cur->kif != prev->kif || cur->ifnot != prev->ifnot)
2434 			PF_SET_SKIP_STEPS(PF_SKIP_IFP);
2435 		if (cur->direction != prev->direction)
2436 			PF_SET_SKIP_STEPS(PF_SKIP_DIR);
2437 		if (cur->af != prev->af)
2438 			PF_SET_SKIP_STEPS(PF_SKIP_AF);
2439 		if (cur->proto != prev->proto)
2440 			PF_SET_SKIP_STEPS(PF_SKIP_PROTO);
2441 		if (cur->src.neg != prev->src.neg ||
2442 		    pf_addr_wrap_neq(&cur->src.addr, &prev->src.addr))
2443 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_ADDR);
2444 		if (cur->src.port[0] != prev->src.port[0] ||
2445 		    cur->src.port[1] != prev->src.port[1] ||
2446 		    cur->src.port_op != prev->src.port_op)
2447 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_PORT);
2448 		if (cur->dst.neg != prev->dst.neg ||
2449 		    pf_addr_wrap_neq(&cur->dst.addr, &prev->dst.addr))
2450 			PF_SET_SKIP_STEPS(PF_SKIP_DST_ADDR);
2451 		if (cur->dst.port[0] != prev->dst.port[0] ||
2452 		    cur->dst.port[1] != prev->dst.port[1] ||
2453 		    cur->dst.port_op != prev->dst.port_op)
2454 			PF_SET_SKIP_STEPS(PF_SKIP_DST_PORT);
2455 
2456 		prev = cur;
2457 		cur = TAILQ_NEXT(cur, entries);
2458 	}
2459 	for (i = 0; i < PF_SKIP_COUNT; ++i)
2460 		PF_SET_SKIP_STEPS(i);
2461 }
2462 
2463 int
2464 pf_addr_wrap_neq(struct pf_addr_wrap *aw1, struct pf_addr_wrap *aw2)
2465 {
2466 	if (aw1->type != aw2->type)
2467 		return (1);
2468 	switch (aw1->type) {
2469 	case PF_ADDR_ADDRMASK:
2470 	case PF_ADDR_RANGE:
2471 		if (PF_ANEQ(&aw1->v.a.addr, &aw2->v.a.addr, AF_INET6))
2472 			return (1);
2473 		if (PF_ANEQ(&aw1->v.a.mask, &aw2->v.a.mask, AF_INET6))
2474 			return (1);
2475 		return (0);
2476 	case PF_ADDR_DYNIFTL:
2477 		return (aw1->p.dyn->pfid_kt != aw2->p.dyn->pfid_kt);
2478 	case PF_ADDR_NOROUTE:
2479 	case PF_ADDR_URPFFAILED:
2480 		return (0);
2481 	case PF_ADDR_TABLE:
2482 		return (aw1->p.tbl != aw2->p.tbl);
2483 	default:
2484 		printf("invalid address type: %d\n", aw1->type);
2485 		return (1);
2486 	}
2487 }
2488 
2489 /**
2490  * Checksum updates are a little complicated because the checksum in the TCP/UDP
2491  * header isn't always a full checksum. In some cases (i.e. output) it's a
2492  * pseudo-header checksum, which is a partial checksum over src/dst IP
2493  * addresses, protocol number and length.
2494  *
2495  * That means we have the following cases:
2496  *  * Input or forwarding: we don't have TSO, the checksum fields are full
2497  *  	checksums, we need to update the checksum whenever we change anything.
2498  *  * Output (i.e. the checksum is a pseudo-header checksum):
2499  *  	x The field being updated is src/dst address or affects the length of
2500  *  	the packet. We need to update the pseudo-header checksum (note that this
2501  *  	checksum is not ones' complement).
2502  *  	x Some other field is being modified (e.g. src/dst port numbers): We
2503  *  	don't have to update anything.
2504  **/
2505 u_int16_t
2506 pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
2507 {
2508 	u_int32_t x;
2509 
2510 	x = cksum + old - new;
2511 	x = (x + (x >> 16)) & 0xffff;
2512 
2513 	/* optimise: eliminate a branch when not udp */
2514 	if (udp && cksum == 0x0000)
2515 		return cksum;
2516 	if (udp && x == 0x0000)
2517 		x = 0xffff;
2518 
2519 	return (u_int16_t)(x);
2520 }
2521 
2522 static void
2523 pf_patch_8(struct mbuf *m, u_int16_t *cksum, u_int8_t *f, u_int8_t v, bool hi,
2524     u_int8_t udp)
2525 {
2526 	u_int16_t old = htons(hi ? (*f << 8) : *f);
2527 	u_int16_t new = htons(hi ? ( v << 8) :  v);
2528 
2529 	if (*f == v)
2530 		return;
2531 
2532 	*f = v;
2533 
2534 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
2535 		return;
2536 
2537 	*cksum = pf_cksum_fixup(*cksum, old, new, udp);
2538 }
2539 
2540 void
2541 pf_patch_16_unaligned(struct mbuf *m, u_int16_t *cksum, void *f, u_int16_t v,
2542     bool hi, u_int8_t udp)
2543 {
2544 	u_int8_t *fb = (u_int8_t *)f;
2545 	u_int8_t *vb = (u_int8_t *)&v;
2546 
2547 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
2548 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
2549 }
2550 
2551 void
2552 pf_patch_32_unaligned(struct mbuf *m, u_int16_t *cksum, void *f, u_int32_t v,
2553     bool hi, u_int8_t udp)
2554 {
2555 	u_int8_t *fb = (u_int8_t *)f;
2556 	u_int8_t *vb = (u_int8_t *)&v;
2557 
2558 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
2559 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
2560 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
2561 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
2562 }
2563 
2564 u_int16_t
2565 pf_proto_cksum_fixup(struct mbuf *m, u_int16_t cksum, u_int16_t old,
2566         u_int16_t new, u_int8_t udp)
2567 {
2568 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
2569 		return (cksum);
2570 
2571 	return (pf_cksum_fixup(cksum, old, new, udp));
2572 }
2573 
2574 static void
2575 pf_change_ap(struct mbuf *m, struct pf_addr *a, u_int16_t *p, u_int16_t *ic,
2576         u_int16_t *pc, struct pf_addr *an, u_int16_t pn, u_int8_t u,
2577         sa_family_t af)
2578 {
2579 	struct pf_addr	ao;
2580 	u_int16_t	po = *p;
2581 
2582 	PF_ACPY(&ao, a, af);
2583 	PF_ACPY(a, an, af);
2584 
2585 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
2586 		*pc = ~*pc;
2587 
2588 	*p = pn;
2589 
2590 	switch (af) {
2591 #ifdef INET
2592 	case AF_INET:
2593 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
2594 		    ao.addr16[0], an->addr16[0], 0),
2595 		    ao.addr16[1], an->addr16[1], 0);
2596 		*p = pn;
2597 
2598 		*pc = pf_cksum_fixup(pf_cksum_fixup(*pc,
2599 		    ao.addr16[0], an->addr16[0], u),
2600 		    ao.addr16[1], an->addr16[1], u);
2601 
2602 		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
2603 		break;
2604 #endif /* INET */
2605 #ifdef INET6
2606 	case AF_INET6:
2607 		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2608 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2609 		    pf_cksum_fixup(pf_cksum_fixup(*pc,
2610 		    ao.addr16[0], an->addr16[0], u),
2611 		    ao.addr16[1], an->addr16[1], u),
2612 		    ao.addr16[2], an->addr16[2], u),
2613 		    ao.addr16[3], an->addr16[3], u),
2614 		    ao.addr16[4], an->addr16[4], u),
2615 		    ao.addr16[5], an->addr16[5], u),
2616 		    ao.addr16[6], an->addr16[6], u),
2617 		    ao.addr16[7], an->addr16[7], u);
2618 
2619 		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
2620 		break;
2621 #endif /* INET6 */
2622 	}
2623 
2624 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA |
2625 	    CSUM_DELAY_DATA_IPV6)) {
2626 		*pc = ~*pc;
2627 		if (! *pc)
2628 			*pc = 0xffff;
2629 	}
2630 }
2631 
2632 /* Changes a u_int32_t.  Uses a void * so there are no align restrictions */
2633 void
2634 pf_change_a(void *a, u_int16_t *c, u_int32_t an, u_int8_t u)
2635 {
2636 	u_int32_t	ao;
2637 
2638 	memcpy(&ao, a, sizeof(ao));
2639 	memcpy(a, &an, sizeof(u_int32_t));
2640 	*c = pf_cksum_fixup(pf_cksum_fixup(*c, ao / 65536, an / 65536, u),
2641 	    ao % 65536, an % 65536, u);
2642 }
2643 
2644 void
2645 pf_change_proto_a(struct mbuf *m, void *a, u_int16_t *c, u_int32_t an, u_int8_t udp)
2646 {
2647 	u_int32_t	ao;
2648 
2649 	memcpy(&ao, a, sizeof(ao));
2650 	memcpy(a, &an, sizeof(u_int32_t));
2651 
2652 	*c = pf_proto_cksum_fixup(m,
2653 	    pf_proto_cksum_fixup(m, *c, ao / 65536, an / 65536, udp),
2654 	    ao % 65536, an % 65536, udp);
2655 }
2656 
2657 #ifdef INET6
2658 static void
2659 pf_change_a6(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u)
2660 {
2661 	struct pf_addr	ao;
2662 
2663 	PF_ACPY(&ao, a, AF_INET6);
2664 	PF_ACPY(a, an, AF_INET6);
2665 
2666 	*c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2667 	    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2668 	    pf_cksum_fixup(pf_cksum_fixup(*c,
2669 	    ao.addr16[0], an->addr16[0], u),
2670 	    ao.addr16[1], an->addr16[1], u),
2671 	    ao.addr16[2], an->addr16[2], u),
2672 	    ao.addr16[3], an->addr16[3], u),
2673 	    ao.addr16[4], an->addr16[4], u),
2674 	    ao.addr16[5], an->addr16[5], u),
2675 	    ao.addr16[6], an->addr16[6], u),
2676 	    ao.addr16[7], an->addr16[7], u);
2677 }
2678 #endif /* INET6 */
2679 
2680 static void
2681 pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa,
2682     struct pf_addr *na, u_int16_t np, u_int16_t *pc, u_int16_t *h2c,
2683     u_int16_t *ic, u_int16_t *hc, u_int8_t u, sa_family_t af)
2684 {
2685 	struct pf_addr	oia, ooa;
2686 
2687 	PF_ACPY(&oia, ia, af);
2688 	if (oa)
2689 		PF_ACPY(&ooa, oa, af);
2690 
2691 	/* Change inner protocol port, fix inner protocol checksum. */
2692 	if (ip != NULL) {
2693 		u_int16_t	oip = *ip;
2694 		u_int32_t	opc;
2695 
2696 		if (pc != NULL)
2697 			opc = *pc;
2698 		*ip = np;
2699 		if (pc != NULL)
2700 			*pc = pf_cksum_fixup(*pc, oip, *ip, u);
2701 		*ic = pf_cksum_fixup(*ic, oip, *ip, 0);
2702 		if (pc != NULL)
2703 			*ic = pf_cksum_fixup(*ic, opc, *pc, 0);
2704 	}
2705 	/* Change inner ip address, fix inner ip and icmp checksums. */
2706 	PF_ACPY(ia, na, af);
2707 	switch (af) {
2708 #ifdef INET
2709 	case AF_INET: {
2710 		u_int32_t	 oh2c = *h2c;
2711 
2712 		*h2c = pf_cksum_fixup(pf_cksum_fixup(*h2c,
2713 		    oia.addr16[0], ia->addr16[0], 0),
2714 		    oia.addr16[1], ia->addr16[1], 0);
2715 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
2716 		    oia.addr16[0], ia->addr16[0], 0),
2717 		    oia.addr16[1], ia->addr16[1], 0);
2718 		*ic = pf_cksum_fixup(*ic, oh2c, *h2c, 0);
2719 		break;
2720 	}
2721 #endif /* INET */
2722 #ifdef INET6
2723 	case AF_INET6:
2724 		*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2725 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2726 		    pf_cksum_fixup(pf_cksum_fixup(*ic,
2727 		    oia.addr16[0], ia->addr16[0], u),
2728 		    oia.addr16[1], ia->addr16[1], u),
2729 		    oia.addr16[2], ia->addr16[2], u),
2730 		    oia.addr16[3], ia->addr16[3], u),
2731 		    oia.addr16[4], ia->addr16[4], u),
2732 		    oia.addr16[5], ia->addr16[5], u),
2733 		    oia.addr16[6], ia->addr16[6], u),
2734 		    oia.addr16[7], ia->addr16[7], u);
2735 		break;
2736 #endif /* INET6 */
2737 	}
2738 	/* Outer ip address, fix outer ip or icmpv6 checksum, if necessary. */
2739 	if (oa) {
2740 		PF_ACPY(oa, na, af);
2741 		switch (af) {
2742 #ifdef INET
2743 		case AF_INET:
2744 			*hc = pf_cksum_fixup(pf_cksum_fixup(*hc,
2745 			    ooa.addr16[0], oa->addr16[0], 0),
2746 			    ooa.addr16[1], oa->addr16[1], 0);
2747 			break;
2748 #endif /* INET */
2749 #ifdef INET6
2750 		case AF_INET6:
2751 			*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2752 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2753 			    pf_cksum_fixup(pf_cksum_fixup(*ic,
2754 			    ooa.addr16[0], oa->addr16[0], u),
2755 			    ooa.addr16[1], oa->addr16[1], u),
2756 			    ooa.addr16[2], oa->addr16[2], u),
2757 			    ooa.addr16[3], oa->addr16[3], u),
2758 			    ooa.addr16[4], oa->addr16[4], u),
2759 			    ooa.addr16[5], oa->addr16[5], u),
2760 			    ooa.addr16[6], oa->addr16[6], u),
2761 			    ooa.addr16[7], oa->addr16[7], u);
2762 			break;
2763 #endif /* INET6 */
2764 		}
2765 	}
2766 }
2767 
2768 /*
2769  * Need to modulate the sequence numbers in the TCP SACK option
2770  * (credits to Krzysztof Pfaff for report and patch)
2771  */
2772 static int
2773 pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd,
2774     struct tcphdr *th, struct pf_state_peer *dst)
2775 {
2776 	int hlen = (th->th_off << 2) - sizeof(*th), thoptlen = hlen;
2777 	u_int8_t opts[TCP_MAXOLEN], *opt = opts;
2778 	int copyback = 0, i, olen;
2779 	struct sackblk sack;
2780 
2781 #define	TCPOLEN_SACKLEN	(TCPOLEN_SACK + 2)
2782 	if (hlen < TCPOLEN_SACKLEN ||
2783 	    !pf_pull_hdr(m, off + sizeof(*th), opts, hlen, NULL, NULL, pd->af))
2784 		return 0;
2785 
2786 	while (hlen >= TCPOLEN_SACKLEN) {
2787 		size_t startoff = opt - opts;
2788 		olen = opt[1];
2789 		switch (*opt) {
2790 		case TCPOPT_EOL:	/* FALLTHROUGH */
2791 		case TCPOPT_NOP:
2792 			opt++;
2793 			hlen--;
2794 			break;
2795 		case TCPOPT_SACK:
2796 			if (olen > hlen)
2797 				olen = hlen;
2798 			if (olen >= TCPOLEN_SACKLEN) {
2799 				for (i = 2; i + TCPOLEN_SACK <= olen;
2800 				    i += TCPOLEN_SACK) {
2801 					memcpy(&sack, &opt[i], sizeof(sack));
2802 					pf_patch_32_unaligned(m,
2803 					    &th->th_sum, &sack.start,
2804 					    htonl(ntohl(sack.start) - dst->seqdiff),
2805 					    PF_ALGNMNT(startoff),
2806 					    0);
2807 					pf_patch_32_unaligned(m, &th->th_sum,
2808 					    &sack.end,
2809 					    htonl(ntohl(sack.end) - dst->seqdiff),
2810 					    PF_ALGNMNT(startoff),
2811 					    0);
2812 					memcpy(&opt[i], &sack, sizeof(sack));
2813 				}
2814 				copyback = 1;
2815 			}
2816 			/* FALLTHROUGH */
2817 		default:
2818 			if (olen < 2)
2819 				olen = 2;
2820 			hlen -= olen;
2821 			opt += olen;
2822 		}
2823 	}
2824 
2825 	if (copyback)
2826 		m_copyback(m, off + sizeof(*th), thoptlen, (caddr_t)opts);
2827 	return (copyback);
2828 }
2829 
2830 struct mbuf *
2831 pf_build_tcp(const struct pf_krule *r, sa_family_t af,
2832     const struct pf_addr *saddr, const struct pf_addr *daddr,
2833     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
2834     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
2835     bool skip_firewall, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
2836 {
2837 	struct mbuf	*m;
2838 	int		 len, tlen;
2839 #ifdef INET
2840 	struct ip	*h = NULL;
2841 #endif /* INET */
2842 #ifdef INET6
2843 	struct ip6_hdr	*h6 = NULL;
2844 #endif /* INET6 */
2845 	struct tcphdr	*th;
2846 	char		*opt;
2847 	struct pf_mtag  *pf_mtag;
2848 
2849 	len = 0;
2850 	th = NULL;
2851 
2852 	/* maximum segment size tcp option */
2853 	tlen = sizeof(struct tcphdr);
2854 	if (mss)
2855 		tlen += 4;
2856 
2857 	switch (af) {
2858 #ifdef INET
2859 	case AF_INET:
2860 		len = sizeof(struct ip) + tlen;
2861 		break;
2862 #endif /* INET */
2863 #ifdef INET6
2864 	case AF_INET6:
2865 		len = sizeof(struct ip6_hdr) + tlen;
2866 		break;
2867 #endif /* INET6 */
2868 	default:
2869 		panic("%s: unsupported af %d", __func__, af);
2870 	}
2871 
2872 	m = m_gethdr(M_NOWAIT, MT_DATA);
2873 	if (m == NULL)
2874 		return (NULL);
2875 
2876 #ifdef MAC
2877 	mac_netinet_firewall_send(m);
2878 #endif
2879 	if ((pf_mtag = pf_get_mtag(m)) == NULL) {
2880 		m_freem(m);
2881 		return (NULL);
2882 	}
2883 	if (skip_firewall)
2884 		m->m_flags |= M_SKIP_FIREWALL;
2885 	pf_mtag->tag = mtag_tag;
2886 	pf_mtag->flags = mtag_flags;
2887 
2888 	if (rtableid >= 0)
2889 		M_SETFIB(m, rtableid);
2890 
2891 #ifdef ALTQ
2892 	if (r != NULL && r->qid) {
2893 		pf_mtag->qid = r->qid;
2894 
2895 		/* add hints for ecn */
2896 		pf_mtag->hdr = mtod(m, struct ip *);
2897 	}
2898 #endif /* ALTQ */
2899 	m->m_data += max_linkhdr;
2900 	m->m_pkthdr.len = m->m_len = len;
2901 	/* The rest of the stack assumes a rcvif, so provide one.
2902 	 * This is a locally generated packet, so .. close enough. */
2903 	m->m_pkthdr.rcvif = V_loif;
2904 	bzero(m->m_data, len);
2905 	switch (af) {
2906 #ifdef INET
2907 	case AF_INET:
2908 		h = mtod(m, struct ip *);
2909 
2910 		/* IP header fields included in the TCP checksum */
2911 		h->ip_p = IPPROTO_TCP;
2912 		h->ip_len = htons(tlen);
2913 		h->ip_src.s_addr = saddr->v4.s_addr;
2914 		h->ip_dst.s_addr = daddr->v4.s_addr;
2915 
2916 		th = (struct tcphdr *)((caddr_t)h + sizeof(struct ip));
2917 		break;
2918 #endif /* INET */
2919 #ifdef INET6
2920 	case AF_INET6:
2921 		h6 = mtod(m, struct ip6_hdr *);
2922 
2923 		/* IP header fields included in the TCP checksum */
2924 		h6->ip6_nxt = IPPROTO_TCP;
2925 		h6->ip6_plen = htons(tlen);
2926 		memcpy(&h6->ip6_src, &saddr->v6, sizeof(struct in6_addr));
2927 		memcpy(&h6->ip6_dst, &daddr->v6, sizeof(struct in6_addr));
2928 
2929 		th = (struct tcphdr *)((caddr_t)h6 + sizeof(struct ip6_hdr));
2930 		break;
2931 #endif /* INET6 */
2932 	}
2933 
2934 	/* TCP header */
2935 	th->th_sport = sport;
2936 	th->th_dport = dport;
2937 	th->th_seq = htonl(seq);
2938 	th->th_ack = htonl(ack);
2939 	th->th_off = tlen >> 2;
2940 	th->th_flags = tcp_flags;
2941 	th->th_win = htons(win);
2942 
2943 	if (mss) {
2944 		opt = (char *)(th + 1);
2945 		opt[0] = TCPOPT_MAXSEG;
2946 		opt[1] = 4;
2947 		HTONS(mss);
2948 		bcopy((caddr_t)&mss, (caddr_t)(opt + 2), 2);
2949 	}
2950 
2951 	switch (af) {
2952 #ifdef INET
2953 	case AF_INET:
2954 		/* TCP checksum */
2955 		th->th_sum = in_cksum(m, len);
2956 
2957 		/* Finish the IP header */
2958 		h->ip_v = 4;
2959 		h->ip_hl = sizeof(*h) >> 2;
2960 		h->ip_tos = IPTOS_LOWDELAY;
2961 		h->ip_off = htons(V_path_mtu_discovery ? IP_DF : 0);
2962 		h->ip_len = htons(len);
2963 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
2964 		h->ip_sum = 0;
2965 		break;
2966 #endif /* INET */
2967 #ifdef INET6
2968 	case AF_INET6:
2969 		/* TCP checksum */
2970 		th->th_sum = in6_cksum(m, IPPROTO_TCP,
2971 		    sizeof(struct ip6_hdr), tlen);
2972 
2973 		h6->ip6_vfc |= IPV6_VERSION;
2974 		h6->ip6_hlim = IPV6_DEFHLIM;
2975 		break;
2976 #endif /* INET6 */
2977 	}
2978 
2979 	return (m);
2980 }
2981 
2982 static void
2983 pf_send_sctp_abort(sa_family_t af, struct pf_pdesc *pd,
2984     uint8_t ttl, int rtableid)
2985 {
2986 	struct mbuf		*m;
2987 #ifdef INET
2988 	struct ip		*h = NULL;
2989 #endif /* INET */
2990 #ifdef INET6
2991 	struct ip6_hdr		*h6 = NULL;
2992 #endif /* INET6 */
2993 	struct sctphdr		*hdr;
2994 	struct sctp_chunkhdr	*chunk;
2995 	struct pf_send_entry	*pfse;
2996 	int			 off = 0;
2997 
2998 	MPASS(af == pd->af);
2999 
3000 	m = m_gethdr(M_NOWAIT, MT_DATA);
3001 	if (m == NULL)
3002 		return;
3003 
3004 	m->m_data += max_linkhdr;
3005 	m->m_flags |= M_SKIP_FIREWALL;
3006 	/* The rest of the stack assumes a rcvif, so provide one.
3007 	 * This is a locally generated packet, so .. close enough. */
3008 	m->m_pkthdr.rcvif = V_loif;
3009 
3010 	/* IPv4|6 header */
3011 	switch (af) {
3012 #ifdef INET
3013 	case AF_INET:
3014 		bzero(m->m_data, sizeof(struct ip) + sizeof(*hdr) + sizeof(*chunk));
3015 
3016 		h = mtod(m, struct ip *);
3017 
3018 		/* IP header fields included in the TCP checksum */
3019 
3020 		h->ip_p = IPPROTO_SCTP;
3021 		h->ip_len = htons(sizeof(*h) + sizeof(*hdr) + sizeof(*chunk));
3022 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
3023 		h->ip_src = pd->dst->v4;
3024 		h->ip_dst = pd->src->v4;
3025 
3026 		off += sizeof(struct ip);
3027 		break;
3028 #endif /* INET */
3029 #ifdef INET6
3030 	case AF_INET6:
3031 		bzero(m->m_data, sizeof(struct ip6_hdr) + sizeof(*hdr) + sizeof(*chunk));
3032 
3033 		h6 = mtod(m, struct ip6_hdr *);
3034 
3035 		/* IP header fields included in the TCP checksum */
3036 		h6->ip6_vfc |= IPV6_VERSION;
3037 		h6->ip6_nxt = IPPROTO_SCTP;
3038 		h6->ip6_plen = htons(sizeof(*h6) + sizeof(*hdr) + sizeof(*chunk));
3039 		h6->ip6_hlim = ttl ? ttl : V_ip6_defhlim;
3040 		memcpy(&h6->ip6_src, &pd->dst->v6, sizeof(struct in6_addr));
3041 		memcpy(&h6->ip6_dst, &pd->src->v6, sizeof(struct in6_addr));
3042 
3043 		off += sizeof(struct ip6_hdr);
3044 		break;
3045 #endif /* INET6 */
3046 	}
3047 
3048 	/* SCTP header */
3049 	hdr = mtodo(m, off);
3050 
3051 	hdr->src_port = pd->hdr.sctp.dest_port;
3052 	hdr->dest_port = pd->hdr.sctp.src_port;
3053 	hdr->v_tag = pd->sctp_initiate_tag;
3054 	hdr->checksum = 0;
3055 
3056 	/* Abort chunk. */
3057 	off += sizeof(struct sctphdr);
3058 	chunk = mtodo(m, off);
3059 
3060 	chunk->chunk_type = SCTP_ABORT_ASSOCIATION;
3061 	chunk->chunk_length = htons(sizeof(*chunk));
3062 
3063 	/* SCTP checksum */
3064 	off += sizeof(*chunk);
3065 	m->m_pkthdr.len = m->m_len = off;
3066 
3067 	pf_sctp_checksum(m, off - sizeof(*hdr) - sizeof(*chunk));;
3068 
3069 	if (rtableid >= 0)
3070 		M_SETFIB(m, rtableid);
3071 
3072 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3073 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3074 	if (pfse == NULL) {
3075 		m_freem(m);
3076 		return;
3077 	}
3078 
3079 	switch (af) {
3080 #ifdef INET
3081 	case AF_INET:
3082 		pfse->pfse_type = PFSE_IP;
3083 		break;
3084 #endif /* INET */
3085 #ifdef INET6
3086 	case AF_INET6:
3087 		pfse->pfse_type = PFSE_IP6;
3088 		break;
3089 #endif /* INET6 */
3090 	}
3091 
3092 	pfse->pfse_m = m;
3093 	pf_send(pfse);
3094 }
3095 
3096 void
3097 pf_send_tcp(const struct pf_krule *r, sa_family_t af,
3098     const struct pf_addr *saddr, const struct pf_addr *daddr,
3099     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
3100     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
3101     bool skip_firewall, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
3102 {
3103 	struct pf_send_entry *pfse;
3104 	struct mbuf	*m;
3105 
3106 	m = pf_build_tcp(r, af, saddr, daddr, sport, dport, seq, ack, tcp_flags,
3107 	    win, mss, ttl, skip_firewall, mtag_tag, mtag_flags, rtableid);
3108 	if (m == NULL)
3109 		return;
3110 
3111 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3112 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3113 	if (pfse == NULL) {
3114 		m_freem(m);
3115 		return;
3116 	}
3117 
3118 	switch (af) {
3119 #ifdef INET
3120 	case AF_INET:
3121 		pfse->pfse_type = PFSE_IP;
3122 		break;
3123 #endif /* INET */
3124 #ifdef INET6
3125 	case AF_INET6:
3126 		pfse->pfse_type = PFSE_IP6;
3127 		break;
3128 #endif /* INET6 */
3129 	}
3130 
3131 	pfse->pfse_m = m;
3132 	pf_send(pfse);
3133 }
3134 
3135 static void
3136 pf_return(struct pf_krule *r, struct pf_krule *nr, struct pf_pdesc *pd,
3137     struct pf_state_key *sk, int off, struct mbuf *m, struct tcphdr *th,
3138     struct pfi_kkif *kif, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen,
3139     u_short *reason, int rtableid)
3140 {
3141 	struct pf_addr	* const saddr = pd->src;
3142 	struct pf_addr	* const daddr = pd->dst;
3143 	sa_family_t	 af = pd->af;
3144 
3145 	/* undo NAT changes, if they have taken place */
3146 	if (nr != NULL) {
3147 		PF_ACPY(saddr, &sk->addr[pd->sidx], af);
3148 		PF_ACPY(daddr, &sk->addr[pd->didx], af);
3149 		if (pd->sport)
3150 			*pd->sport = sk->port[pd->sidx];
3151 		if (pd->dport)
3152 			*pd->dport = sk->port[pd->didx];
3153 		if (pd->proto_sum)
3154 			*pd->proto_sum = bproto_sum;
3155 		if (pd->ip_sum)
3156 			*pd->ip_sum = bip_sum;
3157 		m_copyback(m, off, hdrlen, pd->hdr.any);
3158 	}
3159 	if (pd->proto == IPPROTO_TCP &&
3160 	    ((r->rule_flag & PFRULE_RETURNRST) ||
3161 	    (r->rule_flag & PFRULE_RETURN)) &&
3162 	    !(th->th_flags & TH_RST)) {
3163 		u_int32_t	 ack = ntohl(th->th_seq) + pd->p_len;
3164 		int		 len = 0;
3165 #ifdef INET
3166 		struct ip	*h4;
3167 #endif
3168 #ifdef INET6
3169 		struct ip6_hdr	*h6;
3170 #endif
3171 
3172 		switch (af) {
3173 #ifdef INET
3174 		case AF_INET:
3175 			h4 = mtod(m, struct ip *);
3176 			len = ntohs(h4->ip_len) - off;
3177 			break;
3178 #endif
3179 #ifdef INET6
3180 		case AF_INET6:
3181 			h6 = mtod(m, struct ip6_hdr *);
3182 			len = ntohs(h6->ip6_plen) - (off - sizeof(*h6));
3183 			break;
3184 #endif
3185 		}
3186 
3187 		if (pf_check_proto_cksum(m, off, len, IPPROTO_TCP, af))
3188 			REASON_SET(reason, PFRES_PROTCKSUM);
3189 		else {
3190 			if (th->th_flags & TH_SYN)
3191 				ack++;
3192 			if (th->th_flags & TH_FIN)
3193 				ack++;
3194 			pf_send_tcp(r, af, pd->dst,
3195 				pd->src, th->th_dport, th->th_sport,
3196 				ntohl(th->th_ack), ack, TH_RST|TH_ACK, 0, 0,
3197 				r->return_ttl, true, 0, 0, rtableid);
3198 		}
3199 	} else if (pd->proto == IPPROTO_SCTP &&
3200 	    (r->rule_flag & PFRULE_RETURN)) {
3201 		pf_send_sctp_abort(af, pd, r->return_ttl, rtableid);
3202 	} else if (pd->proto != IPPROTO_ICMP && af == AF_INET &&
3203 		r->return_icmp)
3204 		pf_send_icmp(m, r->return_icmp >> 8,
3205 			r->return_icmp & 255, af, r, rtableid);
3206 	else if (pd->proto != IPPROTO_ICMPV6 && af == AF_INET6 &&
3207 		r->return_icmp6)
3208 		pf_send_icmp(m, r->return_icmp6 >> 8,
3209 			r->return_icmp6 & 255, af, r, rtableid);
3210 }
3211 
3212 static int
3213 pf_match_ieee8021q_pcp(u_int8_t prio, struct mbuf *m)
3214 {
3215 	struct m_tag *mtag;
3216 	u_int8_t mpcp;
3217 
3218 	mtag = m_tag_locate(m, MTAG_8021Q, MTAG_8021Q_PCP_IN, NULL);
3219 	if (mtag == NULL)
3220 		return (0);
3221 
3222 	if (prio == PF_PRIO_ZERO)
3223 		prio = 0;
3224 
3225 	mpcp = *(uint8_t *)(mtag + 1);
3226 
3227 	return (mpcp == prio);
3228 }
3229 
3230 static int
3231 pf_icmp_to_bandlim(uint8_t type)
3232 {
3233 	switch (type) {
3234 		case ICMP_ECHO:
3235 		case ICMP_ECHOREPLY:
3236 			return (BANDLIM_ICMP_ECHO);
3237 		case ICMP_TSTAMP:
3238 		case ICMP_TSTAMPREPLY:
3239 			return (BANDLIM_ICMP_TSTAMP);
3240 		case ICMP_UNREACH:
3241 		default:
3242 			return (BANDLIM_ICMP_UNREACH);
3243 	}
3244 }
3245 
3246 static void
3247 pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af,
3248     struct pf_krule *r, int rtableid)
3249 {
3250 	struct pf_send_entry *pfse;
3251 	struct mbuf *m0;
3252 	struct pf_mtag *pf_mtag;
3253 
3254 	/* ICMP packet rate limitation. */
3255 #ifdef INET6
3256 	if (af == AF_INET6) {
3257 		if (icmp6_ratelimit(NULL, type, code))
3258 			return;
3259 	}
3260 #endif
3261 #ifdef INET
3262 	if (af == AF_INET) {
3263 		if (badport_bandlim(pf_icmp_to_bandlim(type)) != 0)
3264 			return;
3265 	}
3266 #endif
3267 
3268 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3269 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3270 	if (pfse == NULL)
3271 		return;
3272 
3273 	if ((m0 = m_copypacket(m, M_NOWAIT)) == NULL) {
3274 		free(pfse, M_PFTEMP);
3275 		return;
3276 	}
3277 
3278 	if ((pf_mtag = pf_get_mtag(m0)) == NULL) {
3279 		free(pfse, M_PFTEMP);
3280 		return;
3281 	}
3282 	/* XXX: revisit */
3283 	m0->m_flags |= M_SKIP_FIREWALL;
3284 
3285 	if (rtableid >= 0)
3286 		M_SETFIB(m0, rtableid);
3287 
3288 #ifdef ALTQ
3289 	if (r->qid) {
3290 		pf_mtag->qid = r->qid;
3291 		/* add hints for ecn */
3292 		pf_mtag->hdr = mtod(m0, struct ip *);
3293 	}
3294 #endif /* ALTQ */
3295 
3296 	switch (af) {
3297 #ifdef INET
3298 	case AF_INET:
3299 		pfse->pfse_type = PFSE_ICMP;
3300 		break;
3301 #endif /* INET */
3302 #ifdef INET6
3303 	case AF_INET6:
3304 		pfse->pfse_type = PFSE_ICMP6;
3305 		break;
3306 #endif /* INET6 */
3307 	}
3308 	pfse->pfse_m = m0;
3309 	pfse->icmpopts.type = type;
3310 	pfse->icmpopts.code = code;
3311 	pf_send(pfse);
3312 }
3313 
3314 /*
3315  * Return 1 if the addresses a and b match (with mask m), otherwise return 0.
3316  * If n is 0, they match if they are equal. If n is != 0, they match if they
3317  * are different.
3318  */
3319 int
3320 pf_match_addr(u_int8_t n, struct pf_addr *a, struct pf_addr *m,
3321     struct pf_addr *b, sa_family_t af)
3322 {
3323 	int	match = 0;
3324 
3325 	switch (af) {
3326 #ifdef INET
3327 	case AF_INET:
3328 		if ((a->addr32[0] & m->addr32[0]) ==
3329 		    (b->addr32[0] & m->addr32[0]))
3330 			match++;
3331 		break;
3332 #endif /* INET */
3333 #ifdef INET6
3334 	case AF_INET6:
3335 		if (((a->addr32[0] & m->addr32[0]) ==
3336 		     (b->addr32[0] & m->addr32[0])) &&
3337 		    ((a->addr32[1] & m->addr32[1]) ==
3338 		     (b->addr32[1] & m->addr32[1])) &&
3339 		    ((a->addr32[2] & m->addr32[2]) ==
3340 		     (b->addr32[2] & m->addr32[2])) &&
3341 		    ((a->addr32[3] & m->addr32[3]) ==
3342 		     (b->addr32[3] & m->addr32[3])))
3343 			match++;
3344 		break;
3345 #endif /* INET6 */
3346 	}
3347 	if (match) {
3348 		if (n)
3349 			return (0);
3350 		else
3351 			return (1);
3352 	} else {
3353 		if (n)
3354 			return (1);
3355 		else
3356 			return (0);
3357 	}
3358 }
3359 
3360 /*
3361  * Return 1 if b <= a <= e, otherwise return 0.
3362  */
3363 int
3364 pf_match_addr_range(struct pf_addr *b, struct pf_addr *e,
3365     struct pf_addr *a, sa_family_t af)
3366 {
3367 	switch (af) {
3368 #ifdef INET
3369 	case AF_INET:
3370 		if ((ntohl(a->addr32[0]) < ntohl(b->addr32[0])) ||
3371 		    (ntohl(a->addr32[0]) > ntohl(e->addr32[0])))
3372 			return (0);
3373 		break;
3374 #endif /* INET */
3375 #ifdef INET6
3376 	case AF_INET6: {
3377 		int	i;
3378 
3379 		/* check a >= b */
3380 		for (i = 0; i < 4; ++i)
3381 			if (ntohl(a->addr32[i]) > ntohl(b->addr32[i]))
3382 				break;
3383 			else if (ntohl(a->addr32[i]) < ntohl(b->addr32[i]))
3384 				return (0);
3385 		/* check a <= e */
3386 		for (i = 0; i < 4; ++i)
3387 			if (ntohl(a->addr32[i]) < ntohl(e->addr32[i]))
3388 				break;
3389 			else if (ntohl(a->addr32[i]) > ntohl(e->addr32[i]))
3390 				return (0);
3391 		break;
3392 	}
3393 #endif /* INET6 */
3394 	}
3395 	return (1);
3396 }
3397 
3398 static int
3399 pf_match(u_int8_t op, u_int32_t a1, u_int32_t a2, u_int32_t p)
3400 {
3401 	switch (op) {
3402 	case PF_OP_IRG:
3403 		return ((p > a1) && (p < a2));
3404 	case PF_OP_XRG:
3405 		return ((p < a1) || (p > a2));
3406 	case PF_OP_RRG:
3407 		return ((p >= a1) && (p <= a2));
3408 	case PF_OP_EQ:
3409 		return (p == a1);
3410 	case PF_OP_NE:
3411 		return (p != a1);
3412 	case PF_OP_LT:
3413 		return (p < a1);
3414 	case PF_OP_LE:
3415 		return (p <= a1);
3416 	case PF_OP_GT:
3417 		return (p > a1);
3418 	case PF_OP_GE:
3419 		return (p >= a1);
3420 	}
3421 	return (0); /* never reached */
3422 }
3423 
3424 int
3425 pf_match_port(u_int8_t op, u_int16_t a1, u_int16_t a2, u_int16_t p)
3426 {
3427 	NTOHS(a1);
3428 	NTOHS(a2);
3429 	NTOHS(p);
3430 	return (pf_match(op, a1, a2, p));
3431 }
3432 
3433 static int
3434 pf_match_uid(u_int8_t op, uid_t a1, uid_t a2, uid_t u)
3435 {
3436 	if (u == UID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
3437 		return (0);
3438 	return (pf_match(op, a1, a2, u));
3439 }
3440 
3441 static int
3442 pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g)
3443 {
3444 	if (g == GID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
3445 		return (0);
3446 	return (pf_match(op, a1, a2, g));
3447 }
3448 
3449 int
3450 pf_match_tag(struct mbuf *m, struct pf_krule *r, int *tag, int mtag)
3451 {
3452 	if (*tag == -1)
3453 		*tag = mtag;
3454 
3455 	return ((!r->match_tag_not && r->match_tag == *tag) ||
3456 	    (r->match_tag_not && r->match_tag != *tag));
3457 }
3458 
3459 int
3460 pf_tag_packet(struct mbuf *m, struct pf_pdesc *pd, int tag)
3461 {
3462 
3463 	KASSERT(tag > 0, ("%s: tag %d", __func__, tag));
3464 
3465 	if (pd->pf_mtag == NULL && ((pd->pf_mtag = pf_get_mtag(m)) == NULL))
3466 		return (ENOMEM);
3467 
3468 	pd->pf_mtag->tag = tag;
3469 
3470 	return (0);
3471 }
3472 
3473 #define	PF_ANCHOR_STACKSIZE	32
3474 struct pf_kanchor_stackframe {
3475 	struct pf_kruleset	*rs;
3476 	struct pf_krule		*r;	/* XXX: + match bit */
3477 	struct pf_kanchor	*child;
3478 };
3479 
3480 /*
3481  * XXX: We rely on malloc(9) returning pointer aligned addresses.
3482  */
3483 #define	PF_ANCHORSTACK_MATCH	0x00000001
3484 #define	PF_ANCHORSTACK_MASK	(PF_ANCHORSTACK_MATCH)
3485 
3486 #define	PF_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
3487 #define	PF_ANCHOR_RULE(f)	(struct pf_krule *)			\
3488 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
3489 #define	PF_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 			\
3490 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
3491 } while (0)
3492 
3493 void
3494 pf_step_into_anchor(struct pf_kanchor_stackframe *stack, int *depth,
3495     struct pf_kruleset **rs, int n, struct pf_krule **r, struct pf_krule **a,
3496     int *match)
3497 {
3498 	struct pf_kanchor_stackframe	*f;
3499 
3500 	PF_RULES_RASSERT();
3501 
3502 	if (match)
3503 		*match = 0;
3504 	if (*depth >= PF_ANCHOR_STACKSIZE) {
3505 		printf("%s: anchor stack overflow on %s\n",
3506 		    __func__, (*r)->anchor->name);
3507 		*r = TAILQ_NEXT(*r, entries);
3508 		return;
3509 	} else if (*depth == 0 && a != NULL)
3510 		*a = *r;
3511 	f = stack + (*depth)++;
3512 	f->rs = *rs;
3513 	f->r = *r;
3514 	if ((*r)->anchor_wildcard) {
3515 		struct pf_kanchor_node *parent = &(*r)->anchor->children;
3516 
3517 		if ((f->child = RB_MIN(pf_kanchor_node, parent)) == NULL) {
3518 			*r = NULL;
3519 			return;
3520 		}
3521 		*rs = &f->child->ruleset;
3522 	} else {
3523 		f->child = NULL;
3524 		*rs = &(*r)->anchor->ruleset;
3525 	}
3526 	*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
3527 }
3528 
3529 int
3530 pf_step_out_of_anchor(struct pf_kanchor_stackframe *stack, int *depth,
3531     struct pf_kruleset **rs, int n, struct pf_krule **r, struct pf_krule **a,
3532     int *match)
3533 {
3534 	struct pf_kanchor_stackframe	*f;
3535 	struct pf_krule *fr;
3536 	int quick = 0;
3537 
3538 	PF_RULES_RASSERT();
3539 
3540 	do {
3541 		if (*depth <= 0)
3542 			break;
3543 		f = stack + *depth - 1;
3544 		fr = PF_ANCHOR_RULE(f);
3545 		if (f->child != NULL) {
3546 			/*
3547 			 * This block traverses through
3548 			 * a wildcard anchor.
3549 			 */
3550 			if (match != NULL && *match) {
3551 				/*
3552 				 * If any of "*" matched, then
3553 				 * "foo/ *" matched, mark frame
3554 				 * appropriately.
3555 				 */
3556 				PF_ANCHOR_SET_MATCH(f);
3557 				*match = 0;
3558 			}
3559 			f->child = RB_NEXT(pf_kanchor_node,
3560 			    &fr->anchor->children, f->child);
3561 			if (f->child != NULL) {
3562 				*rs = &f->child->ruleset;
3563 				*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
3564 				if (*r == NULL)
3565 					continue;
3566 				else
3567 					break;
3568 			}
3569 		}
3570 		(*depth)--;
3571 		if (*depth == 0 && a != NULL)
3572 			*a = NULL;
3573 		*rs = f->rs;
3574 		if (PF_ANCHOR_MATCH(f) || (match != NULL && *match))
3575 			quick = fr->quick;
3576 		*r = TAILQ_NEXT(fr, entries);
3577 	} while (*r == NULL);
3578 
3579 	return (quick);
3580 }
3581 
3582 struct pf_keth_anchor_stackframe {
3583 	struct pf_keth_ruleset	*rs;
3584 	struct pf_keth_rule	*r;	/* XXX: + match bit */
3585 	struct pf_keth_anchor	*child;
3586 };
3587 
3588 #define	PF_ETH_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
3589 #define	PF_ETH_ANCHOR_RULE(f)	(struct pf_keth_rule *)			\
3590 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
3591 #define	PF_ETH_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 		\
3592 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
3593 } while (0)
3594 
3595 void
3596 pf_step_into_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
3597     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
3598     struct pf_keth_rule **a, int *match)
3599 {
3600 	struct pf_keth_anchor_stackframe	*f;
3601 
3602 	NET_EPOCH_ASSERT();
3603 
3604 	if (match)
3605 		*match = 0;
3606 	if (*depth >= PF_ANCHOR_STACKSIZE) {
3607 		printf("%s: anchor stack overflow on %s\n",
3608 		    __func__, (*r)->anchor->name);
3609 		*r = TAILQ_NEXT(*r, entries);
3610 		return;
3611 	} else if (*depth == 0 && a != NULL)
3612 		*a = *r;
3613 	f = stack + (*depth)++;
3614 	f->rs = *rs;
3615 	f->r = *r;
3616 	if ((*r)->anchor_wildcard) {
3617 		struct pf_keth_anchor_node *parent = &(*r)->anchor->children;
3618 
3619 		if ((f->child = RB_MIN(pf_keth_anchor_node, parent)) == NULL) {
3620 			*r = NULL;
3621 			return;
3622 		}
3623 		*rs = &f->child->ruleset;
3624 	} else {
3625 		f->child = NULL;
3626 		*rs = &(*r)->anchor->ruleset;
3627 	}
3628 	*r = TAILQ_FIRST((*rs)->active.rules);
3629 }
3630 
3631 int
3632 pf_step_out_of_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
3633     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
3634     struct pf_keth_rule **a, int *match)
3635 {
3636 	struct pf_keth_anchor_stackframe	*f;
3637 	struct pf_keth_rule *fr;
3638 	int quick = 0;
3639 
3640 	NET_EPOCH_ASSERT();
3641 
3642 	do {
3643 		if (*depth <= 0)
3644 			break;
3645 		f = stack + *depth - 1;
3646 		fr = PF_ETH_ANCHOR_RULE(f);
3647 		if (f->child != NULL) {
3648 			/*
3649 			 * This block traverses through
3650 			 * a wildcard anchor.
3651 			 */
3652 			if (match != NULL && *match) {
3653 				/*
3654 				 * If any of "*" matched, then
3655 				 * "foo/ *" matched, mark frame
3656 				 * appropriately.
3657 				 */
3658 				PF_ETH_ANCHOR_SET_MATCH(f);
3659 				*match = 0;
3660 			}
3661 			f->child = RB_NEXT(pf_keth_anchor_node,
3662 			    &fr->anchor->children, f->child);
3663 			if (f->child != NULL) {
3664 				*rs = &f->child->ruleset;
3665 				*r = TAILQ_FIRST((*rs)->active.rules);
3666 				if (*r == NULL)
3667 					continue;
3668 				else
3669 					break;
3670 			}
3671 		}
3672 		(*depth)--;
3673 		if (*depth == 0 && a != NULL)
3674 			*a = NULL;
3675 		*rs = f->rs;
3676 		if (PF_ETH_ANCHOR_MATCH(f) || (match != NULL && *match))
3677 			quick = fr->quick;
3678 		*r = TAILQ_NEXT(fr, entries);
3679 	} while (*r == NULL);
3680 
3681 	return (quick);
3682 }
3683 
3684 #ifdef INET6
3685 void
3686 pf_poolmask(struct pf_addr *naddr, struct pf_addr *raddr,
3687     struct pf_addr *rmask, struct pf_addr *saddr, sa_family_t af)
3688 {
3689 	switch (af) {
3690 #ifdef INET
3691 	case AF_INET:
3692 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
3693 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
3694 		break;
3695 #endif /* INET */
3696 	case AF_INET6:
3697 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
3698 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
3699 		naddr->addr32[1] = (raddr->addr32[1] & rmask->addr32[1]) |
3700 		((rmask->addr32[1] ^ 0xffffffff ) & saddr->addr32[1]);
3701 		naddr->addr32[2] = (raddr->addr32[2] & rmask->addr32[2]) |
3702 		((rmask->addr32[2] ^ 0xffffffff ) & saddr->addr32[2]);
3703 		naddr->addr32[3] = (raddr->addr32[3] & rmask->addr32[3]) |
3704 		((rmask->addr32[3] ^ 0xffffffff ) & saddr->addr32[3]);
3705 		break;
3706 	}
3707 }
3708 
3709 void
3710 pf_addr_inc(struct pf_addr *addr, sa_family_t af)
3711 {
3712 	switch (af) {
3713 #ifdef INET
3714 	case AF_INET:
3715 		addr->addr32[0] = htonl(ntohl(addr->addr32[0]) + 1);
3716 		break;
3717 #endif /* INET */
3718 	case AF_INET6:
3719 		if (addr->addr32[3] == 0xffffffff) {
3720 			addr->addr32[3] = 0;
3721 			if (addr->addr32[2] == 0xffffffff) {
3722 				addr->addr32[2] = 0;
3723 				if (addr->addr32[1] == 0xffffffff) {
3724 					addr->addr32[1] = 0;
3725 					addr->addr32[0] =
3726 					    htonl(ntohl(addr->addr32[0]) + 1);
3727 				} else
3728 					addr->addr32[1] =
3729 					    htonl(ntohl(addr->addr32[1]) + 1);
3730 			} else
3731 				addr->addr32[2] =
3732 				    htonl(ntohl(addr->addr32[2]) + 1);
3733 		} else
3734 			addr->addr32[3] =
3735 			    htonl(ntohl(addr->addr32[3]) + 1);
3736 		break;
3737 	}
3738 }
3739 #endif /* INET6 */
3740 
3741 void
3742 pf_rule_to_actions(struct pf_krule *r, struct pf_rule_actions *a)
3743 {
3744 	/*
3745 	 * Modern rules use the same flags in rules as they do in states.
3746 	 */
3747 	a->flags |= (r->scrub_flags & (PFSTATE_NODF|PFSTATE_RANDOMID|
3748 	    PFSTATE_SCRUB_TCP|PFSTATE_SETPRIO));
3749 
3750 	/*
3751 	 * Old-style scrub rules have different flags which need to be translated.
3752 	 */
3753 	if (r->rule_flag & PFRULE_RANDOMID)
3754 		a->flags |= PFSTATE_RANDOMID;
3755 	if (r->scrub_flags & PFSTATE_SETTOS || r->rule_flag & PFRULE_SET_TOS ) {
3756 		a->flags |= PFSTATE_SETTOS;
3757 		a->set_tos = r->set_tos;
3758 	}
3759 
3760 	if (r->qid)
3761 		a->qid = r->qid;
3762 	if (r->pqid)
3763 		a->pqid = r->pqid;
3764 	if (r->rtableid >= 0)
3765 		a->rtableid = r->rtableid;
3766 	a->log |= r->log;
3767 	if (r->min_ttl)
3768 		a->min_ttl = r->min_ttl;
3769 	if (r->max_mss)
3770 		a->max_mss = r->max_mss;
3771 	if (r->dnpipe)
3772 		a->dnpipe = r->dnpipe;
3773 	if (r->dnrpipe)
3774 		a->dnrpipe = r->dnrpipe;
3775 	if (r->dnpipe || r->dnrpipe) {
3776 		if (r->free_flags & PFRULE_DN_IS_PIPE)
3777 			a->flags |= PFSTATE_DN_IS_PIPE;
3778 		else
3779 			a->flags &= ~PFSTATE_DN_IS_PIPE;
3780 	}
3781 	if (r->scrub_flags & PFSTATE_SETPRIO) {
3782 		a->set_prio[0] = r->set_prio[0];
3783 		a->set_prio[1] = r->set_prio[1];
3784 	}
3785 }
3786 
3787 int
3788 pf_socket_lookup(struct pf_pdesc *pd, struct mbuf *m)
3789 {
3790 	struct pf_addr		*saddr, *daddr;
3791 	u_int16_t		 sport, dport;
3792 	struct inpcbinfo	*pi;
3793 	struct inpcb		*inp;
3794 
3795 	pd->lookup.uid = UID_MAX;
3796 	pd->lookup.gid = GID_MAX;
3797 
3798 	switch (pd->proto) {
3799 	case IPPROTO_TCP:
3800 		sport = pd->hdr.tcp.th_sport;
3801 		dport = pd->hdr.tcp.th_dport;
3802 		pi = &V_tcbinfo;
3803 		break;
3804 	case IPPROTO_UDP:
3805 		sport = pd->hdr.udp.uh_sport;
3806 		dport = pd->hdr.udp.uh_dport;
3807 		pi = &V_udbinfo;
3808 		break;
3809 	default:
3810 		return (-1);
3811 	}
3812 	if (pd->dir == PF_IN) {
3813 		saddr = pd->src;
3814 		daddr = pd->dst;
3815 	} else {
3816 		u_int16_t	p;
3817 
3818 		p = sport;
3819 		sport = dport;
3820 		dport = p;
3821 		saddr = pd->dst;
3822 		daddr = pd->src;
3823 	}
3824 	switch (pd->af) {
3825 #ifdef INET
3826 	case AF_INET:
3827 		inp = in_pcblookup_mbuf(pi, saddr->v4, sport, daddr->v4,
3828 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
3829 		if (inp == NULL) {
3830 			inp = in_pcblookup_mbuf(pi, saddr->v4, sport,
3831 			   daddr->v4, dport, INPLOOKUP_WILDCARD |
3832 			   INPLOOKUP_RLOCKPCB, NULL, m);
3833 			if (inp == NULL)
3834 				return (-1);
3835 		}
3836 		break;
3837 #endif /* INET */
3838 #ifdef INET6
3839 	case AF_INET6:
3840 		inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport, &daddr->v6,
3841 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
3842 		if (inp == NULL) {
3843 			inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport,
3844 			    &daddr->v6, dport, INPLOOKUP_WILDCARD |
3845 			    INPLOOKUP_RLOCKPCB, NULL, m);
3846 			if (inp == NULL)
3847 				return (-1);
3848 		}
3849 		break;
3850 #endif /* INET6 */
3851 
3852 	default:
3853 		return (-1);
3854 	}
3855 	INP_RLOCK_ASSERT(inp);
3856 	pd->lookup.uid = inp->inp_cred->cr_uid;
3857 	pd->lookup.gid = inp->inp_cred->cr_groups[0];
3858 	INP_RUNLOCK(inp);
3859 
3860 	return (1);
3861 }
3862 
3863 u_int8_t
3864 pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
3865 {
3866 	int		 hlen;
3867 	u_int8_t	 hdr[60];
3868 	u_int8_t	*opt, optlen;
3869 	u_int8_t	 wscale = 0;
3870 
3871 	hlen = th_off << 2;		/* hlen <= sizeof(hdr) */
3872 	if (hlen <= sizeof(struct tcphdr))
3873 		return (0);
3874 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
3875 		return (0);
3876 	opt = hdr + sizeof(struct tcphdr);
3877 	hlen -= sizeof(struct tcphdr);
3878 	while (hlen >= 3) {
3879 		switch (*opt) {
3880 		case TCPOPT_EOL:
3881 		case TCPOPT_NOP:
3882 			++opt;
3883 			--hlen;
3884 			break;
3885 		case TCPOPT_WINDOW:
3886 			wscale = opt[2];
3887 			if (wscale > TCP_MAX_WINSHIFT)
3888 				wscale = TCP_MAX_WINSHIFT;
3889 			wscale |= PF_WSCALE_FLAG;
3890 			/* FALLTHROUGH */
3891 		default:
3892 			optlen = opt[1];
3893 			if (optlen < 2)
3894 				optlen = 2;
3895 			hlen -= optlen;
3896 			opt += optlen;
3897 			break;
3898 		}
3899 	}
3900 	return (wscale);
3901 }
3902 
3903 u_int16_t
3904 pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
3905 {
3906 	int		 hlen;
3907 	u_int8_t	 hdr[60];
3908 	u_int8_t	*opt, optlen;
3909 	u_int16_t	 mss = V_tcp_mssdflt;
3910 
3911 	hlen = th_off << 2;	/* hlen <= sizeof(hdr) */
3912 	if (hlen <= sizeof(struct tcphdr))
3913 		return (0);
3914 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
3915 		return (0);
3916 	opt = hdr + sizeof(struct tcphdr);
3917 	hlen -= sizeof(struct tcphdr);
3918 	while (hlen >= TCPOLEN_MAXSEG) {
3919 		switch (*opt) {
3920 		case TCPOPT_EOL:
3921 		case TCPOPT_NOP:
3922 			++opt;
3923 			--hlen;
3924 			break;
3925 		case TCPOPT_MAXSEG:
3926 			bcopy((caddr_t)(opt + 2), (caddr_t)&mss, 2);
3927 			NTOHS(mss);
3928 			/* FALLTHROUGH */
3929 		default:
3930 			optlen = opt[1];
3931 			if (optlen < 2)
3932 				optlen = 2;
3933 			hlen -= optlen;
3934 			opt += optlen;
3935 			break;
3936 		}
3937 	}
3938 	return (mss);
3939 }
3940 
3941 static u_int16_t
3942 pf_calc_mss(struct pf_addr *addr, sa_family_t af, int rtableid, u_int16_t offer)
3943 {
3944 	struct nhop_object *nh;
3945 #ifdef INET6
3946 	struct in6_addr		dst6;
3947 	uint32_t		scopeid;
3948 #endif /* INET6 */
3949 	int			 hlen = 0;
3950 	uint16_t		 mss = 0;
3951 
3952 	NET_EPOCH_ASSERT();
3953 
3954 	switch (af) {
3955 #ifdef INET
3956 	case AF_INET:
3957 		hlen = sizeof(struct ip);
3958 		nh = fib4_lookup(rtableid, addr->v4, 0, 0, 0);
3959 		if (nh != NULL)
3960 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
3961 		break;
3962 #endif /* INET */
3963 #ifdef INET6
3964 	case AF_INET6:
3965 		hlen = sizeof(struct ip6_hdr);
3966 		in6_splitscope(&addr->v6, &dst6, &scopeid);
3967 		nh = fib6_lookup(rtableid, &dst6, scopeid, 0, 0);
3968 		if (nh != NULL)
3969 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
3970 		break;
3971 #endif /* INET6 */
3972 	}
3973 
3974 	mss = max(V_tcp_mssdflt, mss);
3975 	mss = min(mss, offer);
3976 	mss = max(mss, 64);		/* sanity - at least max opt space */
3977 	return (mss);
3978 }
3979 
3980 static u_int32_t
3981 pf_tcp_iss(struct pf_pdesc *pd)
3982 {
3983 	MD5_CTX ctx;
3984 	u_int32_t digest[4];
3985 
3986 	if (V_pf_tcp_secret_init == 0) {
3987 		arc4random_buf(&V_pf_tcp_secret, sizeof(V_pf_tcp_secret));
3988 		MD5Init(&V_pf_tcp_secret_ctx);
3989 		MD5Update(&V_pf_tcp_secret_ctx, V_pf_tcp_secret,
3990 		    sizeof(V_pf_tcp_secret));
3991 		V_pf_tcp_secret_init = 1;
3992 	}
3993 
3994 	ctx = V_pf_tcp_secret_ctx;
3995 
3996 	MD5Update(&ctx, (char *)&pd->hdr.tcp.th_sport, sizeof(u_short));
3997 	MD5Update(&ctx, (char *)&pd->hdr.tcp.th_dport, sizeof(u_short));
3998 	if (pd->af == AF_INET6) {
3999 		MD5Update(&ctx, (char *)&pd->src->v6, sizeof(struct in6_addr));
4000 		MD5Update(&ctx, (char *)&pd->dst->v6, sizeof(struct in6_addr));
4001 	} else {
4002 		MD5Update(&ctx, (char *)&pd->src->v4, sizeof(struct in_addr));
4003 		MD5Update(&ctx, (char *)&pd->dst->v4, sizeof(struct in_addr));
4004 	}
4005 	MD5Final((u_char *)digest, &ctx);
4006 	V_pf_tcp_iss_off += 4096;
4007 #define	ISN_RANDOM_INCREMENT (4096 - 1)
4008 	return (digest[0] + (arc4random() & ISN_RANDOM_INCREMENT) +
4009 	    V_pf_tcp_iss_off);
4010 #undef	ISN_RANDOM_INCREMENT
4011 }
4012 
4013 static bool
4014 pf_match_eth_addr(const uint8_t *a, const struct pf_keth_rule_addr *r)
4015 {
4016 	bool match = true;
4017 
4018 	/* Always matches if not set */
4019 	if (! r->isset)
4020 		return (!r->neg);
4021 
4022 	for (int i = 0; i < ETHER_ADDR_LEN; i++) {
4023 		if ((a[i] & r->mask[i]) != (r->addr[i] & r->mask[i])) {
4024 			match = false;
4025 			break;
4026 		}
4027 	}
4028 
4029 	return (match ^ r->neg);
4030 }
4031 
4032 static int
4033 pf_match_eth_tag(struct mbuf *m, struct pf_keth_rule *r, int *tag, int mtag)
4034 {
4035 	if (*tag == -1)
4036 		*tag = mtag;
4037 
4038 	return ((!r->match_tag_not && r->match_tag == *tag) ||
4039 	    (r->match_tag_not && r->match_tag != *tag));
4040 }
4041 
4042 static void
4043 pf_bridge_to(struct ifnet *ifp, struct mbuf *m)
4044 {
4045 	/* If we don't have the interface drop the packet. */
4046 	if (ifp == NULL) {
4047 		m_freem(m);
4048 		return;
4049 	}
4050 
4051 	switch (ifp->if_type) {
4052 	case IFT_ETHER:
4053 	case IFT_XETHER:
4054 	case IFT_L2VLAN:
4055 	case IFT_BRIDGE:
4056 	case IFT_IEEE8023ADLAG:
4057 		break;
4058 	default:
4059 		m_freem(m);
4060 		return;
4061 	}
4062 
4063 	ifp->if_transmit(ifp, m);
4064 }
4065 
4066 static int
4067 pf_test_eth_rule(int dir, struct pfi_kkif *kif, struct mbuf **m0)
4068 {
4069 #ifdef INET
4070 	struct ip ip;
4071 #endif
4072 #ifdef INET6
4073 	struct ip6_hdr ip6;
4074 #endif
4075 	struct mbuf *m = *m0;
4076 	struct ether_header *e;
4077 	struct pf_keth_rule *r, *rm, *a = NULL;
4078 	struct pf_keth_ruleset *ruleset = NULL;
4079 	struct pf_mtag *mtag;
4080 	struct pf_keth_ruleq *rules;
4081 	struct pf_addr *src = NULL, *dst = NULL;
4082 	struct pfi_kkif *bridge_to;
4083 	sa_family_t af = 0;
4084 	uint16_t proto;
4085 	int asd = 0, match = 0;
4086 	int tag = -1;
4087 	uint8_t action;
4088 	struct pf_keth_anchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
4089 
4090 	MPASS(kif->pfik_ifp->if_vnet == curvnet);
4091 	NET_EPOCH_ASSERT();
4092 
4093 	PF_RULES_RLOCK_TRACKER;
4094 
4095 	SDT_PROBE3(pf, eth, test_rule, entry, dir, kif->pfik_ifp, m);
4096 
4097 	mtag = pf_find_mtag(m);
4098 	if (mtag != NULL && mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
4099 		/* Dummynet re-injects packets after they've
4100 		 * completed their delay. We've already
4101 		 * processed them, so pass unconditionally. */
4102 
4103 		/* But only once. We may see the packet multiple times (e.g.
4104 		 * PFIL_IN/PFIL_OUT). */
4105 		mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
4106 
4107 		return (PF_PASS);
4108 	}
4109 
4110 	ruleset = V_pf_keth;
4111 	rules = ck_pr_load_ptr(&ruleset->active.rules);
4112 	r = TAILQ_FIRST(rules);
4113 	rm = NULL;
4114 
4115 	e = mtod(m, struct ether_header *);
4116 	proto = ntohs(e->ether_type);
4117 
4118 	switch (proto) {
4119 #ifdef INET
4120 	case ETHERTYPE_IP: {
4121 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
4122 		    sizeof(ip)))
4123 			return (PF_DROP);
4124 
4125 		af = AF_INET;
4126 		m_copydata(m, sizeof(struct ether_header), sizeof(ip),
4127 		    (caddr_t)&ip);
4128 		src = (struct pf_addr *)&ip.ip_src;
4129 		dst = (struct pf_addr *)&ip.ip_dst;
4130 		break;
4131 	}
4132 #endif /* INET */
4133 #ifdef INET6
4134 	case ETHERTYPE_IPV6: {
4135 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
4136 		    sizeof(ip6)))
4137 			return (PF_DROP);
4138 
4139 		af = AF_INET6;
4140 		m_copydata(m, sizeof(struct ether_header), sizeof(ip6),
4141 		    (caddr_t)&ip6);
4142 		src = (struct pf_addr *)&ip6.ip6_src;
4143 		dst = (struct pf_addr *)&ip6.ip6_dst;
4144 		break;
4145 	}
4146 #endif /* INET6 */
4147 	}
4148 
4149 	PF_RULES_RLOCK();
4150 
4151 	while (r != NULL) {
4152 		counter_u64_add(r->evaluations, 1);
4153 		SDT_PROBE2(pf, eth, test_rule, test, r->nr, r);
4154 
4155 		if (pfi_kkif_match(r->kif, kif) == r->ifnot) {
4156 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4157 			    "kif");
4158 			r = r->skip[PFE_SKIP_IFP].ptr;
4159 		}
4160 		else if (r->direction && r->direction != dir) {
4161 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4162 			    "dir");
4163 			r = r->skip[PFE_SKIP_DIR].ptr;
4164 		}
4165 		else if (r->proto && r->proto != proto) {
4166 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4167 			    "proto");
4168 			r = r->skip[PFE_SKIP_PROTO].ptr;
4169 		}
4170 		else if (! pf_match_eth_addr(e->ether_shost, &r->src)) {
4171 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4172 			    "src");
4173 			r = r->skip[PFE_SKIP_SRC_ADDR].ptr;
4174 		}
4175 		else if (! pf_match_eth_addr(e->ether_dhost, &r->dst)) {
4176 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4177 			    "dst");
4178 			r = r->skip[PFE_SKIP_DST_ADDR].ptr;
4179 		}
4180 		else if (src != NULL && PF_MISMATCHAW(&r->ipsrc.addr, src, af,
4181 		    r->ipsrc.neg, kif, M_GETFIB(m))) {
4182 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4183 			    "ip_src");
4184 			r = r->skip[PFE_SKIP_SRC_IP_ADDR].ptr;
4185 		}
4186 		else if (dst != NULL && PF_MISMATCHAW(&r->ipdst.addr, dst, af,
4187 		    r->ipdst.neg, kif, M_GETFIB(m))) {
4188 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4189 			    "ip_dst");
4190 			r = r->skip[PFE_SKIP_DST_IP_ADDR].ptr;
4191 		}
4192 		else if (r->match_tag && !pf_match_eth_tag(m, r, &tag,
4193 		    mtag ? mtag->tag : 0)) {
4194 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4195 			    "match_tag");
4196 			r = TAILQ_NEXT(r, entries);
4197 		}
4198 		else {
4199 			if (r->tag)
4200 				tag = r->tag;
4201 			if (r->anchor == NULL) {
4202 				/* Rule matches */
4203 				rm = r;
4204 
4205 				SDT_PROBE2(pf, eth, test_rule, match, r->nr, r);
4206 
4207 				if (r->quick)
4208 					break;
4209 
4210 				r = TAILQ_NEXT(r, entries);
4211 			} else {
4212 				pf_step_into_keth_anchor(anchor_stack, &asd,
4213 				    &ruleset, &r, &a, &match);
4214 			}
4215 		}
4216 		if (r == NULL && pf_step_out_of_keth_anchor(anchor_stack, &asd,
4217 		    &ruleset, &r, &a, &match))
4218 			break;
4219 	}
4220 
4221 	r = rm;
4222 
4223 	SDT_PROBE2(pf, eth, test_rule, final_match, (r != NULL ? r->nr : -1), r);
4224 
4225 	/* Default to pass. */
4226 	if (r == NULL) {
4227 		PF_RULES_RUNLOCK();
4228 		return (PF_PASS);
4229 	}
4230 
4231 	/* Execute action. */
4232 	counter_u64_add(r->packets[dir == PF_OUT], 1);
4233 	counter_u64_add(r->bytes[dir == PF_OUT], m_length(m, NULL));
4234 	pf_update_timestamp(r);
4235 
4236 	/* Shortcut. Don't tag if we're just going to drop anyway. */
4237 	if (r->action == PF_DROP) {
4238 		PF_RULES_RUNLOCK();
4239 		return (PF_DROP);
4240 	}
4241 
4242 	if (tag > 0) {
4243 		if (mtag == NULL)
4244 			mtag = pf_get_mtag(m);
4245 		if (mtag == NULL) {
4246 			PF_RULES_RUNLOCK();
4247 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4248 			return (PF_DROP);
4249 		}
4250 		mtag->tag = tag;
4251 	}
4252 
4253 	if (r->qid != 0) {
4254 		if (mtag == NULL)
4255 			mtag = pf_get_mtag(m);
4256 		if (mtag == NULL) {
4257 			PF_RULES_RUNLOCK();
4258 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4259 			return (PF_DROP);
4260 		}
4261 		mtag->qid = r->qid;
4262 	}
4263 
4264 	action = r->action;
4265 	bridge_to = r->bridge_to;
4266 
4267 	/* Dummynet */
4268 	if (r->dnpipe) {
4269 		struct ip_fw_args dnflow;
4270 
4271 		/* Drop packet if dummynet is not loaded. */
4272 		if (ip_dn_io_ptr == NULL) {
4273 			PF_RULES_RUNLOCK();
4274 			m_freem(m);
4275 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4276 			return (PF_DROP);
4277 		}
4278 		if (mtag == NULL)
4279 			mtag = pf_get_mtag(m);
4280 		if (mtag == NULL) {
4281 			PF_RULES_RUNLOCK();
4282 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4283 			return (PF_DROP);
4284 		}
4285 
4286 		bzero(&dnflow, sizeof(dnflow));
4287 
4288 		/* We don't have port numbers here, so we set 0.  That means
4289 		 * that we'll be somewhat limited in distinguishing flows (i.e.
4290 		 * only based on IP addresses, not based on port numbers), but
4291 		 * it's better than nothing. */
4292 		dnflow.f_id.dst_port = 0;
4293 		dnflow.f_id.src_port = 0;
4294 		dnflow.f_id.proto = 0;
4295 
4296 		dnflow.rule.info = r->dnpipe;
4297 		dnflow.rule.info |= IPFW_IS_DUMMYNET;
4298 		if (r->dnflags & PFRULE_DN_IS_PIPE)
4299 			dnflow.rule.info |= IPFW_IS_PIPE;
4300 
4301 		dnflow.f_id.extra = dnflow.rule.info;
4302 
4303 		dnflow.flags = dir == PF_IN ? IPFW_ARGS_IN : IPFW_ARGS_OUT;
4304 		dnflow.flags |= IPFW_ARGS_ETHER;
4305 		dnflow.ifp = kif->pfik_ifp;
4306 
4307 		switch (af) {
4308 		case AF_INET:
4309 			dnflow.f_id.addr_type = 4;
4310 			dnflow.f_id.src_ip = src->v4.s_addr;
4311 			dnflow.f_id.dst_ip = dst->v4.s_addr;
4312 			break;
4313 		case AF_INET6:
4314 			dnflow.flags |= IPFW_ARGS_IP6;
4315 			dnflow.f_id.addr_type = 6;
4316 			dnflow.f_id.src_ip6 = src->v6;
4317 			dnflow.f_id.dst_ip6 = dst->v6;
4318 			break;
4319 		}
4320 
4321 		PF_RULES_RUNLOCK();
4322 
4323 		mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
4324 		ip_dn_io_ptr(m0, &dnflow);
4325 		if (*m0 != NULL)
4326 			mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
4327 	} else {
4328 		PF_RULES_RUNLOCK();
4329 	}
4330 
4331 	if (action == PF_PASS && bridge_to) {
4332 		pf_bridge_to(bridge_to->pfik_ifp, *m0);
4333 		*m0 = NULL; /* We've eaten the packet. */
4334 	}
4335 
4336 	return (action);
4337 }
4338 
4339 static int
4340 pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, struct pfi_kkif *kif,
4341     struct mbuf *m, int off, struct pf_pdesc *pd, struct pf_krule **am,
4342     struct pf_kruleset **rsm, struct inpcb *inp)
4343 {
4344 	struct pf_krule		*nr = NULL;
4345 	struct pf_addr		* const saddr = pd->src;
4346 	struct pf_addr		* const daddr = pd->dst;
4347 	sa_family_t		 af = pd->af;
4348 	struct pf_krule		*r, *a = NULL;
4349 	struct pf_kruleset	*ruleset = NULL;
4350 	struct pf_krule_slist	 match_rules;
4351 	struct pf_krule_item	*ri;
4352 	struct pf_ksrc_node	*nsn = NULL;
4353 	struct tcphdr		*th = &pd->hdr.tcp;
4354 	struct pf_state_key	*sk = NULL, *nk = NULL;
4355 	u_short			 reason;
4356 	int			 rewrite = 0, hdrlen = 0;
4357 	int			 tag = -1;
4358 	int			 asd = 0;
4359 	int			 match = 0;
4360 	int			 state_icmp = 0;
4361 	u_int16_t		 sport = 0, dport = 0;
4362 	u_int16_t		 bproto_sum = 0, bip_sum = 0;
4363 	u_int8_t		 icmptype = 0, icmpcode = 0;
4364 	struct pf_kanchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
4365 
4366 	PF_RULES_RASSERT();
4367 
4368 	if (inp != NULL) {
4369 		INP_LOCK_ASSERT(inp);
4370 		pd->lookup.uid = inp->inp_cred->cr_uid;
4371 		pd->lookup.gid = inp->inp_cred->cr_groups[0];
4372 		pd->lookup.done = 1;
4373 	}
4374 
4375 	switch (pd->proto) {
4376 	case IPPROTO_TCP:
4377 		sport = th->th_sport;
4378 		dport = th->th_dport;
4379 		hdrlen = sizeof(*th);
4380 		break;
4381 	case IPPROTO_UDP:
4382 		sport = pd->hdr.udp.uh_sport;
4383 		dport = pd->hdr.udp.uh_dport;
4384 		hdrlen = sizeof(pd->hdr.udp);
4385 		break;
4386 	case IPPROTO_SCTP:
4387 		sport = pd->hdr.sctp.src_port;
4388 		dport = pd->hdr.sctp.dest_port;
4389 		hdrlen = sizeof(pd->hdr.sctp);
4390 		break;
4391 #ifdef INET
4392 	case IPPROTO_ICMP:
4393 		if (pd->af != AF_INET)
4394 			break;
4395 		sport = dport = pd->hdr.icmp.icmp_id;
4396 		hdrlen = sizeof(pd->hdr.icmp);
4397 		icmptype = pd->hdr.icmp.icmp_type;
4398 		icmpcode = pd->hdr.icmp.icmp_code;
4399 
4400 		if (icmptype == ICMP_UNREACH ||
4401 		    icmptype == ICMP_SOURCEQUENCH ||
4402 		    icmptype == ICMP_REDIRECT ||
4403 		    icmptype == ICMP_TIMXCEED ||
4404 		    icmptype == ICMP_PARAMPROB)
4405 			state_icmp++;
4406 		break;
4407 #endif /* INET */
4408 #ifdef INET6
4409 	case IPPROTO_ICMPV6:
4410 		if (af != AF_INET6)
4411 			break;
4412 		sport = dport = pd->hdr.icmp6.icmp6_id;
4413 		hdrlen = sizeof(pd->hdr.icmp6);
4414 		icmptype = pd->hdr.icmp6.icmp6_type;
4415 		icmpcode = pd->hdr.icmp6.icmp6_code;
4416 
4417 		if (icmptype == ICMP6_DST_UNREACH ||
4418 		    icmptype == ICMP6_PACKET_TOO_BIG ||
4419 		    icmptype == ICMP6_TIME_EXCEEDED ||
4420 		    icmptype == ICMP6_PARAM_PROB)
4421 			state_icmp++;
4422 		break;
4423 #endif /* INET6 */
4424 	default:
4425 		sport = dport = hdrlen = 0;
4426 		break;
4427 	}
4428 
4429 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
4430 
4431 	/* check packet for BINAT/NAT/RDR */
4432 	if ((nr = pf_get_translation(pd, m, off, kif, &nsn, &sk,
4433 	    &nk, saddr, daddr, sport, dport, anchor_stack)) != NULL) {
4434 		KASSERT(sk != NULL, ("%s: null sk", __func__));
4435 		KASSERT(nk != NULL, ("%s: null nk", __func__));
4436 
4437 		if (nr->log) {
4438 			PFLOG_PACKET(kif, m, af, PFRES_MATCH, nr, a,
4439 			    ruleset, pd, 1);
4440 		}
4441 
4442 		if (pd->ip_sum)
4443 			bip_sum = *pd->ip_sum;
4444 
4445 		switch (pd->proto) {
4446 		case IPPROTO_TCP:
4447 			bproto_sum = th->th_sum;
4448 			pd->proto_sum = &th->th_sum;
4449 
4450 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
4451 			    nk->port[pd->sidx] != sport) {
4452 				pf_change_ap(m, saddr, &th->th_sport, pd->ip_sum,
4453 				    &th->th_sum, &nk->addr[pd->sidx],
4454 				    nk->port[pd->sidx], 0, af);
4455 				pd->sport = &th->th_sport;
4456 				sport = th->th_sport;
4457 			}
4458 
4459 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
4460 			    nk->port[pd->didx] != dport) {
4461 				pf_change_ap(m, daddr, &th->th_dport, pd->ip_sum,
4462 				    &th->th_sum, &nk->addr[pd->didx],
4463 				    nk->port[pd->didx], 0, af);
4464 				dport = th->th_dport;
4465 				pd->dport = &th->th_dport;
4466 			}
4467 			rewrite++;
4468 			break;
4469 		case IPPROTO_UDP:
4470 			bproto_sum = pd->hdr.udp.uh_sum;
4471 			pd->proto_sum = &pd->hdr.udp.uh_sum;
4472 
4473 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
4474 			    nk->port[pd->sidx] != sport) {
4475 				pf_change_ap(m, saddr, &pd->hdr.udp.uh_sport,
4476 				    pd->ip_sum, &pd->hdr.udp.uh_sum,
4477 				    &nk->addr[pd->sidx],
4478 				    nk->port[pd->sidx], 1, af);
4479 				sport = pd->hdr.udp.uh_sport;
4480 				pd->sport = &pd->hdr.udp.uh_sport;
4481 			}
4482 
4483 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
4484 			    nk->port[pd->didx] != dport) {
4485 				pf_change_ap(m, daddr, &pd->hdr.udp.uh_dport,
4486 				    pd->ip_sum, &pd->hdr.udp.uh_sum,
4487 				    &nk->addr[pd->didx],
4488 				    nk->port[pd->didx], 1, af);
4489 				dport = pd->hdr.udp.uh_dport;
4490 				pd->dport = &pd->hdr.udp.uh_dport;
4491 			}
4492 			rewrite++;
4493 			break;
4494 		case IPPROTO_SCTP: {
4495 			uint16_t checksum = 0;
4496 
4497 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
4498 			    nk->port[pd->sidx] != sport) {
4499 				pf_change_ap(m, saddr, &pd->hdr.sctp.src_port,
4500 				    pd->ip_sum, &checksum,
4501 				    &nk->addr[pd->sidx],
4502 				    nk->port[pd->sidx], 1, af);
4503 			}
4504 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
4505 			    nk->port[pd->didx] != dport) {
4506 				pf_change_ap(m, daddr, &pd->hdr.sctp.dest_port,
4507 				    pd->ip_sum, &checksum,
4508 				    &nk->addr[pd->didx],
4509 				    nk->port[pd->didx], 1, af);
4510 			}
4511 			break;
4512 		}
4513 #ifdef INET
4514 		case IPPROTO_ICMP:
4515 			nk->port[0] = nk->port[1];
4516 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET))
4517 				pf_change_a(&saddr->v4.s_addr, pd->ip_sum,
4518 				    nk->addr[pd->sidx].v4.s_addr, 0);
4519 
4520 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET))
4521 				pf_change_a(&daddr->v4.s_addr, pd->ip_sum,
4522 				    nk->addr[pd->didx].v4.s_addr, 0);
4523 
4524 			if (nk->port[1] != pd->hdr.icmp.icmp_id) {
4525 				pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
4526 				    pd->hdr.icmp.icmp_cksum, sport,
4527 				    nk->port[1], 0);
4528 				pd->hdr.icmp.icmp_id = nk->port[1];
4529 				pd->sport = &pd->hdr.icmp.icmp_id;
4530 			}
4531 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
4532 			break;
4533 #endif /* INET */
4534 #ifdef INET6
4535 		case IPPROTO_ICMPV6:
4536 			nk->port[0] = nk->port[1];
4537 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET6))
4538 				pf_change_a6(saddr, &pd->hdr.icmp6.icmp6_cksum,
4539 				    &nk->addr[pd->sidx], 0);
4540 
4541 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET6))
4542 				pf_change_a6(daddr, &pd->hdr.icmp6.icmp6_cksum,
4543 				    &nk->addr[pd->didx], 0);
4544 			rewrite++;
4545 			break;
4546 #endif /* INET */
4547 		default:
4548 			switch (af) {
4549 #ifdef INET
4550 			case AF_INET:
4551 				if (PF_ANEQ(saddr,
4552 				    &nk->addr[pd->sidx], AF_INET))
4553 					pf_change_a(&saddr->v4.s_addr,
4554 					    pd->ip_sum,
4555 					    nk->addr[pd->sidx].v4.s_addr, 0);
4556 
4557 				if (PF_ANEQ(daddr,
4558 				    &nk->addr[pd->didx], AF_INET))
4559 					pf_change_a(&daddr->v4.s_addr,
4560 					    pd->ip_sum,
4561 					    nk->addr[pd->didx].v4.s_addr, 0);
4562 				break;
4563 #endif /* INET */
4564 #ifdef INET6
4565 			case AF_INET6:
4566 				if (PF_ANEQ(saddr,
4567 				    &nk->addr[pd->sidx], AF_INET6))
4568 					PF_ACPY(saddr, &nk->addr[pd->sidx], af);
4569 
4570 				if (PF_ANEQ(daddr,
4571 				    &nk->addr[pd->didx], AF_INET6))
4572 					PF_ACPY(daddr, &nk->addr[pd->didx], af);
4573 				break;
4574 #endif /* INET */
4575 			}
4576 			break;
4577 		}
4578 		if (nr->natpass)
4579 			r = NULL;
4580 		pd->nat_rule = nr;
4581 	}
4582 
4583 	SLIST_INIT(&match_rules);
4584 	while (r != NULL) {
4585 		pf_counter_u64_add(&r->evaluations, 1);
4586 		if (pfi_kkif_match(r->kif, kif) == r->ifnot)
4587 			r = r->skip[PF_SKIP_IFP].ptr;
4588 		else if (r->direction && r->direction != pd->dir)
4589 			r = r->skip[PF_SKIP_DIR].ptr;
4590 		else if (r->af && r->af != af)
4591 			r = r->skip[PF_SKIP_AF].ptr;
4592 		else if (r->proto && r->proto != pd->proto)
4593 			r = r->skip[PF_SKIP_PROTO].ptr;
4594 		else if (PF_MISMATCHAW(&r->src.addr, saddr, af,
4595 		    r->src.neg, kif, M_GETFIB(m)))
4596 			r = r->skip[PF_SKIP_SRC_ADDR].ptr;
4597 		/* tcp/udp only. port_op always 0 in other cases */
4598 		else if (r->src.port_op && !pf_match_port(r->src.port_op,
4599 		    r->src.port[0], r->src.port[1], sport))
4600 			r = r->skip[PF_SKIP_SRC_PORT].ptr;
4601 		else if (PF_MISMATCHAW(&r->dst.addr, daddr, af,
4602 		    r->dst.neg, NULL, M_GETFIB(m)))
4603 			r = r->skip[PF_SKIP_DST_ADDR].ptr;
4604 		/* tcp/udp only. port_op always 0 in other cases */
4605 		else if (r->dst.port_op && !pf_match_port(r->dst.port_op,
4606 		    r->dst.port[0], r->dst.port[1], dport))
4607 			r = r->skip[PF_SKIP_DST_PORT].ptr;
4608 		/* icmp only. type always 0 in other cases */
4609 		else if (r->type && r->type != icmptype + 1)
4610 			r = TAILQ_NEXT(r, entries);
4611 		/* icmp only. type always 0 in other cases */
4612 		else if (r->code && r->code != icmpcode + 1)
4613 			r = TAILQ_NEXT(r, entries);
4614 		else if (r->tos && !(r->tos == pd->tos))
4615 			r = TAILQ_NEXT(r, entries);
4616 		else if (r->rule_flag & PFRULE_FRAGMENT)
4617 			r = TAILQ_NEXT(r, entries);
4618 		else if (pd->proto == IPPROTO_TCP &&
4619 		    (r->flagset & th->th_flags) != r->flags)
4620 			r = TAILQ_NEXT(r, entries);
4621 		/* tcp/udp only. uid.op always 0 in other cases */
4622 		else if (r->uid.op && (pd->lookup.done || (pd->lookup.done =
4623 		    pf_socket_lookup(pd, m), 1)) &&
4624 		    !pf_match_uid(r->uid.op, r->uid.uid[0], r->uid.uid[1],
4625 		    pd->lookup.uid))
4626 			r = TAILQ_NEXT(r, entries);
4627 		/* tcp/udp only. gid.op always 0 in other cases */
4628 		else if (r->gid.op && (pd->lookup.done || (pd->lookup.done =
4629 		    pf_socket_lookup(pd, m), 1)) &&
4630 		    !pf_match_gid(r->gid.op, r->gid.gid[0], r->gid.gid[1],
4631 		    pd->lookup.gid))
4632 			r = TAILQ_NEXT(r, entries);
4633 		else if (r->prio &&
4634 		    !pf_match_ieee8021q_pcp(r->prio, m))
4635 			r = TAILQ_NEXT(r, entries);
4636 		else if (r->prob &&
4637 		    r->prob <= arc4random())
4638 			r = TAILQ_NEXT(r, entries);
4639 		else if (r->match_tag && !pf_match_tag(m, r, &tag,
4640 		    pd->pf_mtag ? pd->pf_mtag->tag : 0))
4641 			r = TAILQ_NEXT(r, entries);
4642 		else if (r->os_fingerprint != PF_OSFP_ANY &&
4643 		    (pd->proto != IPPROTO_TCP || !pf_osfp_match(
4644 		    pf_osfp_fingerprint(pd, m, off, th),
4645 		    r->os_fingerprint)))
4646 			r = TAILQ_NEXT(r, entries);
4647 		else {
4648 			if (r->tag)
4649 				tag = r->tag;
4650 			if (r->anchor == NULL) {
4651 				if (r->action == PF_MATCH) {
4652 					ri = malloc(sizeof(struct pf_krule_item), M_PF_RULE_ITEM, M_NOWAIT | M_ZERO);
4653 					if (ri == NULL) {
4654 						REASON_SET(&reason, PFRES_MEMORY);
4655 						goto cleanup;
4656 					}
4657 					ri->r = r;
4658 					SLIST_INSERT_HEAD(&match_rules, ri, entry);
4659 					pf_counter_u64_critical_enter();
4660 					pf_counter_u64_add_protected(&r->packets[pd->dir == PF_OUT], 1);
4661 					pf_counter_u64_add_protected(&r->bytes[pd->dir == PF_OUT], pd->tot_len);
4662 					pf_counter_u64_critical_exit();
4663 					pf_rule_to_actions(r, &pd->act);
4664 					if (r->log)
4665 						PFLOG_PACKET(kif, m, af,
4666 						    PFRES_MATCH, r,
4667 						    a, ruleset, pd, 1);
4668 				} else {
4669 					match = 1;
4670 					*rm = r;
4671 					*am = a;
4672 					*rsm = ruleset;
4673 				}
4674 				if ((*rm)->quick)
4675 					break;
4676 				r = TAILQ_NEXT(r, entries);
4677 			} else
4678 				pf_step_into_anchor(anchor_stack, &asd,
4679 				    &ruleset, PF_RULESET_FILTER, &r, &a,
4680 				    &match);
4681 		}
4682 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
4683 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
4684 			break;
4685 	}
4686 	r = *rm;
4687 	a = *am;
4688 	ruleset = *rsm;
4689 
4690 	REASON_SET(&reason, PFRES_MATCH);
4691 
4692 	/* apply actions for last matching pass/block rule */
4693 	pf_rule_to_actions(r, &pd->act);
4694 
4695 	if (r->log) {
4696 		if (rewrite)
4697 			m_copyback(m, off, hdrlen, pd->hdr.any);
4698 		PFLOG_PACKET(kif, m, af, reason, r, a, ruleset, pd, 1);
4699 	}
4700 
4701 	if ((r->action == PF_DROP) &&
4702 	    ((r->rule_flag & PFRULE_RETURNRST) ||
4703 	    (r->rule_flag & PFRULE_RETURNICMP) ||
4704 	    (r->rule_flag & PFRULE_RETURN))) {
4705 		pf_return(r, nr, pd, sk, off, m, th, kif, bproto_sum,
4706 		    bip_sum, hdrlen, &reason, r->rtableid);
4707 	}
4708 
4709 	if (r->action == PF_DROP)
4710 		goto cleanup;
4711 
4712 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
4713 		REASON_SET(&reason, PFRES_MEMORY);
4714 		goto cleanup;
4715 	}
4716 	if (pd->act.rtableid >= 0)
4717 		M_SETFIB(m, pd->act.rtableid);
4718 
4719 	if (!state_icmp && (r->keep_state || nr != NULL ||
4720 	    (pd->flags & PFDESC_TCP_NORM))) {
4721 		int action;
4722 		action = pf_create_state(r, nr, a, pd, nsn, nk, sk, m, off,
4723 		    sport, dport, &rewrite, kif, sm, tag, bproto_sum, bip_sum,
4724 		    hdrlen, &match_rules);
4725 		if (action != PF_PASS) {
4726 			if (action == PF_DROP &&
4727 			    (r->rule_flag & PFRULE_RETURN))
4728 				pf_return(r, nr, pd, sk, off, m, th, kif,
4729 				    bproto_sum, bip_sum, hdrlen, &reason,
4730 				    pd->act.rtableid);
4731 			return (action);
4732 		}
4733 	} else {
4734 		uma_zfree(V_pf_state_key_z, sk);
4735 		uma_zfree(V_pf_state_key_z, nk);
4736 	}
4737 
4738 	/* copy back packet headers if we performed NAT operations */
4739 	if (rewrite)
4740 		m_copyback(m, off, hdrlen, pd->hdr.any);
4741 
4742 	if (*sm != NULL && !((*sm)->state_flags & PFSTATE_NOSYNC) &&
4743 	    pd->dir == PF_OUT &&
4744 	    V_pfsync_defer_ptr != NULL && V_pfsync_defer_ptr(*sm, m))
4745 		/*
4746 		 * We want the state created, but we dont
4747 		 * want to send this in case a partner
4748 		 * firewall has to know about it to allow
4749 		 * replies through it.
4750 		 */
4751 		return (PF_DEFER);
4752 
4753 	return (PF_PASS);
4754 
4755 cleanup:
4756 	while ((ri = SLIST_FIRST(&match_rules))) {
4757 		SLIST_REMOVE_HEAD(&match_rules, entry);
4758 		free(ri, M_PF_RULE_ITEM);
4759 	}
4760 
4761 	uma_zfree(V_pf_state_key_z, sk);
4762 	uma_zfree(V_pf_state_key_z, nk);
4763 	return (PF_DROP);
4764 }
4765 
4766 static int
4767 pf_create_state(struct pf_krule *r, struct pf_krule *nr, struct pf_krule *a,
4768     struct pf_pdesc *pd, struct pf_ksrc_node *nsn, struct pf_state_key *nk,
4769     struct pf_state_key *sk, struct mbuf *m, int off, u_int16_t sport,
4770     u_int16_t dport, int *rewrite, struct pfi_kkif *kif, struct pf_kstate **sm,
4771     int tag, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen,
4772     struct pf_krule_slist *match_rules)
4773 {
4774 	struct pf_kstate	*s = NULL;
4775 	struct pf_ksrc_node	*sn = NULL;
4776 	struct tcphdr		*th = &pd->hdr.tcp;
4777 	u_int16_t		 mss = V_tcp_mssdflt;
4778 	u_short			 reason, sn_reason;
4779 
4780 	/* check maximums */
4781 	if (r->max_states &&
4782 	    (counter_u64_fetch(r->states_cur) >= r->max_states)) {
4783 		counter_u64_add(V_pf_status.lcounters[LCNT_STATES], 1);
4784 		REASON_SET(&reason, PFRES_MAXSTATES);
4785 		goto csfailed;
4786 	}
4787 	/* src node for filter rule */
4788 	if ((r->rule_flag & PFRULE_SRCTRACK ||
4789 	    r->rpool.opts & PF_POOL_STICKYADDR) &&
4790 	    (sn_reason = pf_insert_src_node(&sn, r, pd->src, pd->af)) != 0) {
4791 		REASON_SET(&reason, sn_reason);
4792 		goto csfailed;
4793 	}
4794 	/* src node for translation rule */
4795 	if (nr != NULL && (nr->rpool.opts & PF_POOL_STICKYADDR) &&
4796 	    (sn_reason = pf_insert_src_node(&nsn, nr, &sk->addr[pd->sidx],
4797 	    pd->af)) != 0 ) {
4798 		REASON_SET(&reason, sn_reason);
4799 		goto csfailed;
4800 	}
4801 	s = pf_alloc_state(M_NOWAIT);
4802 	if (s == NULL) {
4803 		REASON_SET(&reason, PFRES_MEMORY);
4804 		goto csfailed;
4805 	}
4806 	s->rule.ptr = r;
4807 	s->nat_rule.ptr = nr;
4808 	s->anchor.ptr = a;
4809 	bcopy(match_rules, &s->match_rules, sizeof(s->match_rules));
4810 	memcpy(&s->act, &pd->act, sizeof(struct pf_rule_actions));
4811 
4812 	STATE_INC_COUNTERS(s);
4813 	if (r->allow_opts)
4814 		s->state_flags |= PFSTATE_ALLOWOPTS;
4815 	if (r->rule_flag & PFRULE_STATESLOPPY)
4816 		s->state_flags |= PFSTATE_SLOPPY;
4817 	if (pd->flags & PFDESC_TCP_NORM) /* Set by old-style scrub rules */
4818 		s->state_flags |= PFSTATE_SCRUB_TCP;
4819 
4820 	s->act.log = pd->act.log & PF_LOG_ALL;
4821 	s->sync_state = PFSYNC_S_NONE;
4822 	s->state_flags |= pd->act.flags; /* Only needed for pfsync and state export */
4823 
4824 	if (nr != NULL)
4825 		s->act.log |= nr->log & PF_LOG_ALL;
4826 	switch (pd->proto) {
4827 	case IPPROTO_TCP:
4828 		s->src.seqlo = ntohl(th->th_seq);
4829 		s->src.seqhi = s->src.seqlo + pd->p_len + 1;
4830 		if ((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN &&
4831 		    r->keep_state == PF_STATE_MODULATE) {
4832 			/* Generate sequence number modulator */
4833 			if ((s->src.seqdiff = pf_tcp_iss(pd) - s->src.seqlo) ==
4834 			    0)
4835 				s->src.seqdiff = 1;
4836 			pf_change_proto_a(m, &th->th_seq, &th->th_sum,
4837 			    htonl(s->src.seqlo + s->src.seqdiff), 0);
4838 			*rewrite = 1;
4839 		} else
4840 			s->src.seqdiff = 0;
4841 		if (th->th_flags & TH_SYN) {
4842 			s->src.seqhi++;
4843 			s->src.wscale = pf_get_wscale(m, off,
4844 			    th->th_off, pd->af);
4845 		}
4846 		s->src.max_win = MAX(ntohs(th->th_win), 1);
4847 		if (s->src.wscale & PF_WSCALE_MASK) {
4848 			/* Remove scale factor from initial window */
4849 			int win = s->src.max_win;
4850 			win += 1 << (s->src.wscale & PF_WSCALE_MASK);
4851 			s->src.max_win = (win - 1) >>
4852 			    (s->src.wscale & PF_WSCALE_MASK);
4853 		}
4854 		if (th->th_flags & TH_FIN)
4855 			s->src.seqhi++;
4856 		s->dst.seqhi = 1;
4857 		s->dst.max_win = 1;
4858 		pf_set_protostate(s, PF_PEER_SRC, TCPS_SYN_SENT);
4859 		pf_set_protostate(s, PF_PEER_DST, TCPS_CLOSED);
4860 		s->timeout = PFTM_TCP_FIRST_PACKET;
4861 		atomic_add_32(&V_pf_status.states_halfopen, 1);
4862 		break;
4863 	case IPPROTO_UDP:
4864 		pf_set_protostate(s, PF_PEER_SRC, PFUDPS_SINGLE);
4865 		pf_set_protostate(s, PF_PEER_DST, PFUDPS_NO_TRAFFIC);
4866 		s->timeout = PFTM_UDP_FIRST_PACKET;
4867 		break;
4868 	case IPPROTO_SCTP:
4869 		pf_set_protostate(s, PF_PEER_SRC, SCTP_COOKIE_WAIT);
4870 		pf_set_protostate(s, PF_PEER_DST, SCTP_CLOSED);
4871 		s->timeout = PFTM_TCP_FIRST_PACKET;
4872 		break;
4873 	case IPPROTO_ICMP:
4874 #ifdef INET6
4875 	case IPPROTO_ICMPV6:
4876 #endif
4877 		s->timeout = PFTM_ICMP_FIRST_PACKET;
4878 		break;
4879 	default:
4880 		pf_set_protostate(s, PF_PEER_SRC, PFOTHERS_SINGLE);
4881 		pf_set_protostate(s, PF_PEER_DST, PFOTHERS_NO_TRAFFIC);
4882 		s->timeout = PFTM_OTHER_FIRST_PACKET;
4883 	}
4884 
4885 	if (r->rt) {
4886 		/* pf_map_addr increases the reason counters */
4887 		if ((reason = pf_map_addr(pd->af, r, pd->src, &s->rt_addr,
4888 		    &s->rt_kif, NULL, &sn)) != 0)
4889 			goto csfailed;
4890 		s->rt = r->rt;
4891 	}
4892 
4893 	s->creation = time_uptime;
4894 	s->expire = time_uptime;
4895 
4896 	if (sn != NULL)
4897 		s->src_node = sn;
4898 	if (nsn != NULL) {
4899 		/* XXX We only modify one side for now. */
4900 		PF_ACPY(&nsn->raddr, &nk->addr[1], pd->af);
4901 		s->nat_src_node = nsn;
4902 	}
4903 	if (pd->proto == IPPROTO_TCP) {
4904 		if (s->state_flags & PFSTATE_SCRUB_TCP &&
4905 		    pf_normalize_tcp_init(m, off, pd, th, &s->src, &s->dst)) {
4906 			REASON_SET(&reason, PFRES_MEMORY);
4907 			goto drop;
4908 		}
4909 		if (s->state_flags & PFSTATE_SCRUB_TCP && s->src.scrub &&
4910 		    pf_normalize_tcp_stateful(m, off, pd, &reason, th, s,
4911 		    &s->src, &s->dst, rewrite)) {
4912 			/* This really shouldn't happen!!! */
4913 			DPFPRINTF(PF_DEBUG_URGENT,
4914 			    ("pf_normalize_tcp_stateful failed on first "
4915 			     "pkt\n"));
4916 			goto drop;
4917 		}
4918 	} else if (pd->proto == IPPROTO_SCTP) {
4919 		if (pf_normalize_sctp_init(m, off, pd, &s->src, &s->dst))
4920 			goto drop;
4921 		if (! (pd->sctp_flags & (PFDESC_SCTP_INIT | PFDESC_SCTP_ADD_IP)))
4922 			goto drop;
4923 	}
4924 	s->direction = pd->dir;
4925 
4926 	/*
4927 	 * sk/nk could already been setup by pf_get_translation().
4928 	 */
4929 	if (nr == NULL) {
4930 		KASSERT((sk == NULL && nk == NULL), ("%s: nr %p sk %p, nk %p",
4931 		    __func__, nr, sk, nk));
4932 		sk = pf_state_key_setup(pd, pd->src, pd->dst, sport, dport);
4933 		if (sk == NULL)
4934 			goto csfailed;
4935 		nk = sk;
4936 	} else
4937 		KASSERT((sk != NULL && nk != NULL), ("%s: nr %p sk %p, nk %p",
4938 		    __func__, nr, sk, nk));
4939 
4940 	/* Swap sk/nk for PF_OUT. */
4941 	if (pf_state_insert(BOUND_IFACE(r, kif), kif,
4942 	    (pd->dir == PF_IN) ? sk : nk,
4943 	    (pd->dir == PF_IN) ? nk : sk, s)) {
4944 		REASON_SET(&reason, PFRES_STATEINS);
4945 		goto drop;
4946 	} else
4947 		*sm = s;
4948 
4949 	if (tag > 0)
4950 		s->tag = tag;
4951 	if (pd->proto == IPPROTO_TCP && (th->th_flags & (TH_SYN|TH_ACK)) ==
4952 	    TH_SYN && r->keep_state == PF_STATE_SYNPROXY) {
4953 		pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_SRC);
4954 		/* undo NAT changes, if they have taken place */
4955 		if (nr != NULL) {
4956 			struct pf_state_key *skt = s->key[PF_SK_WIRE];
4957 			if (pd->dir == PF_OUT)
4958 				skt = s->key[PF_SK_STACK];
4959 			PF_ACPY(pd->src, &skt->addr[pd->sidx], pd->af);
4960 			PF_ACPY(pd->dst, &skt->addr[pd->didx], pd->af);
4961 			if (pd->sport)
4962 				*pd->sport = skt->port[pd->sidx];
4963 			if (pd->dport)
4964 				*pd->dport = skt->port[pd->didx];
4965 			if (pd->proto_sum)
4966 				*pd->proto_sum = bproto_sum;
4967 			if (pd->ip_sum)
4968 				*pd->ip_sum = bip_sum;
4969 			m_copyback(m, off, hdrlen, pd->hdr.any);
4970 		}
4971 		s->src.seqhi = htonl(arc4random());
4972 		/* Find mss option */
4973 		int rtid = M_GETFIB(m);
4974 		mss = pf_get_mss(m, off, th->th_off, pd->af);
4975 		mss = pf_calc_mss(pd->src, pd->af, rtid, mss);
4976 		mss = pf_calc_mss(pd->dst, pd->af, rtid, mss);
4977 		s->src.mss = mss;
4978 		pf_send_tcp(r, pd->af, pd->dst, pd->src, th->th_dport,
4979 		    th->th_sport, s->src.seqhi, ntohl(th->th_seq) + 1,
4980 		    TH_SYN|TH_ACK, 0, s->src.mss, 0, true, 0, 0,
4981 		    pd->act.rtableid);
4982 		REASON_SET(&reason, PFRES_SYNPROXY);
4983 		return (PF_SYNPROXY_DROP);
4984 	}
4985 
4986 	return (PF_PASS);
4987 
4988 csfailed:
4989 	uma_zfree(V_pf_state_key_z, sk);
4990 	uma_zfree(V_pf_state_key_z, nk);
4991 
4992 	if (sn != NULL) {
4993 		PF_SRC_NODE_LOCK(sn);
4994 		if (--sn->states == 0 && sn->expire == 0) {
4995 			pf_unlink_src_node(sn);
4996 			uma_zfree(V_pf_sources_z, sn);
4997 			counter_u64_add(
4998 			    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
4999 		}
5000 		PF_SRC_NODE_UNLOCK(sn);
5001 	}
5002 
5003 	if (nsn != sn && nsn != NULL) {
5004 		PF_SRC_NODE_LOCK(nsn);
5005 		if (--nsn->states == 0 && nsn->expire == 0) {
5006 			pf_unlink_src_node(nsn);
5007 			uma_zfree(V_pf_sources_z, nsn);
5008 			counter_u64_add(
5009 			    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
5010 		}
5011 		PF_SRC_NODE_UNLOCK(nsn);
5012 	}
5013 
5014 drop:
5015 	if (s != NULL) {
5016 		pf_src_tree_remove_state(s);
5017 		s->timeout = PFTM_UNLINKED;
5018 		STATE_DEC_COUNTERS(s);
5019 		pf_free_state(s);
5020 	}
5021 
5022 	return (PF_DROP);
5023 }
5024 
5025 static int
5026 pf_test_fragment(struct pf_krule **rm, struct pfi_kkif *kif,
5027     struct mbuf *m, void *h, struct pf_pdesc *pd, struct pf_krule **am,
5028     struct pf_kruleset **rsm)
5029 {
5030 	struct pf_krule		*r, *a = NULL;
5031 	struct pf_kruleset	*ruleset = NULL;
5032 	struct pf_krule_slist	 match_rules;
5033 	struct pf_krule_item	*ri;
5034 	sa_family_t		 af = pd->af;
5035 	u_short			 reason;
5036 	int			 tag = -1;
5037 	int			 asd = 0;
5038 	int			 match = 0;
5039 	struct pf_kanchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
5040 
5041 	PF_RULES_RASSERT();
5042 
5043 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
5044 	SLIST_INIT(&match_rules);
5045 	while (r != NULL) {
5046 		pf_counter_u64_add(&r->evaluations, 1);
5047 		if (pfi_kkif_match(r->kif, kif) == r->ifnot)
5048 			r = r->skip[PF_SKIP_IFP].ptr;
5049 		else if (r->direction && r->direction != pd->dir)
5050 			r = r->skip[PF_SKIP_DIR].ptr;
5051 		else if (r->af && r->af != af)
5052 			r = r->skip[PF_SKIP_AF].ptr;
5053 		else if (r->proto && r->proto != pd->proto)
5054 			r = r->skip[PF_SKIP_PROTO].ptr;
5055 		else if (PF_MISMATCHAW(&r->src.addr, pd->src, af,
5056 		    r->src.neg, kif, M_GETFIB(m)))
5057 			r = r->skip[PF_SKIP_SRC_ADDR].ptr;
5058 		else if (PF_MISMATCHAW(&r->dst.addr, pd->dst, af,
5059 		    r->dst.neg, NULL, M_GETFIB(m)))
5060 			r = r->skip[PF_SKIP_DST_ADDR].ptr;
5061 		else if (r->tos && !(r->tos == pd->tos))
5062 			r = TAILQ_NEXT(r, entries);
5063 		else if (r->os_fingerprint != PF_OSFP_ANY)
5064 			r = TAILQ_NEXT(r, entries);
5065 		else if (pd->proto == IPPROTO_UDP &&
5066 		    (r->src.port_op || r->dst.port_op))
5067 			r = TAILQ_NEXT(r, entries);
5068 		else if (pd->proto == IPPROTO_TCP &&
5069 		    (r->src.port_op || r->dst.port_op || r->flagset))
5070 			r = TAILQ_NEXT(r, entries);
5071 		else if ((pd->proto == IPPROTO_ICMP ||
5072 		    pd->proto == IPPROTO_ICMPV6) &&
5073 		    (r->type || r->code))
5074 			r = TAILQ_NEXT(r, entries);
5075 		else if (r->prio &&
5076 		    !pf_match_ieee8021q_pcp(r->prio, m))
5077 			r = TAILQ_NEXT(r, entries);
5078 		else if (r->prob && r->prob <=
5079 		    (arc4random() % (UINT_MAX - 1) + 1))
5080 			r = TAILQ_NEXT(r, entries);
5081 		else if (r->match_tag && !pf_match_tag(m, r, &tag,
5082 		    pd->pf_mtag ? pd->pf_mtag->tag : 0))
5083 			r = TAILQ_NEXT(r, entries);
5084 		else {
5085 			if (r->anchor == NULL) {
5086 				if (r->action == PF_MATCH) {
5087 					ri = malloc(sizeof(struct pf_krule_item), M_PF_RULE_ITEM, M_NOWAIT | M_ZERO);
5088 					if (ri == NULL) {
5089 						REASON_SET(&reason, PFRES_MEMORY);
5090 						goto cleanup;
5091 					}
5092 					ri->r = r;
5093 					SLIST_INSERT_HEAD(&match_rules, ri, entry);
5094 					pf_counter_u64_critical_enter();
5095 					pf_counter_u64_add_protected(&r->packets[pd->dir == PF_OUT], 1);
5096 					pf_counter_u64_add_protected(&r->bytes[pd->dir == PF_OUT], pd->tot_len);
5097 					pf_counter_u64_critical_exit();
5098 					pf_rule_to_actions(r, &pd->act);
5099 					if (r->log)
5100 						PFLOG_PACKET(kif, m, af,
5101 						    PFRES_MATCH, r,
5102 						    a, ruleset, pd, 1);
5103 				} else {
5104 					match = 1;
5105 					*rm = r;
5106 					*am = a;
5107 					*rsm = ruleset;
5108 				}
5109 				if ((*rm)->quick)
5110 					break;
5111 				r = TAILQ_NEXT(r, entries);
5112 			} else
5113 				pf_step_into_anchor(anchor_stack, &asd,
5114 				    &ruleset, PF_RULESET_FILTER, &r, &a,
5115 				    &match);
5116 		}
5117 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
5118 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
5119 			break;
5120 	}
5121 	r = *rm;
5122 	a = *am;
5123 	ruleset = *rsm;
5124 
5125 	REASON_SET(&reason, PFRES_MATCH);
5126 
5127 	/* apply actions for last matching pass/block rule */
5128 	pf_rule_to_actions(r, &pd->act);
5129 
5130 	if (r->log)
5131 		PFLOG_PACKET(kif, m, af, reason, r, a, ruleset, pd, 1);
5132 
5133 	if (r->action != PF_PASS)
5134 		return (PF_DROP);
5135 
5136 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
5137 		REASON_SET(&reason, PFRES_MEMORY);
5138 		goto cleanup;
5139 	}
5140 
5141 	return (PF_PASS);
5142 
5143 cleanup:
5144 	while ((ri = SLIST_FIRST(&match_rules))) {
5145 		SLIST_REMOVE_HEAD(&match_rules, entry);
5146 		free(ri, M_PF_RULE_ITEM);
5147 	}
5148 
5149 	return (PF_DROP);
5150 }
5151 
5152 static int
5153 pf_tcp_track_full(struct pf_kstate **state, struct pfi_kkif *kif,
5154     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason,
5155     int *copyback)
5156 {
5157 	struct tcphdr		*th = &pd->hdr.tcp;
5158 	struct pf_state_peer	*src, *dst;
5159 	u_int16_t		 win = ntohs(th->th_win);
5160 	u_int32_t		 ack, end, seq, orig_seq;
5161 	u_int8_t		 sws, dws, psrc, pdst;
5162 	int			 ackskew;
5163 
5164 	if (pd->dir == (*state)->direction) {
5165 		src = &(*state)->src;
5166 		dst = &(*state)->dst;
5167 		psrc = PF_PEER_SRC;
5168 		pdst = PF_PEER_DST;
5169 	} else {
5170 		src = &(*state)->dst;
5171 		dst = &(*state)->src;
5172 		psrc = PF_PEER_DST;
5173 		pdst = PF_PEER_SRC;
5174 	}
5175 
5176 	if (src->wscale && dst->wscale && !(th->th_flags & TH_SYN)) {
5177 		sws = src->wscale & PF_WSCALE_MASK;
5178 		dws = dst->wscale & PF_WSCALE_MASK;
5179 	} else
5180 		sws = dws = 0;
5181 
5182 	/*
5183 	 * Sequence tracking algorithm from Guido van Rooij's paper:
5184 	 *   http://www.madison-gurkha.com/publications/tcp_filtering/
5185 	 *	tcp_filtering.ps
5186 	 */
5187 
5188 	orig_seq = seq = ntohl(th->th_seq);
5189 	if (src->seqlo == 0) {
5190 		/* First packet from this end. Set its state */
5191 
5192 		if (((*state)->state_flags & PFSTATE_SCRUB_TCP || dst->scrub) &&
5193 		    src->scrub == NULL) {
5194 			if (pf_normalize_tcp_init(m, off, pd, th, src, dst)) {
5195 				REASON_SET(reason, PFRES_MEMORY);
5196 				return (PF_DROP);
5197 			}
5198 		}
5199 
5200 		/* Deferred generation of sequence number modulator */
5201 		if (dst->seqdiff && !src->seqdiff) {
5202 			/* use random iss for the TCP server */
5203 			while ((src->seqdiff = arc4random() - seq) == 0)
5204 				;
5205 			ack = ntohl(th->th_ack) - dst->seqdiff;
5206 			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
5207 			    src->seqdiff), 0);
5208 			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
5209 			*copyback = 1;
5210 		} else {
5211 			ack = ntohl(th->th_ack);
5212 		}
5213 
5214 		end = seq + pd->p_len;
5215 		if (th->th_flags & TH_SYN) {
5216 			end++;
5217 			if (dst->wscale & PF_WSCALE_FLAG) {
5218 				src->wscale = pf_get_wscale(m, off, th->th_off,
5219 				    pd->af);
5220 				if (src->wscale & PF_WSCALE_FLAG) {
5221 					/* Remove scale factor from initial
5222 					 * window */
5223 					sws = src->wscale & PF_WSCALE_MASK;
5224 					win = ((u_int32_t)win + (1 << sws) - 1)
5225 					    >> sws;
5226 					dws = dst->wscale & PF_WSCALE_MASK;
5227 				} else {
5228 					/* fixup other window */
5229 					dst->max_win <<= dst->wscale &
5230 					    PF_WSCALE_MASK;
5231 					/* in case of a retrans SYN|ACK */
5232 					dst->wscale = 0;
5233 				}
5234 			}
5235 		}
5236 		if (th->th_flags & TH_FIN)
5237 			end++;
5238 
5239 		src->seqlo = seq;
5240 		if (src->state < TCPS_SYN_SENT)
5241 			pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5242 
5243 		/*
5244 		 * May need to slide the window (seqhi may have been set by
5245 		 * the crappy stack check or if we picked up the connection
5246 		 * after establishment)
5247 		 */
5248 		if (src->seqhi == 1 ||
5249 		    SEQ_GEQ(end + MAX(1, dst->max_win << dws), src->seqhi))
5250 			src->seqhi = end + MAX(1, dst->max_win << dws);
5251 		if (win > src->max_win)
5252 			src->max_win = win;
5253 
5254 	} else {
5255 		ack = ntohl(th->th_ack) - dst->seqdiff;
5256 		if (src->seqdiff) {
5257 			/* Modulate sequence numbers */
5258 			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
5259 			    src->seqdiff), 0);
5260 			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
5261 			*copyback = 1;
5262 		}
5263 		end = seq + pd->p_len;
5264 		if (th->th_flags & TH_SYN)
5265 			end++;
5266 		if (th->th_flags & TH_FIN)
5267 			end++;
5268 	}
5269 
5270 	if ((th->th_flags & TH_ACK) == 0) {
5271 		/* Let it pass through the ack skew check */
5272 		ack = dst->seqlo;
5273 	} else if ((ack == 0 &&
5274 	    (th->th_flags & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) ||
5275 	    /* broken tcp stacks do not set ack */
5276 	    (dst->state < TCPS_SYN_SENT)) {
5277 		/*
5278 		 * Many stacks (ours included) will set the ACK number in an
5279 		 * FIN|ACK if the SYN times out -- no sequence to ACK.
5280 		 */
5281 		ack = dst->seqlo;
5282 	}
5283 
5284 	if (seq == end) {
5285 		/* Ease sequencing restrictions on no data packets */
5286 		seq = src->seqlo;
5287 		end = seq;
5288 	}
5289 
5290 	ackskew = dst->seqlo - ack;
5291 
5292 	/*
5293 	 * Need to demodulate the sequence numbers in any TCP SACK options
5294 	 * (Selective ACK). We could optionally validate the SACK values
5295 	 * against the current ACK window, either forwards or backwards, but
5296 	 * I'm not confident that SACK has been implemented properly
5297 	 * everywhere. It wouldn't surprise me if several stacks accidentally
5298 	 * SACK too far backwards of previously ACKed data. There really aren't
5299 	 * any security implications of bad SACKing unless the target stack
5300 	 * doesn't validate the option length correctly. Someone trying to
5301 	 * spoof into a TCP connection won't bother blindly sending SACK
5302 	 * options anyway.
5303 	 */
5304 	if (dst->seqdiff && (th->th_off << 2) > sizeof(struct tcphdr)) {
5305 		if (pf_modulate_sack(m, off, pd, th, dst))
5306 			*copyback = 1;
5307 	}
5308 
5309 #define	MAXACKWINDOW (0xffff + 1500)	/* 1500 is an arbitrary fudge factor */
5310 	if (SEQ_GEQ(src->seqhi, end) &&
5311 	    /* Last octet inside other's window space */
5312 	    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) &&
5313 	    /* Retrans: not more than one window back */
5314 	    (ackskew >= -MAXACKWINDOW) &&
5315 	    /* Acking not more than one reassembled fragment backwards */
5316 	    (ackskew <= (MAXACKWINDOW << sws)) &&
5317 	    /* Acking not more than one window forward */
5318 	    ((th->th_flags & TH_RST) == 0 || orig_seq == src->seqlo ||
5319 	    (orig_seq == src->seqlo + 1) || (orig_seq + 1 == src->seqlo) ||
5320 	    (pd->flags & PFDESC_IP_REAS) == 0)) {
5321 	    /* Require an exact/+1 sequence match on resets when possible */
5322 
5323 		if (dst->scrub || src->scrub) {
5324 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
5325 			    *state, src, dst, copyback))
5326 				return (PF_DROP);
5327 		}
5328 
5329 		/* update max window */
5330 		if (src->max_win < win)
5331 			src->max_win = win;
5332 		/* synchronize sequencing */
5333 		if (SEQ_GT(end, src->seqlo))
5334 			src->seqlo = end;
5335 		/* slide the window of what the other end can send */
5336 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
5337 			dst->seqhi = ack + MAX((win << sws), 1);
5338 
5339 		/* update states */
5340 		if (th->th_flags & TH_SYN)
5341 			if (src->state < TCPS_SYN_SENT)
5342 				pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5343 		if (th->th_flags & TH_FIN)
5344 			if (src->state < TCPS_CLOSING)
5345 				pf_set_protostate(*state, psrc, TCPS_CLOSING);
5346 		if (th->th_flags & TH_ACK) {
5347 			if (dst->state == TCPS_SYN_SENT) {
5348 				pf_set_protostate(*state, pdst,
5349 				    TCPS_ESTABLISHED);
5350 				if (src->state == TCPS_ESTABLISHED &&
5351 				    (*state)->src_node != NULL &&
5352 				    pf_src_connlimit(state)) {
5353 					REASON_SET(reason, PFRES_SRCLIMIT);
5354 					return (PF_DROP);
5355 				}
5356 			} else if (dst->state == TCPS_CLOSING)
5357 				pf_set_protostate(*state, pdst,
5358 				    TCPS_FIN_WAIT_2);
5359 		}
5360 		if (th->th_flags & TH_RST)
5361 			pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
5362 
5363 		/* update expire time */
5364 		(*state)->expire = time_uptime;
5365 		if (src->state >= TCPS_FIN_WAIT_2 &&
5366 		    dst->state >= TCPS_FIN_WAIT_2)
5367 			(*state)->timeout = PFTM_TCP_CLOSED;
5368 		else if (src->state >= TCPS_CLOSING &&
5369 		    dst->state >= TCPS_CLOSING)
5370 			(*state)->timeout = PFTM_TCP_FIN_WAIT;
5371 		else if (src->state < TCPS_ESTABLISHED ||
5372 		    dst->state < TCPS_ESTABLISHED)
5373 			(*state)->timeout = PFTM_TCP_OPENING;
5374 		else if (src->state >= TCPS_CLOSING ||
5375 		    dst->state >= TCPS_CLOSING)
5376 			(*state)->timeout = PFTM_TCP_CLOSING;
5377 		else
5378 			(*state)->timeout = PFTM_TCP_ESTABLISHED;
5379 
5380 		/* Fall through to PASS packet */
5381 
5382 	} else if ((dst->state < TCPS_SYN_SENT ||
5383 		dst->state >= TCPS_FIN_WAIT_2 ||
5384 		src->state >= TCPS_FIN_WAIT_2) &&
5385 	    SEQ_GEQ(src->seqhi + MAXACKWINDOW, end) &&
5386 	    /* Within a window forward of the originating packet */
5387 	    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW)) {
5388 	    /* Within a window backward of the originating packet */
5389 
5390 		/*
5391 		 * This currently handles three situations:
5392 		 *  1) Stupid stacks will shotgun SYNs before their peer
5393 		 *     replies.
5394 		 *  2) When PF catches an already established stream (the
5395 		 *     firewall rebooted, the state table was flushed, routes
5396 		 *     changed...)
5397 		 *  3) Packets get funky immediately after the connection
5398 		 *     closes (this should catch Solaris spurious ACK|FINs
5399 		 *     that web servers like to spew after a close)
5400 		 *
5401 		 * This must be a little more careful than the above code
5402 		 * since packet floods will also be caught here. We don't
5403 		 * update the TTL here to mitigate the damage of a packet
5404 		 * flood and so the same code can handle awkward establishment
5405 		 * and a loosened connection close.
5406 		 * In the establishment case, a correct peer response will
5407 		 * validate the connection, go through the normal state code
5408 		 * and keep updating the state TTL.
5409 		 */
5410 
5411 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
5412 			printf("pf: loose state match: ");
5413 			pf_print_state(*state);
5414 			pf_print_flags(th->th_flags);
5415 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
5416 			    "pkts=%llu:%llu dir=%s,%s\n", seq, orig_seq, ack,
5417 			    pd->p_len, ackskew, (unsigned long long)(*state)->packets[0],
5418 			    (unsigned long long)(*state)->packets[1],
5419 			    pd->dir == PF_IN ? "in" : "out",
5420 			    pd->dir == (*state)->direction ? "fwd" : "rev");
5421 		}
5422 
5423 		if (dst->scrub || src->scrub) {
5424 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
5425 			    *state, src, dst, copyback))
5426 				return (PF_DROP);
5427 		}
5428 
5429 		/* update max window */
5430 		if (src->max_win < win)
5431 			src->max_win = win;
5432 		/* synchronize sequencing */
5433 		if (SEQ_GT(end, src->seqlo))
5434 			src->seqlo = end;
5435 		/* slide the window of what the other end can send */
5436 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
5437 			dst->seqhi = ack + MAX((win << sws), 1);
5438 
5439 		/*
5440 		 * Cannot set dst->seqhi here since this could be a shotgunned
5441 		 * SYN and not an already established connection.
5442 		 */
5443 
5444 		if (th->th_flags & TH_FIN)
5445 			if (src->state < TCPS_CLOSING)
5446 				pf_set_protostate(*state, psrc, TCPS_CLOSING);
5447 		if (th->th_flags & TH_RST)
5448 			pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
5449 
5450 		/* Fall through to PASS packet */
5451 
5452 	} else {
5453 		if ((*state)->dst.state == TCPS_SYN_SENT &&
5454 		    (*state)->src.state == TCPS_SYN_SENT) {
5455 			/* Send RST for state mismatches during handshake */
5456 			if (!(th->th_flags & TH_RST))
5457 				pf_send_tcp((*state)->rule.ptr, pd->af,
5458 				    pd->dst, pd->src, th->th_dport,
5459 				    th->th_sport, ntohl(th->th_ack), 0,
5460 				    TH_RST, 0, 0,
5461 				    (*state)->rule.ptr->return_ttl, true, 0, 0,
5462 				    (*state)->act.rtableid);
5463 			src->seqlo = 0;
5464 			src->seqhi = 1;
5465 			src->max_win = 1;
5466 		} else if (V_pf_status.debug >= PF_DEBUG_MISC) {
5467 			printf("pf: BAD state: ");
5468 			pf_print_state(*state);
5469 			pf_print_flags(th->th_flags);
5470 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
5471 			    "pkts=%llu:%llu dir=%s,%s\n",
5472 			    seq, orig_seq, ack, pd->p_len, ackskew,
5473 			    (unsigned long long)(*state)->packets[0],
5474 			    (unsigned long long)(*state)->packets[1],
5475 			    pd->dir == PF_IN ? "in" : "out",
5476 			    pd->dir == (*state)->direction ? "fwd" : "rev");
5477 			printf("pf: State failure on: %c %c %c %c | %c %c\n",
5478 			    SEQ_GEQ(src->seqhi, end) ? ' ' : '1',
5479 			    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) ?
5480 			    ' ': '2',
5481 			    (ackskew >= -MAXACKWINDOW) ? ' ' : '3',
5482 			    (ackskew <= (MAXACKWINDOW << sws)) ? ' ' : '4',
5483 			    SEQ_GEQ(src->seqhi + MAXACKWINDOW, end) ?' ' :'5',
5484 			    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW) ?' ' :'6');
5485 		}
5486 		REASON_SET(reason, PFRES_BADSTATE);
5487 		return (PF_DROP);
5488 	}
5489 
5490 	return (PF_PASS);
5491 }
5492 
5493 static int
5494 pf_tcp_track_sloppy(struct pf_kstate **state, struct pf_pdesc *pd, u_short *reason)
5495 {
5496 	struct tcphdr		*th = &pd->hdr.tcp;
5497 	struct pf_state_peer	*src, *dst;
5498 	u_int8_t		 psrc, pdst;
5499 
5500 	if (pd->dir == (*state)->direction) {
5501 		src = &(*state)->src;
5502 		dst = &(*state)->dst;
5503 		psrc = PF_PEER_SRC;
5504 		pdst = PF_PEER_DST;
5505 	} else {
5506 		src = &(*state)->dst;
5507 		dst = &(*state)->src;
5508 		psrc = PF_PEER_DST;
5509 		pdst = PF_PEER_SRC;
5510 	}
5511 
5512 	if (th->th_flags & TH_SYN)
5513 		if (src->state < TCPS_SYN_SENT)
5514 			pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5515 	if (th->th_flags & TH_FIN)
5516 		if (src->state < TCPS_CLOSING)
5517 			pf_set_protostate(*state, psrc, TCPS_CLOSING);
5518 	if (th->th_flags & TH_ACK) {
5519 		if (dst->state == TCPS_SYN_SENT) {
5520 			pf_set_protostate(*state, pdst, TCPS_ESTABLISHED);
5521 			if (src->state == TCPS_ESTABLISHED &&
5522 			    (*state)->src_node != NULL &&
5523 			    pf_src_connlimit(state)) {
5524 				REASON_SET(reason, PFRES_SRCLIMIT);
5525 				return (PF_DROP);
5526 			}
5527 		} else if (dst->state == TCPS_CLOSING) {
5528 			pf_set_protostate(*state, pdst, TCPS_FIN_WAIT_2);
5529 		} else if (src->state == TCPS_SYN_SENT &&
5530 		    dst->state < TCPS_SYN_SENT) {
5531 			/*
5532 			 * Handle a special sloppy case where we only see one
5533 			 * half of the connection. If there is a ACK after
5534 			 * the initial SYN without ever seeing a packet from
5535 			 * the destination, set the connection to established.
5536 			 */
5537 			pf_set_protostate(*state, PF_PEER_BOTH,
5538 			    TCPS_ESTABLISHED);
5539 			dst->state = src->state = TCPS_ESTABLISHED;
5540 			if ((*state)->src_node != NULL &&
5541 			    pf_src_connlimit(state)) {
5542 				REASON_SET(reason, PFRES_SRCLIMIT);
5543 				return (PF_DROP);
5544 			}
5545 		} else if (src->state == TCPS_CLOSING &&
5546 		    dst->state == TCPS_ESTABLISHED &&
5547 		    dst->seqlo == 0) {
5548 			/*
5549 			 * Handle the closing of half connections where we
5550 			 * don't see the full bidirectional FIN/ACK+ACK
5551 			 * handshake.
5552 			 */
5553 			pf_set_protostate(*state, pdst, TCPS_CLOSING);
5554 		}
5555 	}
5556 	if (th->th_flags & TH_RST)
5557 		pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
5558 
5559 	/* update expire time */
5560 	(*state)->expire = time_uptime;
5561 	if (src->state >= TCPS_FIN_WAIT_2 &&
5562 	    dst->state >= TCPS_FIN_WAIT_2)
5563 		(*state)->timeout = PFTM_TCP_CLOSED;
5564 	else if (src->state >= TCPS_CLOSING &&
5565 	    dst->state >= TCPS_CLOSING)
5566 		(*state)->timeout = PFTM_TCP_FIN_WAIT;
5567 	else if (src->state < TCPS_ESTABLISHED ||
5568 	    dst->state < TCPS_ESTABLISHED)
5569 		(*state)->timeout = PFTM_TCP_OPENING;
5570 	else if (src->state >= TCPS_CLOSING ||
5571 	    dst->state >= TCPS_CLOSING)
5572 		(*state)->timeout = PFTM_TCP_CLOSING;
5573 	else
5574 		(*state)->timeout = PFTM_TCP_ESTABLISHED;
5575 
5576 	return (PF_PASS);
5577 }
5578 
5579 static int
5580 pf_synproxy(struct pf_pdesc *pd, struct pf_kstate **state, u_short *reason)
5581 {
5582 	struct pf_state_key	*sk = (*state)->key[pd->didx];
5583 	struct tcphdr		*th = &pd->hdr.tcp;
5584 
5585 	if ((*state)->src.state == PF_TCPS_PROXY_SRC) {
5586 		if (pd->dir != (*state)->direction) {
5587 			REASON_SET(reason, PFRES_SYNPROXY);
5588 			return (PF_SYNPROXY_DROP);
5589 		}
5590 		if (th->th_flags & TH_SYN) {
5591 			if (ntohl(th->th_seq) != (*state)->src.seqlo) {
5592 				REASON_SET(reason, PFRES_SYNPROXY);
5593 				return (PF_DROP);
5594 			}
5595 			pf_send_tcp((*state)->rule.ptr, pd->af, pd->dst,
5596 			    pd->src, th->th_dport, th->th_sport,
5597 			    (*state)->src.seqhi, ntohl(th->th_seq) + 1,
5598 			    TH_SYN|TH_ACK, 0, (*state)->src.mss, 0, true, 0, 0,
5599 			    (*state)->act.rtableid);
5600 			REASON_SET(reason, PFRES_SYNPROXY);
5601 			return (PF_SYNPROXY_DROP);
5602 		} else if ((th->th_flags & (TH_ACK|TH_RST|TH_FIN)) != TH_ACK ||
5603 		    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
5604 		    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
5605 			REASON_SET(reason, PFRES_SYNPROXY);
5606 			return (PF_DROP);
5607 		} else if ((*state)->src_node != NULL &&
5608 		    pf_src_connlimit(state)) {
5609 			REASON_SET(reason, PFRES_SRCLIMIT);
5610 			return (PF_DROP);
5611 		} else
5612 			pf_set_protostate(*state, PF_PEER_SRC,
5613 			    PF_TCPS_PROXY_DST);
5614 	}
5615 	if ((*state)->src.state == PF_TCPS_PROXY_DST) {
5616 		if (pd->dir == (*state)->direction) {
5617 			if (((th->th_flags & (TH_SYN|TH_ACK)) != TH_ACK) ||
5618 			    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
5619 			    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
5620 				REASON_SET(reason, PFRES_SYNPROXY);
5621 				return (PF_DROP);
5622 			}
5623 			(*state)->src.max_win = MAX(ntohs(th->th_win), 1);
5624 			if ((*state)->dst.seqhi == 1)
5625 				(*state)->dst.seqhi = htonl(arc4random());
5626 			pf_send_tcp((*state)->rule.ptr, pd->af,
5627 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
5628 			    sk->port[pd->sidx], sk->port[pd->didx],
5629 			    (*state)->dst.seqhi, 0, TH_SYN, 0,
5630 			    (*state)->src.mss, 0, false, (*state)->tag, 0,
5631 			    (*state)->act.rtableid);
5632 			REASON_SET(reason, PFRES_SYNPROXY);
5633 			return (PF_SYNPROXY_DROP);
5634 		} else if (((th->th_flags & (TH_SYN|TH_ACK)) !=
5635 		    (TH_SYN|TH_ACK)) ||
5636 		    (ntohl(th->th_ack) != (*state)->dst.seqhi + 1)) {
5637 			REASON_SET(reason, PFRES_SYNPROXY);
5638 			return (PF_DROP);
5639 		} else {
5640 			(*state)->dst.max_win = MAX(ntohs(th->th_win), 1);
5641 			(*state)->dst.seqlo = ntohl(th->th_seq);
5642 			pf_send_tcp((*state)->rule.ptr, pd->af, pd->dst,
5643 			    pd->src, th->th_dport, th->th_sport,
5644 			    ntohl(th->th_ack), ntohl(th->th_seq) + 1,
5645 			    TH_ACK, (*state)->src.max_win, 0, 0, false,
5646 			    (*state)->tag, 0, (*state)->act.rtableid);
5647 			pf_send_tcp((*state)->rule.ptr, pd->af,
5648 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
5649 			    sk->port[pd->sidx], sk->port[pd->didx],
5650 			    (*state)->src.seqhi + 1, (*state)->src.seqlo + 1,
5651 			    TH_ACK, (*state)->dst.max_win, 0, 0, true, 0, 0,
5652 			    (*state)->act.rtableid);
5653 			(*state)->src.seqdiff = (*state)->dst.seqhi -
5654 			    (*state)->src.seqlo;
5655 			(*state)->dst.seqdiff = (*state)->src.seqhi -
5656 			    (*state)->dst.seqlo;
5657 			(*state)->src.seqhi = (*state)->src.seqlo +
5658 			    (*state)->dst.max_win;
5659 			(*state)->dst.seqhi = (*state)->dst.seqlo +
5660 			    (*state)->src.max_win;
5661 			(*state)->src.wscale = (*state)->dst.wscale = 0;
5662 			pf_set_protostate(*state, PF_PEER_BOTH,
5663 			    TCPS_ESTABLISHED);
5664 			REASON_SET(reason, PFRES_SYNPROXY);
5665 			return (PF_SYNPROXY_DROP);
5666 		}
5667 	}
5668 
5669 	return (PF_PASS);
5670 }
5671 
5672 static int
5673 pf_test_state_tcp(struct pf_kstate **state, struct pfi_kkif *kif,
5674     struct mbuf *m, int off, void *h, struct pf_pdesc *pd,
5675     u_short *reason)
5676 {
5677 	struct pf_state_key_cmp	 key;
5678 	struct tcphdr		*th = &pd->hdr.tcp;
5679 	int			 copyback = 0;
5680 	int			 action;
5681 	struct pf_state_peer	*src, *dst;
5682 
5683 	bzero(&key, sizeof(key));
5684 	key.af = pd->af;
5685 	key.proto = IPPROTO_TCP;
5686 	if (pd->dir == PF_IN)	{	/* wire side, straight */
5687 		PF_ACPY(&key.addr[0], pd->src, key.af);
5688 		PF_ACPY(&key.addr[1], pd->dst, key.af);
5689 		key.port[0] = th->th_sport;
5690 		key.port[1] = th->th_dport;
5691 	} else {			/* stack side, reverse */
5692 		PF_ACPY(&key.addr[1], pd->src, key.af);
5693 		PF_ACPY(&key.addr[0], pd->dst, key.af);
5694 		key.port[1] = th->th_sport;
5695 		key.port[0] = th->th_dport;
5696 	}
5697 
5698 	STATE_LOOKUP(kif, &key, *state, pd);
5699 
5700 	if (pd->dir == (*state)->direction) {
5701 		src = &(*state)->src;
5702 		dst = &(*state)->dst;
5703 	} else {
5704 		src = &(*state)->dst;
5705 		dst = &(*state)->src;
5706 	}
5707 
5708 	if ((action = pf_synproxy(pd, state, reason)) != PF_PASS)
5709 		return (action);
5710 
5711 	if (dst->state >= TCPS_FIN_WAIT_2 &&
5712 	    src->state >= TCPS_FIN_WAIT_2 &&
5713 	    (((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN) ||
5714 	    ((th->th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_ACK &&
5715 	    pf_syncookie_check(pd) && pd->dir == PF_IN))) {
5716 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
5717 			printf("pf: state reuse ");
5718 			pf_print_state(*state);
5719 			pf_print_flags(th->th_flags);
5720 			printf("\n");
5721 		}
5722 		/* XXX make sure it's the same direction ?? */
5723 		pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
5724 		pf_unlink_state(*state);
5725 		*state = NULL;
5726 		return (PF_DROP);
5727 	}
5728 
5729 	if ((*state)->state_flags & PFSTATE_SLOPPY) {
5730 		if (pf_tcp_track_sloppy(state, pd, reason) == PF_DROP)
5731 			return (PF_DROP);
5732 	} else {
5733 		if (pf_tcp_track_full(state, kif, m, off, pd, reason,
5734 		    &copyback) == PF_DROP)
5735 			return (PF_DROP);
5736 	}
5737 
5738 	/* translate source/destination address, if necessary */
5739 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
5740 		struct pf_state_key *nk = (*state)->key[pd->didx];
5741 
5742 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
5743 		    nk->port[pd->sidx] != th->th_sport)
5744 			pf_change_ap(m, pd->src, &th->th_sport,
5745 			    pd->ip_sum, &th->th_sum, &nk->addr[pd->sidx],
5746 			    nk->port[pd->sidx], 0, pd->af);
5747 
5748 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
5749 		    nk->port[pd->didx] != th->th_dport)
5750 			pf_change_ap(m, pd->dst, &th->th_dport,
5751 			    pd->ip_sum, &th->th_sum, &nk->addr[pd->didx],
5752 			    nk->port[pd->didx], 0, pd->af);
5753 		copyback = 1;
5754 	}
5755 
5756 	/* Copyback sequence modulation or stateful scrub changes if needed */
5757 	if (copyback)
5758 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
5759 
5760 	return (PF_PASS);
5761 }
5762 
5763 static int
5764 pf_test_state_udp(struct pf_kstate **state, struct pfi_kkif *kif,
5765     struct mbuf *m, int off, void *h, struct pf_pdesc *pd)
5766 {
5767 	struct pf_state_peer	*src, *dst;
5768 	struct pf_state_key_cmp	 key;
5769 	struct udphdr		*uh = &pd->hdr.udp;
5770 	uint8_t			 psrc, pdst;
5771 
5772 	bzero(&key, sizeof(key));
5773 	key.af = pd->af;
5774 	key.proto = IPPROTO_UDP;
5775 	if (pd->dir == PF_IN)	{	/* wire side, straight */
5776 		PF_ACPY(&key.addr[0], pd->src, key.af);
5777 		PF_ACPY(&key.addr[1], pd->dst, key.af);
5778 		key.port[0] = uh->uh_sport;
5779 		key.port[1] = uh->uh_dport;
5780 	} else {			/* stack side, reverse */
5781 		PF_ACPY(&key.addr[1], pd->src, key.af);
5782 		PF_ACPY(&key.addr[0], pd->dst, key.af);
5783 		key.port[1] = uh->uh_sport;
5784 		key.port[0] = uh->uh_dport;
5785 	}
5786 
5787 	STATE_LOOKUP(kif, &key, *state, pd);
5788 
5789 	if (pd->dir == (*state)->direction) {
5790 		src = &(*state)->src;
5791 		dst = &(*state)->dst;
5792 		psrc = PF_PEER_SRC;
5793 		pdst = PF_PEER_DST;
5794 	} else {
5795 		src = &(*state)->dst;
5796 		dst = &(*state)->src;
5797 		psrc = PF_PEER_DST;
5798 		pdst = PF_PEER_SRC;
5799 	}
5800 
5801 	/* update states */
5802 	if (src->state < PFUDPS_SINGLE)
5803 		pf_set_protostate(*state, psrc, PFUDPS_SINGLE);
5804 	if (dst->state == PFUDPS_SINGLE)
5805 		pf_set_protostate(*state, pdst, PFUDPS_MULTIPLE);
5806 
5807 	/* update expire time */
5808 	(*state)->expire = time_uptime;
5809 	if (src->state == PFUDPS_MULTIPLE && dst->state == PFUDPS_MULTIPLE)
5810 		(*state)->timeout = PFTM_UDP_MULTIPLE;
5811 	else
5812 		(*state)->timeout = PFTM_UDP_SINGLE;
5813 
5814 	/* translate source/destination address, if necessary */
5815 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
5816 		struct pf_state_key *nk = (*state)->key[pd->didx];
5817 
5818 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
5819 		    nk->port[pd->sidx] != uh->uh_sport)
5820 			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
5821 			    &uh->uh_sum, &nk->addr[pd->sidx],
5822 			    nk->port[pd->sidx], 1, pd->af);
5823 
5824 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
5825 		    nk->port[pd->didx] != uh->uh_dport)
5826 			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
5827 			    &uh->uh_sum, &nk->addr[pd->didx],
5828 			    nk->port[pd->didx], 1, pd->af);
5829 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
5830 	}
5831 
5832 	return (PF_PASS);
5833 }
5834 
5835 static int
5836 pf_test_state_sctp(struct pf_kstate **state, struct pfi_kkif *kif,
5837     struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason)
5838 {
5839 	struct pf_state_key_cmp	 key;
5840 	struct pf_state_peer	*src; //, *dst;
5841 	struct sctphdr		*sh = &pd->hdr.sctp;
5842 	u_int8_t		 psrc; //, pdst;
5843 
5844 	bzero(&key, sizeof(key));
5845 	key.af = pd->af;
5846 	key.proto = IPPROTO_SCTP;
5847 	if (pd->dir == PF_IN)	{	/* wire side, straight */
5848 		PF_ACPY(&key.addr[0], pd->src, key.af);
5849 		PF_ACPY(&key.addr[1], pd->dst, key.af);
5850 		key.port[0] = sh->src_port;
5851 		key.port[1] = sh->dest_port;
5852 	} else {			/* stack side, reverse */
5853 		PF_ACPY(&key.addr[1], pd->src, key.af);
5854 		PF_ACPY(&key.addr[0], pd->dst, key.af);
5855 		key.port[1] = sh->src_port;
5856 		key.port[0] = sh->dest_port;
5857 	}
5858 
5859 	STATE_LOOKUP(kif, &key, *state, pd);
5860 
5861 	if (pd->dir == (*state)->direction) {
5862 		src = &(*state)->src;
5863 		psrc = PF_PEER_SRC;
5864 	} else {
5865 		src = &(*state)->dst;
5866 		psrc = PF_PEER_DST;
5867 	}
5868 
5869 	/* Track state. */
5870 	if (pd->sctp_flags & PFDESC_SCTP_INIT) {
5871 		if (src->state < SCTP_COOKIE_WAIT) {
5872 			pf_set_protostate(*state, psrc, SCTP_COOKIE_WAIT);
5873 			(*state)->timeout = PFTM_TCP_OPENING;
5874 		}
5875 	}
5876 	if (pd->sctp_flags & PFDESC_SCTP_COOKIE) {
5877 		if (src->state < SCTP_ESTABLISHED) {
5878 			pf_set_protostate(*state, psrc, SCTP_ESTABLISHED);
5879 			(*state)->timeout = PFTM_TCP_ESTABLISHED;
5880 		}
5881 	}
5882 	if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN | PFDESC_SCTP_ABORT |
5883 	    PFDESC_SCTP_SHUTDOWN_COMPLETE)) {
5884 		if (src->state < SCTP_SHUTDOWN_PENDING) {
5885 			pf_set_protostate(*state, psrc, SCTP_SHUTDOWN_PENDING);
5886 			(*state)->timeout = PFTM_TCP_CLOSING;
5887 		}
5888 	}
5889 
5890 	if (src->scrub != NULL) {
5891 		if (src->scrub->pfss_v_tag == 0) {
5892 			src->scrub->pfss_v_tag = pd->hdr.sctp.v_tag;
5893 		} else  if (src->scrub->pfss_v_tag != pd->hdr.sctp.v_tag)
5894 			return (PF_DROP);
5895 	}
5896 
5897 	(*state)->expire = time_uptime;
5898 
5899 	/* translate source/destination address, if necessary */
5900 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
5901 		uint16_t checksum = 0;
5902 		struct pf_state_key *nk = (*state)->key[pd->didx];
5903 
5904 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
5905 		    nk->port[pd->sidx] != pd->hdr.sctp.src_port) {
5906 			pf_change_ap(m, pd->src, &pd->hdr.sctp.src_port,
5907 			    pd->ip_sum, &checksum, &nk->addr[pd->sidx],
5908 			    nk->port[pd->sidx], 1, pd->af);
5909 		}
5910 
5911 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
5912 		    nk->port[pd->didx] != pd->hdr.sctp.dest_port) {
5913 			pf_change_ap(m, pd->dst, &pd->hdr.sctp.dest_port,
5914 			    pd->ip_sum, &checksum, &nk->addr[pd->didx],
5915 			    nk->port[pd->didx], 1, pd->af);
5916 		}
5917 	}
5918 
5919 	return (PF_PASS);
5920 }
5921 
5922 static void
5923 pf_sctp_multihome_delayed(struct pf_pdesc *pd, int off, struct pfi_kkif *kif,
5924     struct pf_kstate *s, int action)
5925 {
5926 	struct pf_sctp_multihome_job	*j, *tmp;
5927 	int			 ret __unused;;
5928 	struct pf_kstate	*sm = NULL;
5929 	struct pf_krule		*ra = NULL;
5930 	struct pf_krule		*r = &V_pf_default_rule;
5931 	struct pf_kruleset	*rs = NULL;
5932 
5933 	PF_RULES_RLOCK_TRACKER;
5934 
5935 	TAILQ_FOREACH_SAFE(j, &pd->sctp_multihome_jobs, next, tmp) {
5936 		if (s == NULL || action != PF_PASS)
5937 			goto free;
5938 
5939 		switch (j->op) {
5940 		case  SCTP_ADD_IP_ADDRESS: {
5941 			j->pd.sctp_flags |= PFDESC_SCTP_ADD_IP;
5942 			PF_RULES_RLOCK();
5943 			ret = pf_test_rule(&r, &sm, kif,
5944 			    j->m, off, &j->pd, &ra, &rs, NULL);
5945 			PF_RULES_RUNLOCK();
5946 			SDT_PROBE4(pf, sctp, multihome, test, kif, r, j->m, ret);
5947 			if (sm) {
5948 				/* Inherit v_tag values. */
5949 				sm->src.scrub->pfss_v_tag = s->src.scrub->pfss_flags;
5950 				sm->dst.scrub->pfss_v_tag = s->dst.scrub->pfss_flags;
5951 				PF_STATE_UNLOCK(sm);
5952 			}
5953 			break;
5954 		}
5955 		case SCTP_DEL_IP_ADDRESS: {
5956 			struct pf_state_key_cmp key;
5957 			uint8_t psrc;
5958 
5959 			bzero(&key, sizeof(key));
5960 			key.af = j->pd.af;
5961 			key.proto = IPPROTO_SCTP;
5962 			if (j->pd.dir == PF_IN)	{	/* wire side, straight */
5963 				PF_ACPY(&key.addr[0], j->pd.src, key.af);
5964 				PF_ACPY(&key.addr[1], j->pd.dst, key.af);
5965 				key.port[0] = j->pd.hdr.sctp.src_port;
5966 				key.port[1] = j->pd.hdr.sctp.dest_port;
5967 			} else {			/* stack side, reverse */
5968 				PF_ACPY(&key.addr[1], j->pd.src, key.af);
5969 				PF_ACPY(&key.addr[0], j->pd.dst, key.af);
5970 				key.port[1] = j->pd.hdr.sctp.src_port;
5971 				key.port[0] = j->pd.hdr.sctp.dest_port;
5972 			}
5973 
5974 			sm = pf_find_state(kif, &key, j->pd.dir);
5975 			if (sm != NULL) {
5976 				PF_STATE_LOCK_ASSERT(sm);
5977 				if (j->pd.dir == sm->direction) {
5978 					psrc = PF_PEER_SRC;
5979 				} else {
5980 					psrc = PF_PEER_DST;
5981 				}
5982 				pf_set_protostate(sm, psrc, SCTP_SHUTDOWN_PENDING);
5983 				sm->timeout = PFTM_TCP_CLOSING;
5984 				PF_STATE_UNLOCK(sm);
5985 			}
5986 			break;
5987 		default:
5988 			panic("Unknown op %#x", j->op);
5989 		}
5990 		}
5991 
5992 free:
5993 		free(j, M_PFTEMP);
5994 	}
5995 }
5996 
5997 static int
5998 pf_multihome_scan(struct mbuf *m, int start, int len, struct pf_pdesc *pd,
5999     struct pfi_kkif *kif, int op)
6000 {
6001 	int			 off = 0;
6002 	struct pf_sctp_multihome_job	*job;
6003 
6004 	while (off < len) {
6005 		struct sctp_paramhdr h;
6006 
6007 		if (!pf_pull_hdr(m, start + off, &h, sizeof(h), NULL, NULL,
6008 		    pd->af))
6009 			return (PF_DROP);
6010 
6011 		/* Parameters are at least 4 bytes. */
6012 		if (ntohs(h.param_length) < 4)
6013 			return (PF_DROP);
6014 
6015 		switch (ntohs(h.param_type)) {
6016 		case  SCTP_IPV4_ADDRESS: {
6017 			struct in_addr t;
6018 
6019 			if (ntohs(h.param_length) !=
6020 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
6021 				return (PF_DROP);
6022 
6023 			if (!pf_pull_hdr(m, start + off + sizeof(h), &t, sizeof(t),
6024 			    NULL, NULL, pd->af))
6025 				return (PF_DROP);
6026 
6027 			/*
6028 			 * Avoid duplicating states. We'll already have
6029 			 * created a state based on the source address of
6030 			 * the packet, but SCTP endpoints may also list this
6031 			 * address again in the INIT(_ACK) parameters.
6032 			 */
6033 			if (t.s_addr == pd->src->v4.s_addr)
6034 				break;
6035 
6036 			if (in_nullhost(t))
6037 				t.s_addr = pd->src->v4.s_addr;
6038 
6039 			/*
6040 			 * We hold the state lock (idhash) here, which means
6041 			 * that we can't acquire the keyhash, or we'll get a
6042 			 * LOR (and potentially double-lock things too). We also
6043 			 * can't release the state lock here, so instead we'll
6044 			 * enqueue this for async handling.
6045 			 * There's a relatively small race here, in that a
6046 			 * packet using the new addresses could arrive already,
6047 			 * but that's just though luck for it.
6048 			 */
6049 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
6050 			if (! job)
6051 				return (PF_DROP);
6052 
6053 			memcpy(&job->pd, pd, sizeof(*pd));
6054 
6055 			// New source address!
6056 			memcpy(&job->src, &t, sizeof(t));
6057 			job->pd.src = &job->src;
6058 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
6059 			job->pd.dst = &job->dst;
6060 			job->m = m;
6061 			job->op = op;
6062 
6063 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
6064 			break;
6065 		}
6066 #ifdef INET6
6067 		case SCTP_IPV6_ADDRESS: {
6068 			struct in6_addr t;
6069 
6070 			if (ntohs(h.param_length) !=
6071 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
6072 				return (PF_DROP);
6073 
6074 			if (!pf_pull_hdr(m, start + off + sizeof(h), &t, sizeof(t),
6075 			    NULL, NULL, pd->af))
6076 				return (PF_DROP);
6077 			if (memcmp(&t, &pd->src->v6, sizeof(t)) == 0)
6078 				break;
6079 			if (memcmp(&t, &in6addr_any, sizeof(t)) == 0)
6080 				memcpy(&t, &pd->src->v6, sizeof(t));
6081 
6082 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
6083 			if (! job)
6084 				return (PF_DROP);
6085 
6086 			memcpy(&job->pd, pd, sizeof(*pd));
6087 			memcpy(&job->src, &t, sizeof(t));
6088 			job->pd.src = &job->src;
6089 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
6090 			job->pd.dst = &job->dst;
6091 			job->m = m;
6092 			job->op = op;
6093 
6094 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
6095 			break;
6096 		}
6097 #endif
6098 		case SCTP_ADD_IP_ADDRESS: {
6099 			int ret;
6100 			struct sctp_asconf_paramhdr ah;
6101 
6102 			if (!pf_pull_hdr(m, start + off, &ah, sizeof(ah),
6103 			    NULL, NULL, pd->af))
6104 				return (PF_DROP);
6105 
6106 			ret = pf_multihome_scan(m, start + off + sizeof(ah),
6107 			    ntohs(ah.ph.param_length) - sizeof(ah), pd, kif,
6108 			    SCTP_ADD_IP_ADDRESS);
6109 			if (ret != PF_PASS)
6110 				return (ret);
6111 			break;
6112 		}
6113 		case SCTP_DEL_IP_ADDRESS: {
6114 			int ret;
6115 			struct sctp_asconf_paramhdr ah;
6116 
6117 			if (!pf_pull_hdr(m, start + off, &ah, sizeof(ah),
6118 			    NULL, NULL, pd->af))
6119 				return (PF_DROP);
6120 			ret = pf_multihome_scan(m, start + off + sizeof(ah),
6121 			    ntohs(ah.ph.param_length) - sizeof(ah), pd, kif,
6122 			    SCTP_DEL_IP_ADDRESS);
6123 			if (ret != PF_PASS)
6124 				return (ret);
6125 			break;
6126 		}
6127 		default:
6128 			break;
6129 		}
6130 
6131 		off += roundup(ntohs(h.param_length), 4);
6132 	}
6133 
6134 	return (PF_PASS);
6135 }
6136 int
6137 pf_multihome_scan_init(struct mbuf *m, int start, int len, struct pf_pdesc *pd,
6138     struct pfi_kkif *kif)
6139 {
6140 	start += sizeof(struct sctp_init_chunk);
6141 	len -= sizeof(struct sctp_init_chunk);
6142 
6143 	return (pf_multihome_scan(m, start, len, pd, kif, SCTP_ADD_IP_ADDRESS));
6144 }
6145 
6146 int
6147 pf_multihome_scan_asconf(struct mbuf *m, int start, int len,
6148     struct pf_pdesc *pd, struct pfi_kkif *kif)
6149 {
6150 	start += sizeof(struct sctp_asconf_chunk);
6151 	len -= sizeof(struct sctp_asconf_chunk);
6152 
6153 	return (pf_multihome_scan(m, start, len, pd, kif, SCTP_ADD_IP_ADDRESS));
6154 }
6155 
6156 static int
6157 pf_test_state_icmp(struct pf_kstate **state, struct pfi_kkif *kif,
6158     struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason)
6159 {
6160 	struct pf_addr  *saddr = pd->src, *daddr = pd->dst;
6161 	u_int16_t	 icmpid = 0, *icmpsum;
6162 	u_int8_t	 icmptype, icmpcode;
6163 	int		 state_icmp = 0;
6164 	struct pf_state_key_cmp key;
6165 
6166 	bzero(&key, sizeof(key));
6167 	switch (pd->proto) {
6168 #ifdef INET
6169 	case IPPROTO_ICMP:
6170 		icmptype = pd->hdr.icmp.icmp_type;
6171 		icmpcode = pd->hdr.icmp.icmp_code;
6172 		icmpid = pd->hdr.icmp.icmp_id;
6173 		icmpsum = &pd->hdr.icmp.icmp_cksum;
6174 
6175 		if (icmptype == ICMP_UNREACH ||
6176 		    icmptype == ICMP_SOURCEQUENCH ||
6177 		    icmptype == ICMP_REDIRECT ||
6178 		    icmptype == ICMP_TIMXCEED ||
6179 		    icmptype == ICMP_PARAMPROB)
6180 			state_icmp++;
6181 		break;
6182 #endif /* INET */
6183 #ifdef INET6
6184 	case IPPROTO_ICMPV6:
6185 		icmptype = pd->hdr.icmp6.icmp6_type;
6186 		icmpcode = pd->hdr.icmp6.icmp6_code;
6187 		icmpid = pd->hdr.icmp6.icmp6_id;
6188 		icmpsum = &pd->hdr.icmp6.icmp6_cksum;
6189 
6190 		if (icmptype == ICMP6_DST_UNREACH ||
6191 		    icmptype == ICMP6_PACKET_TOO_BIG ||
6192 		    icmptype == ICMP6_TIME_EXCEEDED ||
6193 		    icmptype == ICMP6_PARAM_PROB)
6194 			state_icmp++;
6195 		break;
6196 #endif /* INET6 */
6197 	}
6198 
6199 	if (!state_icmp) {
6200 		/*
6201 		 * ICMP query/reply message not related to a TCP/UDP packet.
6202 		 * Search for an ICMP state.
6203 		 */
6204 		key.af = pd->af;
6205 		key.proto = pd->proto;
6206 		key.port[0] = key.port[1] = icmpid;
6207 		if (pd->dir == PF_IN)	{	/* wire side, straight */
6208 			PF_ACPY(&key.addr[0], pd->src, key.af);
6209 			PF_ACPY(&key.addr[1], pd->dst, key.af);
6210 		} else {			/* stack side, reverse */
6211 			PF_ACPY(&key.addr[1], pd->src, key.af);
6212 			PF_ACPY(&key.addr[0], pd->dst, key.af);
6213 		}
6214 
6215 		STATE_LOOKUP(kif, &key, *state, pd);
6216 
6217 		(*state)->expire = time_uptime;
6218 		(*state)->timeout = PFTM_ICMP_ERROR_REPLY;
6219 
6220 		/* translate source/destination address, if necessary */
6221 		if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6222 			struct pf_state_key *nk = (*state)->key[pd->didx];
6223 
6224 			switch (pd->af) {
6225 #ifdef INET
6226 			case AF_INET:
6227 				if (PF_ANEQ(pd->src,
6228 				    &nk->addr[pd->sidx], AF_INET))
6229 					pf_change_a(&saddr->v4.s_addr,
6230 					    pd->ip_sum,
6231 					    nk->addr[pd->sidx].v4.s_addr, 0);
6232 
6233 				if (PF_ANEQ(pd->dst, &nk->addr[pd->didx],
6234 				    AF_INET))
6235 					pf_change_a(&daddr->v4.s_addr,
6236 					    pd->ip_sum,
6237 					    nk->addr[pd->didx].v4.s_addr, 0);
6238 
6239 				if (nk->port[0] !=
6240 				    pd->hdr.icmp.icmp_id) {
6241 					pd->hdr.icmp.icmp_cksum =
6242 					    pf_cksum_fixup(
6243 					    pd->hdr.icmp.icmp_cksum, icmpid,
6244 					    nk->port[pd->sidx], 0);
6245 					pd->hdr.icmp.icmp_id =
6246 					    nk->port[pd->sidx];
6247 				}
6248 
6249 				m_copyback(m, off, ICMP_MINLEN,
6250 				    (caddr_t )&pd->hdr.icmp);
6251 				break;
6252 #endif /* INET */
6253 #ifdef INET6
6254 			case AF_INET6:
6255 				if (PF_ANEQ(pd->src,
6256 				    &nk->addr[pd->sidx], AF_INET6))
6257 					pf_change_a6(saddr,
6258 					    &pd->hdr.icmp6.icmp6_cksum,
6259 					    &nk->addr[pd->sidx], 0);
6260 
6261 				if (PF_ANEQ(pd->dst,
6262 				    &nk->addr[pd->didx], AF_INET6))
6263 					pf_change_a6(daddr,
6264 					    &pd->hdr.icmp6.icmp6_cksum,
6265 					    &nk->addr[pd->didx], 0);
6266 
6267 				m_copyback(m, off, sizeof(struct icmp6_hdr),
6268 				    (caddr_t )&pd->hdr.icmp6);
6269 				break;
6270 #endif /* INET6 */
6271 			}
6272 		}
6273 		return (PF_PASS);
6274 
6275 	} else {
6276 		/*
6277 		 * ICMP error message in response to a TCP/UDP packet.
6278 		 * Extract the inner TCP/UDP header and search for that state.
6279 		 */
6280 
6281 		struct pf_pdesc	pd2;
6282 		bzero(&pd2, sizeof pd2);
6283 #ifdef INET
6284 		struct ip	h2;
6285 #endif /* INET */
6286 #ifdef INET6
6287 		struct ip6_hdr	h2_6;
6288 		int		terminal = 0;
6289 #endif /* INET6 */
6290 		int		ipoff2 = 0;
6291 		int		off2 = 0;
6292 
6293 		pd2.af = pd->af;
6294 		/* Payload packet is from the opposite direction. */
6295 		pd2.sidx = (pd->dir == PF_IN) ? 1 : 0;
6296 		pd2.didx = (pd->dir == PF_IN) ? 0 : 1;
6297 		switch (pd->af) {
6298 #ifdef INET
6299 		case AF_INET:
6300 			/* offset of h2 in mbuf chain */
6301 			ipoff2 = off + ICMP_MINLEN;
6302 
6303 			if (!pf_pull_hdr(m, ipoff2, &h2, sizeof(h2),
6304 			    NULL, reason, pd2.af)) {
6305 				DPFPRINTF(PF_DEBUG_MISC,
6306 				    ("pf: ICMP error message too short "
6307 				    "(ip)\n"));
6308 				return (PF_DROP);
6309 			}
6310 			/*
6311 			 * ICMP error messages don't refer to non-first
6312 			 * fragments
6313 			 */
6314 			if (h2.ip_off & htons(IP_OFFMASK)) {
6315 				REASON_SET(reason, PFRES_FRAG);
6316 				return (PF_DROP);
6317 			}
6318 
6319 			/* offset of protocol header that follows h2 */
6320 			off2 = ipoff2 + (h2.ip_hl << 2);
6321 
6322 			pd2.proto = h2.ip_p;
6323 			pd2.src = (struct pf_addr *)&h2.ip_src;
6324 			pd2.dst = (struct pf_addr *)&h2.ip_dst;
6325 			pd2.ip_sum = &h2.ip_sum;
6326 			break;
6327 #endif /* INET */
6328 #ifdef INET6
6329 		case AF_INET6:
6330 			ipoff2 = off + sizeof(struct icmp6_hdr);
6331 
6332 			if (!pf_pull_hdr(m, ipoff2, &h2_6, sizeof(h2_6),
6333 			    NULL, reason, pd2.af)) {
6334 				DPFPRINTF(PF_DEBUG_MISC,
6335 				    ("pf: ICMP error message too short "
6336 				    "(ip6)\n"));
6337 				return (PF_DROP);
6338 			}
6339 			pd2.proto = h2_6.ip6_nxt;
6340 			pd2.src = (struct pf_addr *)&h2_6.ip6_src;
6341 			pd2.dst = (struct pf_addr *)&h2_6.ip6_dst;
6342 			pd2.ip_sum = NULL;
6343 			off2 = ipoff2 + sizeof(h2_6);
6344 			do {
6345 				switch (pd2.proto) {
6346 				case IPPROTO_FRAGMENT:
6347 					/*
6348 					 * ICMPv6 error messages for
6349 					 * non-first fragments
6350 					 */
6351 					REASON_SET(reason, PFRES_FRAG);
6352 					return (PF_DROP);
6353 				case IPPROTO_AH:
6354 				case IPPROTO_HOPOPTS:
6355 				case IPPROTO_ROUTING:
6356 				case IPPROTO_DSTOPTS: {
6357 					/* get next header and header length */
6358 					struct ip6_ext opt6;
6359 
6360 					if (!pf_pull_hdr(m, off2, &opt6,
6361 					    sizeof(opt6), NULL, reason,
6362 					    pd2.af)) {
6363 						DPFPRINTF(PF_DEBUG_MISC,
6364 						    ("pf: ICMPv6 short opt\n"));
6365 						return (PF_DROP);
6366 					}
6367 					if (pd2.proto == IPPROTO_AH)
6368 						off2 += (opt6.ip6e_len + 2) * 4;
6369 					else
6370 						off2 += (opt6.ip6e_len + 1) * 8;
6371 					pd2.proto = opt6.ip6e_nxt;
6372 					/* goto the next header */
6373 					break;
6374 				}
6375 				default:
6376 					terminal++;
6377 					break;
6378 				}
6379 			} while (!terminal);
6380 			break;
6381 #endif /* INET6 */
6382 		}
6383 
6384 		if (PF_ANEQ(pd->dst, pd2.src, pd->af)) {
6385 			if (V_pf_status.debug >= PF_DEBUG_MISC) {
6386 				printf("pf: BAD ICMP %d:%d outer dst: ",
6387 				    icmptype, icmpcode);
6388 				pf_print_host(pd->src, 0, pd->af);
6389 				printf(" -> ");
6390 				pf_print_host(pd->dst, 0, pd->af);
6391 				printf(" inner src: ");
6392 				pf_print_host(pd2.src, 0, pd2.af);
6393 				printf(" -> ");
6394 				pf_print_host(pd2.dst, 0, pd2.af);
6395 				printf("\n");
6396 			}
6397 			REASON_SET(reason, PFRES_BADSTATE);
6398 			return (PF_DROP);
6399 		}
6400 
6401 		switch (pd2.proto) {
6402 		case IPPROTO_TCP: {
6403 			struct tcphdr		 th;
6404 			u_int32_t		 seq;
6405 			struct pf_state_peer	*src, *dst;
6406 			u_int8_t		 dws;
6407 			int			 copyback = 0;
6408 
6409 			/*
6410 			 * Only the first 8 bytes of the TCP header can be
6411 			 * expected. Don't access any TCP header fields after
6412 			 * th_seq, an ackskew test is not possible.
6413 			 */
6414 			if (!pf_pull_hdr(m, off2, &th, 8, NULL, reason,
6415 			    pd2.af)) {
6416 				DPFPRINTF(PF_DEBUG_MISC,
6417 				    ("pf: ICMP error message too short "
6418 				    "(tcp)\n"));
6419 				return (PF_DROP);
6420 			}
6421 
6422 			key.af = pd2.af;
6423 			key.proto = IPPROTO_TCP;
6424 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
6425 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
6426 			key.port[pd2.sidx] = th.th_sport;
6427 			key.port[pd2.didx] = th.th_dport;
6428 
6429 			STATE_LOOKUP(kif, &key, *state, pd);
6430 
6431 			if (pd->dir == (*state)->direction) {
6432 				src = &(*state)->dst;
6433 				dst = &(*state)->src;
6434 			} else {
6435 				src = &(*state)->src;
6436 				dst = &(*state)->dst;
6437 			}
6438 
6439 			if (src->wscale && dst->wscale)
6440 				dws = dst->wscale & PF_WSCALE_MASK;
6441 			else
6442 				dws = 0;
6443 
6444 			/* Demodulate sequence number */
6445 			seq = ntohl(th.th_seq) - src->seqdiff;
6446 			if (src->seqdiff) {
6447 				pf_change_a(&th.th_seq, icmpsum,
6448 				    htonl(seq), 0);
6449 				copyback = 1;
6450 			}
6451 
6452 			if (!((*state)->state_flags & PFSTATE_SLOPPY) &&
6453 			    (!SEQ_GEQ(src->seqhi, seq) ||
6454 			    !SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)))) {
6455 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
6456 					printf("pf: BAD ICMP %d:%d ",
6457 					    icmptype, icmpcode);
6458 					pf_print_host(pd->src, 0, pd->af);
6459 					printf(" -> ");
6460 					pf_print_host(pd->dst, 0, pd->af);
6461 					printf(" state: ");
6462 					pf_print_state(*state);
6463 					printf(" seq=%u\n", seq);
6464 				}
6465 				REASON_SET(reason, PFRES_BADSTATE);
6466 				return (PF_DROP);
6467 			} else {
6468 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
6469 					printf("pf: OK ICMP %d:%d ",
6470 					    icmptype, icmpcode);
6471 					pf_print_host(pd->src, 0, pd->af);
6472 					printf(" -> ");
6473 					pf_print_host(pd->dst, 0, pd->af);
6474 					printf(" state: ");
6475 					pf_print_state(*state);
6476 					printf(" seq=%u\n", seq);
6477 				}
6478 			}
6479 
6480 			/* translate source/destination address, if necessary */
6481 			if ((*state)->key[PF_SK_WIRE] !=
6482 			    (*state)->key[PF_SK_STACK]) {
6483 				struct pf_state_key *nk =
6484 				    (*state)->key[pd->didx];
6485 
6486 				if (PF_ANEQ(pd2.src,
6487 				    &nk->addr[pd2.sidx], pd2.af) ||
6488 				    nk->port[pd2.sidx] != th.th_sport)
6489 					pf_change_icmp(pd2.src, &th.th_sport,
6490 					    daddr, &nk->addr[pd2.sidx],
6491 					    nk->port[pd2.sidx], NULL,
6492 					    pd2.ip_sum, icmpsum,
6493 					    pd->ip_sum, 0, pd2.af);
6494 
6495 				if (PF_ANEQ(pd2.dst,
6496 				    &nk->addr[pd2.didx], pd2.af) ||
6497 				    nk->port[pd2.didx] != th.th_dport)
6498 					pf_change_icmp(pd2.dst, &th.th_dport,
6499 					    saddr, &nk->addr[pd2.didx],
6500 					    nk->port[pd2.didx], NULL,
6501 					    pd2.ip_sum, icmpsum,
6502 					    pd->ip_sum, 0, pd2.af);
6503 				copyback = 1;
6504 			}
6505 
6506 			if (copyback) {
6507 				switch (pd2.af) {
6508 #ifdef INET
6509 				case AF_INET:
6510 					m_copyback(m, off, ICMP_MINLEN,
6511 					    (caddr_t )&pd->hdr.icmp);
6512 					m_copyback(m, ipoff2, sizeof(h2),
6513 					    (caddr_t )&h2);
6514 					break;
6515 #endif /* INET */
6516 #ifdef INET6
6517 				case AF_INET6:
6518 					m_copyback(m, off,
6519 					    sizeof(struct icmp6_hdr),
6520 					    (caddr_t )&pd->hdr.icmp6);
6521 					m_copyback(m, ipoff2, sizeof(h2_6),
6522 					    (caddr_t )&h2_6);
6523 					break;
6524 #endif /* INET6 */
6525 				}
6526 				m_copyback(m, off2, 8, (caddr_t)&th);
6527 			}
6528 
6529 			return (PF_PASS);
6530 			break;
6531 		}
6532 		case IPPROTO_UDP: {
6533 			struct udphdr		uh;
6534 
6535 			if (!pf_pull_hdr(m, off2, &uh, sizeof(uh),
6536 			    NULL, reason, pd2.af)) {
6537 				DPFPRINTF(PF_DEBUG_MISC,
6538 				    ("pf: ICMP error message too short "
6539 				    "(udp)\n"));
6540 				return (PF_DROP);
6541 			}
6542 
6543 			key.af = pd2.af;
6544 			key.proto = IPPROTO_UDP;
6545 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
6546 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
6547 			key.port[pd2.sidx] = uh.uh_sport;
6548 			key.port[pd2.didx] = uh.uh_dport;
6549 
6550 			STATE_LOOKUP(kif, &key, *state, pd);
6551 
6552 			/* translate source/destination address, if necessary */
6553 			if ((*state)->key[PF_SK_WIRE] !=
6554 			    (*state)->key[PF_SK_STACK]) {
6555 				struct pf_state_key *nk =
6556 				    (*state)->key[pd->didx];
6557 
6558 				if (PF_ANEQ(pd2.src,
6559 				    &nk->addr[pd2.sidx], pd2.af) ||
6560 				    nk->port[pd2.sidx] != uh.uh_sport)
6561 					pf_change_icmp(pd2.src, &uh.uh_sport,
6562 					    daddr, &nk->addr[pd2.sidx],
6563 					    nk->port[pd2.sidx], &uh.uh_sum,
6564 					    pd2.ip_sum, icmpsum,
6565 					    pd->ip_sum, 1, pd2.af);
6566 
6567 				if (PF_ANEQ(pd2.dst,
6568 				    &nk->addr[pd2.didx], pd2.af) ||
6569 				    nk->port[pd2.didx] != uh.uh_dport)
6570 					pf_change_icmp(pd2.dst, &uh.uh_dport,
6571 					    saddr, &nk->addr[pd2.didx],
6572 					    nk->port[pd2.didx], &uh.uh_sum,
6573 					    pd2.ip_sum, icmpsum,
6574 					    pd->ip_sum, 1, pd2.af);
6575 
6576 				switch (pd2.af) {
6577 #ifdef INET
6578 				case AF_INET:
6579 					m_copyback(m, off, ICMP_MINLEN,
6580 					    (caddr_t )&pd->hdr.icmp);
6581 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
6582 					break;
6583 #endif /* INET */
6584 #ifdef INET6
6585 				case AF_INET6:
6586 					m_copyback(m, off,
6587 					    sizeof(struct icmp6_hdr),
6588 					    (caddr_t )&pd->hdr.icmp6);
6589 					m_copyback(m, ipoff2, sizeof(h2_6),
6590 					    (caddr_t )&h2_6);
6591 					break;
6592 #endif /* INET6 */
6593 				}
6594 				m_copyback(m, off2, sizeof(uh), (caddr_t)&uh);
6595 			}
6596 			return (PF_PASS);
6597 			break;
6598 		}
6599 #ifdef INET
6600 		case IPPROTO_ICMP: {
6601 			struct icmp		iih;
6602 
6603 			if (!pf_pull_hdr(m, off2, &iih, ICMP_MINLEN,
6604 			    NULL, reason, pd2.af)) {
6605 				DPFPRINTF(PF_DEBUG_MISC,
6606 				    ("pf: ICMP error message too short i"
6607 				    "(icmp)\n"));
6608 				return (PF_DROP);
6609 			}
6610 
6611 			key.af = pd2.af;
6612 			key.proto = IPPROTO_ICMP;
6613 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
6614 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
6615 			key.port[0] = key.port[1] = iih.icmp_id;
6616 
6617 			STATE_LOOKUP(kif, &key, *state, pd);
6618 
6619 			/* translate source/destination address, if necessary */
6620 			if ((*state)->key[PF_SK_WIRE] !=
6621 			    (*state)->key[PF_SK_STACK]) {
6622 				struct pf_state_key *nk =
6623 				    (*state)->key[pd->didx];
6624 
6625 				if (PF_ANEQ(pd2.src,
6626 				    &nk->addr[pd2.sidx], pd2.af) ||
6627 				    nk->port[pd2.sidx] != iih.icmp_id)
6628 					pf_change_icmp(pd2.src, &iih.icmp_id,
6629 					    daddr, &nk->addr[pd2.sidx],
6630 					    nk->port[pd2.sidx], NULL,
6631 					    pd2.ip_sum, icmpsum,
6632 					    pd->ip_sum, 0, AF_INET);
6633 
6634 				if (PF_ANEQ(pd2.dst,
6635 				    &nk->addr[pd2.didx], pd2.af) ||
6636 				    nk->port[pd2.didx] != iih.icmp_id)
6637 					pf_change_icmp(pd2.dst, &iih.icmp_id,
6638 					    saddr, &nk->addr[pd2.didx],
6639 					    nk->port[pd2.didx], NULL,
6640 					    pd2.ip_sum, icmpsum,
6641 					    pd->ip_sum, 0, AF_INET);
6642 
6643 				m_copyback(m, off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
6644 				m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
6645 				m_copyback(m, off2, ICMP_MINLEN, (caddr_t)&iih);
6646 			}
6647 			return (PF_PASS);
6648 			break;
6649 		}
6650 #endif /* INET */
6651 #ifdef INET6
6652 		case IPPROTO_ICMPV6: {
6653 			struct icmp6_hdr	iih;
6654 
6655 			if (!pf_pull_hdr(m, off2, &iih,
6656 			    sizeof(struct icmp6_hdr), NULL, reason, pd2.af)) {
6657 				DPFPRINTF(PF_DEBUG_MISC,
6658 				    ("pf: ICMP error message too short "
6659 				    "(icmp6)\n"));
6660 				return (PF_DROP);
6661 			}
6662 
6663 			key.af = pd2.af;
6664 			key.proto = IPPROTO_ICMPV6;
6665 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
6666 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
6667 			key.port[0] = key.port[1] = iih.icmp6_id;
6668 
6669 			STATE_LOOKUP(kif, &key, *state, pd);
6670 
6671 			/* translate source/destination address, if necessary */
6672 			if ((*state)->key[PF_SK_WIRE] !=
6673 			    (*state)->key[PF_SK_STACK]) {
6674 				struct pf_state_key *nk =
6675 				    (*state)->key[pd->didx];
6676 
6677 				if (PF_ANEQ(pd2.src,
6678 				    &nk->addr[pd2.sidx], pd2.af) ||
6679 				    nk->port[pd2.sidx] != iih.icmp6_id)
6680 					pf_change_icmp(pd2.src, &iih.icmp6_id,
6681 					    daddr, &nk->addr[pd2.sidx],
6682 					    nk->port[pd2.sidx], NULL,
6683 					    pd2.ip_sum, icmpsum,
6684 					    pd->ip_sum, 0, AF_INET6);
6685 
6686 				if (PF_ANEQ(pd2.dst,
6687 				    &nk->addr[pd2.didx], pd2.af) ||
6688 				    nk->port[pd2.didx] != iih.icmp6_id)
6689 					pf_change_icmp(pd2.dst, &iih.icmp6_id,
6690 					    saddr, &nk->addr[pd2.didx],
6691 					    nk->port[pd2.didx], NULL,
6692 					    pd2.ip_sum, icmpsum,
6693 					    pd->ip_sum, 0, AF_INET6);
6694 
6695 				m_copyback(m, off, sizeof(struct icmp6_hdr),
6696 				    (caddr_t)&pd->hdr.icmp6);
6697 				m_copyback(m, ipoff2, sizeof(h2_6), (caddr_t)&h2_6);
6698 				m_copyback(m, off2, sizeof(struct icmp6_hdr),
6699 				    (caddr_t)&iih);
6700 			}
6701 			return (PF_PASS);
6702 			break;
6703 		}
6704 #endif /* INET6 */
6705 		default: {
6706 			key.af = pd2.af;
6707 			key.proto = pd2.proto;
6708 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
6709 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
6710 			key.port[0] = key.port[1] = 0;
6711 
6712 			STATE_LOOKUP(kif, &key, *state, pd);
6713 
6714 			/* translate source/destination address, if necessary */
6715 			if ((*state)->key[PF_SK_WIRE] !=
6716 			    (*state)->key[PF_SK_STACK]) {
6717 				struct pf_state_key *nk =
6718 				    (*state)->key[pd->didx];
6719 
6720 				if (PF_ANEQ(pd2.src,
6721 				    &nk->addr[pd2.sidx], pd2.af))
6722 					pf_change_icmp(pd2.src, NULL, daddr,
6723 					    &nk->addr[pd2.sidx], 0, NULL,
6724 					    pd2.ip_sum, icmpsum,
6725 					    pd->ip_sum, 0, pd2.af);
6726 
6727 				if (PF_ANEQ(pd2.dst,
6728 				    &nk->addr[pd2.didx], pd2.af))
6729 					pf_change_icmp(pd2.dst, NULL, saddr,
6730 					    &nk->addr[pd2.didx], 0, NULL,
6731 					    pd2.ip_sum, icmpsum,
6732 					    pd->ip_sum, 0, pd2.af);
6733 
6734 				switch (pd2.af) {
6735 #ifdef INET
6736 				case AF_INET:
6737 					m_copyback(m, off, ICMP_MINLEN,
6738 					    (caddr_t)&pd->hdr.icmp);
6739 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
6740 					break;
6741 #endif /* INET */
6742 #ifdef INET6
6743 				case AF_INET6:
6744 					m_copyback(m, off,
6745 					    sizeof(struct icmp6_hdr),
6746 					    (caddr_t )&pd->hdr.icmp6);
6747 					m_copyback(m, ipoff2, sizeof(h2_6),
6748 					    (caddr_t )&h2_6);
6749 					break;
6750 #endif /* INET6 */
6751 				}
6752 			}
6753 			return (PF_PASS);
6754 			break;
6755 		}
6756 		}
6757 	}
6758 }
6759 
6760 static int
6761 pf_test_state_other(struct pf_kstate **state, struct pfi_kkif *kif,
6762     struct mbuf *m, struct pf_pdesc *pd)
6763 {
6764 	struct pf_state_peer	*src, *dst;
6765 	struct pf_state_key_cmp	 key;
6766 	uint8_t			 psrc, pdst;
6767 
6768 	bzero(&key, sizeof(key));
6769 	key.af = pd->af;
6770 	key.proto = pd->proto;
6771 	if (pd->dir == PF_IN)	{
6772 		PF_ACPY(&key.addr[0], pd->src, key.af);
6773 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6774 		key.port[0] = key.port[1] = 0;
6775 	} else {
6776 		PF_ACPY(&key.addr[1], pd->src, key.af);
6777 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6778 		key.port[1] = key.port[0] = 0;
6779 	}
6780 
6781 	STATE_LOOKUP(kif, &key, *state, pd);
6782 
6783 	if (pd->dir == (*state)->direction) {
6784 		src = &(*state)->src;
6785 		dst = &(*state)->dst;
6786 		psrc = PF_PEER_SRC;
6787 		pdst = PF_PEER_DST;
6788 	} else {
6789 		src = &(*state)->dst;
6790 		dst = &(*state)->src;
6791 		psrc = PF_PEER_DST;
6792 		pdst = PF_PEER_SRC;
6793 	}
6794 
6795 	/* update states */
6796 	if (src->state < PFOTHERS_SINGLE)
6797 		pf_set_protostate(*state, psrc, PFOTHERS_SINGLE);
6798 	if (dst->state == PFOTHERS_SINGLE)
6799 		pf_set_protostate(*state, pdst, PFOTHERS_MULTIPLE);
6800 
6801 	/* update expire time */
6802 	(*state)->expire = time_uptime;
6803 	if (src->state == PFOTHERS_MULTIPLE && dst->state == PFOTHERS_MULTIPLE)
6804 		(*state)->timeout = PFTM_OTHER_MULTIPLE;
6805 	else
6806 		(*state)->timeout = PFTM_OTHER_SINGLE;
6807 
6808 	/* translate source/destination address, if necessary */
6809 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6810 		struct pf_state_key *nk = (*state)->key[pd->didx];
6811 
6812 		KASSERT(nk, ("%s: nk is null", __func__));
6813 		KASSERT(pd, ("%s: pd is null", __func__));
6814 		KASSERT(pd->src, ("%s: pd->src is null", __func__));
6815 		KASSERT(pd->dst, ("%s: pd->dst is null", __func__));
6816 		switch (pd->af) {
6817 #ifdef INET
6818 		case AF_INET:
6819 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET))
6820 				pf_change_a(&pd->src->v4.s_addr,
6821 				    pd->ip_sum,
6822 				    nk->addr[pd->sidx].v4.s_addr,
6823 				    0);
6824 
6825 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET))
6826 				pf_change_a(&pd->dst->v4.s_addr,
6827 				    pd->ip_sum,
6828 				    nk->addr[pd->didx].v4.s_addr,
6829 				    0);
6830 
6831 			break;
6832 #endif /* INET */
6833 #ifdef INET6
6834 		case AF_INET6:
6835 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET))
6836 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
6837 
6838 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET))
6839 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
6840 #endif /* INET6 */
6841 		}
6842 	}
6843 	return (PF_PASS);
6844 }
6845 
6846 /*
6847  * ipoff and off are measured from the start of the mbuf chain.
6848  * h must be at "ipoff" on the mbuf chain.
6849  */
6850 void *
6851 pf_pull_hdr(struct mbuf *m, int off, void *p, int len,
6852     u_short *actionp, u_short *reasonp, sa_family_t af)
6853 {
6854 	switch (af) {
6855 #ifdef INET
6856 	case AF_INET: {
6857 		struct ip	*h = mtod(m, struct ip *);
6858 		u_int16_t	 fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3;
6859 
6860 		if (fragoff) {
6861 			if (fragoff >= len)
6862 				ACTION_SET(actionp, PF_PASS);
6863 			else {
6864 				ACTION_SET(actionp, PF_DROP);
6865 				REASON_SET(reasonp, PFRES_FRAG);
6866 			}
6867 			return (NULL);
6868 		}
6869 		if (m->m_pkthdr.len < off + len ||
6870 		    ntohs(h->ip_len) < off + len) {
6871 			ACTION_SET(actionp, PF_DROP);
6872 			REASON_SET(reasonp, PFRES_SHORT);
6873 			return (NULL);
6874 		}
6875 		break;
6876 	}
6877 #endif /* INET */
6878 #ifdef INET6
6879 	case AF_INET6: {
6880 		struct ip6_hdr	*h = mtod(m, struct ip6_hdr *);
6881 
6882 		if (m->m_pkthdr.len < off + len ||
6883 		    (ntohs(h->ip6_plen) + sizeof(struct ip6_hdr)) <
6884 		    (unsigned)(off + len)) {
6885 			ACTION_SET(actionp, PF_DROP);
6886 			REASON_SET(reasonp, PFRES_SHORT);
6887 			return (NULL);
6888 		}
6889 		break;
6890 	}
6891 #endif /* INET6 */
6892 	}
6893 	m_copydata(m, off, len, p);
6894 	return (p);
6895 }
6896 
6897 int
6898 pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kkif *kif,
6899     int rtableid)
6900 {
6901 	struct ifnet		*ifp;
6902 
6903 	/*
6904 	 * Skip check for addresses with embedded interface scope,
6905 	 * as they would always match anyway.
6906 	 */
6907 	if (af == AF_INET6 && IN6_IS_SCOPE_EMBED(&addr->v6))
6908 		return (1);
6909 
6910 	if (af != AF_INET && af != AF_INET6)
6911 		return (0);
6912 
6913 	/* Skip checks for ipsec interfaces */
6914 	if (kif != NULL && kif->pfik_ifp->if_type == IFT_ENC)
6915 		return (1);
6916 
6917 	ifp = (kif != NULL) ? kif->pfik_ifp : NULL;
6918 
6919 	switch (af) {
6920 #ifdef INET6
6921 	case AF_INET6:
6922 		return (fib6_check_urpf(rtableid, &addr->v6, 0, NHR_NONE,
6923 		    ifp));
6924 #endif
6925 #ifdef INET
6926 	case AF_INET:
6927 		return (fib4_check_urpf(rtableid, addr->v4, 0, NHR_NONE,
6928 		    ifp));
6929 #endif
6930 	}
6931 
6932 	return (0);
6933 }
6934 
6935 #ifdef INET
6936 static void
6937 pf_route(struct mbuf **m, struct pf_krule *r, struct ifnet *oifp,
6938     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
6939 {
6940 	struct mbuf		*m0, *m1, *md;
6941 	struct sockaddr_in	dst;
6942 	struct ip		*ip;
6943 	struct pfi_kkif		*nkif = NULL;
6944 	struct ifnet		*ifp = NULL;
6945 	struct pf_addr		 naddr;
6946 	struct pf_ksrc_node	*sn = NULL;
6947 	int			 error = 0;
6948 	uint16_t		 ip_len, ip_off;
6949 	int			 r_rt, r_dir;
6950 
6951 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
6952 
6953 	if (s) {
6954 		r_rt = s->rt;
6955 		r_dir = s->direction;
6956 	} else {
6957 		r_rt = r->rt;
6958 		r_dir = r->direction;
6959 	}
6960 
6961 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
6962 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
6963 	    __func__));
6964 
6965 	if ((pd->pf_mtag == NULL &&
6966 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
6967 	    pd->pf_mtag->routed++ > 3) {
6968 		m0 = *m;
6969 		*m = NULL;
6970 		goto bad_locked;
6971 	}
6972 
6973 	if (r_rt == PF_DUPTO) {
6974 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
6975 			if (s == NULL) {
6976 				ifp = r->rpool.cur->kif ?
6977 				    r->rpool.cur->kif->pfik_ifp : NULL;
6978 			} else {
6979 				ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
6980 				/* If pfsync'd */
6981 				if (ifp == NULL && r->rpool.cur != NULL)
6982 					ifp = r->rpool.cur->kif ?
6983 					    r->rpool.cur->kif->pfik_ifp : NULL;
6984 				PF_STATE_UNLOCK(s);
6985 			}
6986 			if (ifp == oifp) {
6987 				/* When the 2nd interface is not skipped */
6988 				return;
6989 			} else {
6990 				m0 = *m;
6991 				*m = NULL;
6992 				goto bad;
6993 			}
6994 		} else {
6995 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
6996 			if (((m0 = m_dup(*m, M_NOWAIT)) == NULL)) {
6997 				if (s)
6998 					PF_STATE_UNLOCK(s);
6999 				return;
7000 			}
7001 		}
7002 	} else {
7003 		if ((r_rt == PF_REPLYTO) == (r_dir == pd->dir)) {
7004 			pf_dummynet(pd, s, r, m);
7005 			if (s)
7006 				PF_STATE_UNLOCK(s);
7007 			return;
7008 		}
7009 		m0 = *m;
7010 	}
7011 
7012 	ip = mtod(m0, struct ip *);
7013 
7014 	bzero(&dst, sizeof(dst));
7015 	dst.sin_family = AF_INET;
7016 	dst.sin_len = sizeof(dst);
7017 	dst.sin_addr = ip->ip_dst;
7018 
7019 	bzero(&naddr, sizeof(naddr));
7020 
7021 	if (s == NULL) {
7022 		if (TAILQ_EMPTY(&r->rpool.list)) {
7023 			DPFPRINTF(PF_DEBUG_URGENT,
7024 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
7025 			goto bad_locked;
7026 		}
7027 		pf_map_addr(AF_INET, r, (struct pf_addr *)&ip->ip_src,
7028 		    &naddr, &nkif, NULL, &sn);
7029 		if (!PF_AZERO(&naddr, AF_INET))
7030 			dst.sin_addr.s_addr = naddr.v4.s_addr;
7031 		ifp = nkif ? nkif->pfik_ifp : NULL;
7032 	} else {
7033 		if (!PF_AZERO(&s->rt_addr, AF_INET))
7034 			dst.sin_addr.s_addr =
7035 			    s->rt_addr.v4.s_addr;
7036 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7037 		/* If pfsync'd */
7038 		if (ifp == NULL && r->rpool.cur != NULL) {
7039 			ifp = r->rpool.cur->kif ?
7040 			    r->rpool.cur->kif->pfik_ifp : NULL;
7041 		}
7042 		PF_STATE_UNLOCK(s);
7043 	}
7044 
7045 	if (ifp == NULL)
7046 		goto bad;
7047 
7048 	if (pd->dir == PF_IN) {
7049 		if (pf_test(PF_OUT, 0, ifp, &m0, inp, &pd->act) != PF_PASS)
7050 			goto bad;
7051 		else if (m0 == NULL)
7052 			goto done;
7053 		if (m0->m_len < sizeof(struct ip)) {
7054 			DPFPRINTF(PF_DEBUG_URGENT,
7055 			    ("%s: m0->m_len < sizeof(struct ip)\n", __func__));
7056 			goto bad;
7057 		}
7058 		ip = mtod(m0, struct ip *);
7059 	}
7060 
7061 	if (ifp->if_flags & IFF_LOOPBACK)
7062 		m0->m_flags |= M_SKIP_FIREWALL;
7063 
7064 	ip_len = ntohs(ip->ip_len);
7065 	ip_off = ntohs(ip->ip_off);
7066 
7067 	/* Copied from FreeBSD 10.0-CURRENT ip_output. */
7068 	m0->m_pkthdr.csum_flags |= CSUM_IP;
7069 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~ifp->if_hwassist) {
7070 		in_delayed_cksum(m0);
7071 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
7072 	}
7073 	if (m0->m_pkthdr.csum_flags & CSUM_SCTP & ~ifp->if_hwassist) {
7074 		pf_sctp_checksum(m0, (uint32_t)(ip->ip_hl << 2));
7075 		m0->m_pkthdr.csum_flags &= ~CSUM_SCTP;
7076 	}
7077 
7078 	/*
7079 	 * If small enough for interface, or the interface will take
7080 	 * care of the fragmentation for us, we can just send directly.
7081 	 */
7082 	if (ip_len <= ifp->if_mtu ||
7083 	    (m0->m_pkthdr.csum_flags & ifp->if_hwassist & CSUM_TSO) != 0) {
7084 		ip->ip_sum = 0;
7085 		if (m0->m_pkthdr.csum_flags & CSUM_IP & ~ifp->if_hwassist) {
7086 			ip->ip_sum = in_cksum(m0, ip->ip_hl << 2);
7087 			m0->m_pkthdr.csum_flags &= ~CSUM_IP;
7088 		}
7089 		m_clrprotoflags(m0);	/* Avoid confusing lower layers. */
7090 
7091 		md = m0;
7092 		error = pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
7093 		if (md != NULL)
7094 			error = (*ifp->if_output)(ifp, md, sintosa(&dst), NULL);
7095 		goto done;
7096 	}
7097 
7098 	/* Balk when DF bit is set or the interface didn't support TSO. */
7099 	if ((ip_off & IP_DF) || (m0->m_pkthdr.csum_flags & CSUM_TSO)) {
7100 		error = EMSGSIZE;
7101 		KMOD_IPSTAT_INC(ips_cantfrag);
7102 		if (r_rt != PF_DUPTO) {
7103 			if (s && pd->nat_rule != NULL)
7104 				PACKET_UNDO_NAT(m0, pd,
7105 				    (ip->ip_hl << 2) + (ip_off & IP_OFFMASK),
7106 				    s);
7107 
7108 			icmp_error(m0, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG, 0,
7109 			    ifp->if_mtu);
7110 			goto done;
7111 		} else
7112 			goto bad;
7113 	}
7114 
7115 	error = ip_fragment(ip, &m0, ifp->if_mtu, ifp->if_hwassist);
7116 	if (error)
7117 		goto bad;
7118 
7119 	for (; m0; m0 = m1) {
7120 		m1 = m0->m_nextpkt;
7121 		m0->m_nextpkt = NULL;
7122 		if (error == 0) {
7123 			m_clrprotoflags(m0);
7124 			md = m0;
7125 			error = pf_dummynet_route(pd, s, r, ifp,
7126 			    sintosa(&dst), &md);
7127 			if (md != NULL)
7128 				error = (*ifp->if_output)(ifp, md,
7129 				    sintosa(&dst), NULL);
7130 		} else
7131 			m_freem(m0);
7132 	}
7133 
7134 	if (error == 0)
7135 		KMOD_IPSTAT_INC(ips_fragmented);
7136 
7137 done:
7138 	if (r_rt != PF_DUPTO)
7139 		*m = NULL;
7140 	return;
7141 
7142 bad_locked:
7143 	if (s)
7144 		PF_STATE_UNLOCK(s);
7145 bad:
7146 	m_freem(m0);
7147 	goto done;
7148 }
7149 #endif /* INET */
7150 
7151 #ifdef INET6
7152 static void
7153 pf_route6(struct mbuf **m, struct pf_krule *r, struct ifnet *oifp,
7154     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
7155 {
7156 	struct mbuf		*m0, *md;
7157 	struct sockaddr_in6	dst;
7158 	struct ip6_hdr		*ip6;
7159 	struct pfi_kkif		*nkif = NULL;
7160 	struct ifnet		*ifp = NULL;
7161 	struct pf_addr		 naddr;
7162 	struct pf_ksrc_node	*sn = NULL;
7163 	int			 r_rt, r_dir;
7164 
7165 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
7166 
7167 	if (s) {
7168 		r_rt = s->rt;
7169 		r_dir = s->direction;
7170 	} else {
7171 		r_rt = r->rt;
7172 		r_dir = r->direction;
7173 	}
7174 
7175 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
7176 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
7177 	    __func__));
7178 
7179 	if ((pd->pf_mtag == NULL &&
7180 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
7181 	    pd->pf_mtag->routed++ > 3) {
7182 		m0 = *m;
7183 		*m = NULL;
7184 		goto bad_locked;
7185 	}
7186 
7187 	if (r_rt == PF_DUPTO) {
7188 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
7189 			if (s == NULL) {
7190 				ifp = r->rpool.cur->kif ?
7191 				    r->rpool.cur->kif->pfik_ifp : NULL;
7192 			} else {
7193 				ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7194 				/* If pfsync'd */
7195 				if (ifp == NULL && r->rpool.cur != NULL)
7196 					ifp = r->rpool.cur->kif ?
7197 					    r->rpool.cur->kif->pfik_ifp : NULL;
7198 				PF_STATE_UNLOCK(s);
7199 			}
7200 			if (ifp == oifp) {
7201 				/* When the 2nd interface is not skipped */
7202 				return;
7203 			} else {
7204 				m0 = *m;
7205 				*m = NULL;
7206 				goto bad;
7207 			}
7208 		} else {
7209 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
7210 			if (((m0 = m_dup(*m, M_NOWAIT)) == NULL)) {
7211 				if (s)
7212 					PF_STATE_UNLOCK(s);
7213 				return;
7214 			}
7215 		}
7216 	} else {
7217 		if ((r_rt == PF_REPLYTO) == (r_dir == pd->dir)) {
7218 			pf_dummynet(pd, s, r, m);
7219 			if (s)
7220 				PF_STATE_UNLOCK(s);
7221 			return;
7222 		}
7223 		m0 = *m;
7224 	}
7225 
7226 	ip6 = mtod(m0, struct ip6_hdr *);
7227 
7228 	bzero(&dst, sizeof(dst));
7229 	dst.sin6_family = AF_INET6;
7230 	dst.sin6_len = sizeof(dst);
7231 	dst.sin6_addr = ip6->ip6_dst;
7232 
7233 	bzero(&naddr, sizeof(naddr));
7234 
7235 	if (s == NULL) {
7236 		if (TAILQ_EMPTY(&r->rpool.list)) {
7237 			DPFPRINTF(PF_DEBUG_URGENT,
7238 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
7239 			goto bad_locked;
7240 		}
7241 		pf_map_addr(AF_INET6, r, (struct pf_addr *)&ip6->ip6_src,
7242 		    &naddr, &nkif, NULL, &sn);
7243 		if (!PF_AZERO(&naddr, AF_INET6))
7244 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
7245 			    &naddr, AF_INET6);
7246 		ifp = nkif ? nkif->pfik_ifp : NULL;
7247 	} else {
7248 		if (!PF_AZERO(&s->rt_addr, AF_INET6))
7249 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
7250 			    &s->rt_addr, AF_INET6);
7251 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7252 		/* If pfsync'd */
7253 		if (ifp == NULL && r->rpool.cur != NULL)
7254 			ifp = r->rpool.cur->kif ?
7255 			    r->rpool.cur->kif->pfik_ifp : NULL;
7256 	}
7257 
7258 	if (s)
7259 		PF_STATE_UNLOCK(s);
7260 
7261 	if (ifp == NULL)
7262 		goto bad;
7263 
7264 	if (pd->dir == PF_IN) {
7265 		if (pf_test6(PF_OUT, 0, ifp, &m0, inp, &pd->act) != PF_PASS)
7266 			goto bad;
7267 		else if (m0 == NULL)
7268 			goto done;
7269 		if (m0->m_len < sizeof(struct ip6_hdr)) {
7270 			DPFPRINTF(PF_DEBUG_URGENT,
7271 			    ("%s: m0->m_len < sizeof(struct ip6_hdr)\n",
7272 			    __func__));
7273 			goto bad;
7274 		}
7275 		ip6 = mtod(m0, struct ip6_hdr *);
7276 	}
7277 
7278 	if (ifp->if_flags & IFF_LOOPBACK)
7279 		m0->m_flags |= M_SKIP_FIREWALL;
7280 
7281 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 &
7282 	    ~ifp->if_hwassist) {
7283 		uint32_t plen = m0->m_pkthdr.len - sizeof(*ip6);
7284 		in6_delayed_cksum(m0, plen, sizeof(struct ip6_hdr));
7285 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
7286 	}
7287 
7288 	/*
7289 	 * If the packet is too large for the outgoing interface,
7290 	 * send back an icmp6 error.
7291 	 */
7292 	if (IN6_IS_SCOPE_EMBED(&dst.sin6_addr))
7293 		dst.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
7294 	if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu) {
7295 		md = m0;
7296 		pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
7297 		if (md != NULL)
7298 			nd6_output_ifp(ifp, ifp, md, &dst, NULL);
7299 	}
7300 	else {
7301 		in6_ifstat_inc(ifp, ifs6_in_toobig);
7302 		if (r_rt != PF_DUPTO) {
7303 			if (s && pd->nat_rule != NULL)
7304 				PACKET_UNDO_NAT(m0, pd,
7305 				    ((caddr_t)ip6 - m0->m_data) +
7306 				    sizeof(struct ip6_hdr), s);
7307 
7308 			icmp6_error(m0, ICMP6_PACKET_TOO_BIG, 0, ifp->if_mtu);
7309 		} else
7310 			goto bad;
7311 	}
7312 
7313 done:
7314 	if (r_rt != PF_DUPTO)
7315 		*m = NULL;
7316 	return;
7317 
7318 bad_locked:
7319 	if (s)
7320 		PF_STATE_UNLOCK(s);
7321 bad:
7322 	m_freem(m0);
7323 	goto done;
7324 }
7325 #endif /* INET6 */
7326 
7327 /*
7328  * FreeBSD supports cksum offloads for the following drivers.
7329  *  em(4), fxp(4), lge(4), nge(4), re(4), ti(4), txp(4), xl(4)
7330  *
7331  * CSUM_DATA_VALID | CSUM_PSEUDO_HDR :
7332  *  network driver performed cksum including pseudo header, need to verify
7333  *   csum_data
7334  * CSUM_DATA_VALID :
7335  *  network driver performed cksum, needs to additional pseudo header
7336  *  cksum computation with partial csum_data(i.e. lack of H/W support for
7337  *  pseudo header, for instance sk(4) and possibly gem(4))
7338  *
7339  * After validating the cksum of packet, set both flag CSUM_DATA_VALID and
7340  * CSUM_PSEUDO_HDR in order to avoid recomputation of the cksum in upper
7341  * TCP/UDP layer.
7342  * Also, set csum_data to 0xffff to force cksum validation.
7343  */
7344 static int
7345 pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t af)
7346 {
7347 	u_int16_t sum = 0;
7348 	int hw_assist = 0;
7349 	struct ip *ip;
7350 
7351 	if (off < sizeof(struct ip) || len < sizeof(struct udphdr))
7352 		return (1);
7353 	if (m->m_pkthdr.len < off + len)
7354 		return (1);
7355 
7356 	switch (p) {
7357 	case IPPROTO_TCP:
7358 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
7359 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
7360 				sum = m->m_pkthdr.csum_data;
7361 			} else {
7362 				ip = mtod(m, struct ip *);
7363 				sum = in_pseudo(ip->ip_src.s_addr,
7364 				ip->ip_dst.s_addr, htonl((u_short)len +
7365 				m->m_pkthdr.csum_data + IPPROTO_TCP));
7366 			}
7367 			sum ^= 0xffff;
7368 			++hw_assist;
7369 		}
7370 		break;
7371 	case IPPROTO_UDP:
7372 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
7373 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
7374 				sum = m->m_pkthdr.csum_data;
7375 			} else {
7376 				ip = mtod(m, struct ip *);
7377 				sum = in_pseudo(ip->ip_src.s_addr,
7378 				ip->ip_dst.s_addr, htonl((u_short)len +
7379 				m->m_pkthdr.csum_data + IPPROTO_UDP));
7380 			}
7381 			sum ^= 0xffff;
7382 			++hw_assist;
7383 		}
7384 		break;
7385 	case IPPROTO_ICMP:
7386 #ifdef INET6
7387 	case IPPROTO_ICMPV6:
7388 #endif /* INET6 */
7389 		break;
7390 	default:
7391 		return (1);
7392 	}
7393 
7394 	if (!hw_assist) {
7395 		switch (af) {
7396 		case AF_INET:
7397 			if (p == IPPROTO_ICMP) {
7398 				if (m->m_len < off)
7399 					return (1);
7400 				m->m_data += off;
7401 				m->m_len -= off;
7402 				sum = in_cksum(m, len);
7403 				m->m_data -= off;
7404 				m->m_len += off;
7405 			} else {
7406 				if (m->m_len < sizeof(struct ip))
7407 					return (1);
7408 				sum = in4_cksum(m, p, off, len);
7409 			}
7410 			break;
7411 #ifdef INET6
7412 		case AF_INET6:
7413 			if (m->m_len < sizeof(struct ip6_hdr))
7414 				return (1);
7415 			sum = in6_cksum(m, p, off, len);
7416 			break;
7417 #endif /* INET6 */
7418 		default:
7419 			return (1);
7420 		}
7421 	}
7422 	if (sum) {
7423 		switch (p) {
7424 		case IPPROTO_TCP:
7425 		    {
7426 			KMOD_TCPSTAT_INC(tcps_rcvbadsum);
7427 			break;
7428 		    }
7429 		case IPPROTO_UDP:
7430 		    {
7431 			KMOD_UDPSTAT_INC(udps_badsum);
7432 			break;
7433 		    }
7434 #ifdef INET
7435 		case IPPROTO_ICMP:
7436 		    {
7437 			KMOD_ICMPSTAT_INC(icps_checksum);
7438 			break;
7439 		    }
7440 #endif
7441 #ifdef INET6
7442 		case IPPROTO_ICMPV6:
7443 		    {
7444 			KMOD_ICMP6STAT_INC(icp6s_checksum);
7445 			break;
7446 		    }
7447 #endif /* INET6 */
7448 		}
7449 		return (1);
7450 	} else {
7451 		if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
7452 			m->m_pkthdr.csum_flags |=
7453 			    (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
7454 			m->m_pkthdr.csum_data = 0xffff;
7455 		}
7456 	}
7457 	return (0);
7458 }
7459 
7460 static bool
7461 pf_pdesc_to_dnflow(const struct pf_pdesc *pd, const struct pf_krule *r,
7462     const struct pf_kstate *s, struct ip_fw_args *dnflow)
7463 {
7464 	int dndir = r->direction;
7465 
7466 	if (s && dndir == PF_INOUT) {
7467 		dndir = s->direction;
7468 	} else if (dndir == PF_INOUT) {
7469 		/* Assume primary direction. Happens when we've set dnpipe in
7470 		 * the ethernet level code. */
7471 		dndir = pd->dir;
7472 	}
7473 
7474 	memset(dnflow, 0, sizeof(*dnflow));
7475 
7476 	if (pd->dport != NULL)
7477 		dnflow->f_id.dst_port = ntohs(*pd->dport);
7478 	if (pd->sport != NULL)
7479 		dnflow->f_id.src_port = ntohs(*pd->sport);
7480 
7481 	if (pd->dir == PF_IN)
7482 		dnflow->flags |= IPFW_ARGS_IN;
7483 	else
7484 		dnflow->flags |= IPFW_ARGS_OUT;
7485 
7486 	if (pd->dir != dndir && pd->act.dnrpipe) {
7487 		dnflow->rule.info = pd->act.dnrpipe;
7488 	}
7489 	else if (pd->dir == dndir && pd->act.dnpipe) {
7490 		dnflow->rule.info = pd->act.dnpipe;
7491 	}
7492 	else {
7493 		return (false);
7494 	}
7495 
7496 	dnflow->rule.info |= IPFW_IS_DUMMYNET;
7497 	if (r->free_flags & PFRULE_DN_IS_PIPE || pd->act.flags & PFSTATE_DN_IS_PIPE)
7498 		dnflow->rule.info |= IPFW_IS_PIPE;
7499 
7500 	dnflow->f_id.proto = pd->proto;
7501 	dnflow->f_id.extra = dnflow->rule.info;
7502 	switch (pd->af) {
7503 	case AF_INET:
7504 		dnflow->f_id.addr_type = 4;
7505 		dnflow->f_id.src_ip = ntohl(pd->src->v4.s_addr);
7506 		dnflow->f_id.dst_ip = ntohl(pd->dst->v4.s_addr);
7507 		break;
7508 	case AF_INET6:
7509 		dnflow->flags |= IPFW_ARGS_IP6;
7510 		dnflow->f_id.addr_type = 6;
7511 		dnflow->f_id.src_ip6 = pd->src->v6;
7512 		dnflow->f_id.dst_ip6 = pd->dst->v6;
7513 		break;
7514 	default:
7515 		panic("Invalid AF");
7516 		break;
7517 	}
7518 
7519 	return (true);
7520 }
7521 
7522 int
7523 pf_test_eth(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
7524     struct inpcb *inp)
7525 {
7526 	struct pfi_kkif		*kif;
7527 	struct mbuf		*m = *m0;
7528 
7529 	M_ASSERTPKTHDR(m);
7530 	MPASS(ifp->if_vnet == curvnet);
7531 	NET_EPOCH_ASSERT();
7532 
7533 	if (!V_pf_status.running)
7534 		return (PF_PASS);
7535 
7536 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
7537 
7538 	if (kif == NULL) {
7539 		DPFPRINTF(PF_DEBUG_URGENT,
7540 		    ("%s: kif == NULL, if_xname %s\n", __func__, ifp->if_xname));
7541 		return (PF_DROP);
7542 	}
7543 	if (kif->pfik_flags & PFI_IFLAG_SKIP)
7544 		return (PF_PASS);
7545 
7546 	if (m->m_flags & M_SKIP_FIREWALL)
7547 		return (PF_PASS);
7548 
7549 	/* Stateless! */
7550 	return (pf_test_eth_rule(dir, kif, m0));
7551 }
7552 
7553 static int
7554 pf_dummynet(struct pf_pdesc *pd, struct pf_kstate *s,
7555     struct pf_krule *r, struct mbuf **m0)
7556 {
7557 	return (pf_dummynet_route(pd, s, r, NULL, NULL, m0));
7558 }
7559 
7560 static int
7561 pf_dummynet_route(struct pf_pdesc *pd, struct pf_kstate *s,
7562     struct pf_krule *r, struct ifnet *ifp, struct sockaddr *sa,
7563     struct mbuf **m0)
7564 {
7565 	NET_EPOCH_ASSERT();
7566 
7567 	if (pd->act.dnpipe || pd->act.dnrpipe) {
7568 		struct ip_fw_args dnflow;
7569 		if (ip_dn_io_ptr == NULL) {
7570 			m_freem(*m0);
7571 			*m0 = NULL;
7572 			return (ENOMEM);
7573 		}
7574 
7575 		if (pd->pf_mtag == NULL &&
7576 		    ((pd->pf_mtag = pf_get_mtag(*m0)) == NULL)) {
7577 			m_freem(*m0);
7578 			*m0 = NULL;
7579 			return (ENOMEM);
7580 		}
7581 
7582 		if (ifp != NULL) {
7583 			pd->pf_mtag->flags |= PF_MTAG_FLAG_ROUTE_TO;
7584 
7585 			pd->pf_mtag->if_index = ifp->if_index;
7586 			pd->pf_mtag->if_idxgen = ifp->if_idxgen;
7587 
7588 			MPASS(sa != NULL);
7589 
7590 			if (pd->af == AF_INET)
7591 				memcpy(&pd->pf_mtag->dst, sa,
7592 				    sizeof(struct sockaddr_in));
7593 			else
7594 				memcpy(&pd->pf_mtag->dst, sa,
7595 				    sizeof(struct sockaddr_in6));
7596 		}
7597 
7598 		if (pf_pdesc_to_dnflow(pd, r, s, &dnflow)) {
7599 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
7600 			ip_dn_io_ptr(m0, &dnflow);
7601 			if (*m0 != NULL) {
7602 				pd->pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
7603 				pd->pf_mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
7604 			}
7605 		}
7606 	}
7607 
7608 	return (0);
7609 }
7610 
7611 #ifdef INET
7612 int
7613 pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
7614     struct inpcb *inp, struct pf_rule_actions *default_actions)
7615 {
7616 	struct pfi_kkif		*kif;
7617 	u_short			 action, reason = 0;
7618 	struct mbuf		*m = *m0;
7619 	struct ip		*h = NULL;
7620 	struct m_tag		*ipfwtag;
7621 	struct pf_krule		*a = NULL, *r = &V_pf_default_rule, *tr, *nr;
7622 	struct pf_kstate	*s = NULL;
7623 	struct pf_kruleset	*ruleset = NULL;
7624 	struct pf_pdesc		 pd;
7625 	int			 off, dirndx, use_2nd_queue = 0;
7626 	uint16_t		 tag;
7627 	uint8_t			 rt;
7628 
7629 	PF_RULES_RLOCK_TRACKER;
7630 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: bad direction %d\n", __func__, dir));
7631 	M_ASSERTPKTHDR(m);
7632 
7633 	if (!V_pf_status.running)
7634 		return (PF_PASS);
7635 
7636 	PF_RULES_RLOCK();
7637 
7638 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
7639 
7640 	if (__predict_false(kif == NULL)) {
7641 		DPFPRINTF(PF_DEBUG_URGENT,
7642 		    ("pf_test: kif == NULL, if_xname %s\n", ifp->if_xname));
7643 		PF_RULES_RUNLOCK();
7644 		return (PF_DROP);
7645 	}
7646 	if (kif->pfik_flags & PFI_IFLAG_SKIP) {
7647 		PF_RULES_RUNLOCK();
7648 		return (PF_PASS);
7649 	}
7650 
7651 	if (m->m_flags & M_SKIP_FIREWALL) {
7652 		PF_RULES_RUNLOCK();
7653 		return (PF_PASS);
7654 	}
7655 
7656 	memset(&pd, 0, sizeof(pd));
7657 	TAILQ_INIT(&pd.sctp_multihome_jobs);
7658 	if (default_actions != NULL)
7659 		memcpy(&pd.act, default_actions, sizeof(pd.act));
7660 	pd.pf_mtag = pf_find_mtag(m);
7661 
7662 	if (pd.pf_mtag != NULL && (pd.pf_mtag->flags & PF_MTAG_FLAG_ROUTE_TO)) {
7663 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
7664 
7665 		ifp = ifnet_byindexgen(pd.pf_mtag->if_index,
7666 		    pd.pf_mtag->if_idxgen);
7667 		if (ifp == NULL || ifp->if_flags & IFF_DYING) {
7668 			PF_RULES_RUNLOCK();
7669 			m_freem(*m0);
7670 			*m0 = NULL;
7671 			return (PF_PASS);
7672 		}
7673 		PF_RULES_RUNLOCK();
7674 		(ifp->if_output)(ifp, m, sintosa(&pd.pf_mtag->dst), NULL);
7675 		*m0 = NULL;
7676 		return (PF_PASS);
7677 	}
7678 
7679 	if (pd.pf_mtag && pd.pf_mtag->dnpipe) {
7680 		pd.act.dnpipe = pd.pf_mtag->dnpipe;
7681 		pd.act.flags = pd.pf_mtag->dnflags;
7682 	}
7683 
7684 	if (ip_dn_io_ptr != NULL && pd.pf_mtag != NULL &&
7685 	    pd.pf_mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
7686 		/* Dummynet re-injects packets after they've
7687 		 * completed their delay. We've already
7688 		 * processed them, so pass unconditionally. */
7689 
7690 		/* But only once. We may see the packet multiple times (e.g.
7691 		 * PFIL_IN/PFIL_OUT). */
7692 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
7693 		PF_RULES_RUNLOCK();
7694 
7695 		return (PF_PASS);
7696 	}
7697 
7698 	pd.sport = pd.dport = NULL;
7699 	pd.proto_sum = NULL;
7700 	pd.dir = dir;
7701 	pd.sidx = (dir == PF_IN) ? 0 : 1;
7702 	pd.didx = (dir == PF_IN) ? 1 : 0;
7703 	pd.af = AF_INET;
7704 	pd.act.rtableid = -1;
7705 
7706 	h = mtod(m, struct ip *);
7707 	off = h->ip_hl << 2;
7708 
7709 	if (__predict_false(ip_divert_ptr != NULL) &&
7710 	    ((ipfwtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL)) != NULL)) {
7711 		struct ipfw_rule_ref *rr = (struct ipfw_rule_ref *)(ipfwtag+1);
7712 		if (rr->info & IPFW_IS_DIVERT && rr->rulenum == 0) {
7713 			if (pd.pf_mtag == NULL &&
7714 			    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
7715 				action = PF_DROP;
7716 				goto done;
7717 			}
7718 			pd.pf_mtag->flags |= PF_MTAG_FLAG_PACKET_LOOPED;
7719 			m_tag_delete(m, ipfwtag);
7720 		}
7721 		if (pd.pf_mtag && pd.pf_mtag->flags & PF_MTAG_FLAG_FASTFWD_OURS_PRESENT) {
7722 			m->m_flags |= M_FASTFWD_OURS;
7723 			pd.pf_mtag->flags &= ~PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
7724 		}
7725 	} else if (pf_normalize_ip(m0, kif, &reason, &pd) != PF_PASS) {
7726 		/* We do IP header normalization and packet reassembly here */
7727 		action = PF_DROP;
7728 		goto done;
7729 	}
7730 	m = *m0;	/* pf_normalize messes with m0 */
7731 	h = mtod(m, struct ip *);
7732 
7733 	off = h->ip_hl << 2;
7734 	if (off < (int)sizeof(struct ip)) {
7735 		action = PF_DROP;
7736 		REASON_SET(&reason, PFRES_SHORT);
7737 		pd.act.log = PF_LOG_FORCE;
7738 		goto done;
7739 	}
7740 
7741 	pd.src = (struct pf_addr *)&h->ip_src;
7742 	pd.dst = (struct pf_addr *)&h->ip_dst;
7743 	pd.ip_sum = &h->ip_sum;
7744 	pd.proto = h->ip_p;
7745 	pd.tos = h->ip_tos & ~IPTOS_ECN_MASK;
7746 	pd.tot_len = ntohs(h->ip_len);
7747 
7748 	/* handle fragments that didn't get reassembled by normalization */
7749 	if (h->ip_off & htons(IP_MF | IP_OFFMASK)) {
7750 		action = pf_test_fragment(&r, kif, m, h, &pd, &a, &ruleset);
7751 		goto done;
7752 	}
7753 
7754 	switch (h->ip_p) {
7755 	case IPPROTO_TCP: {
7756 		if (!pf_pull_hdr(m, off, &pd.hdr.tcp, sizeof(pd.hdr.tcp),
7757 		    &action, &reason, AF_INET)) {
7758 			if (action != PF_PASS)
7759 				pd.act.log = PF_LOG_FORCE;
7760 			goto done;
7761 		}
7762 		pd.p_len = pd.tot_len - off - (pd.hdr.tcp.th_off << 2);
7763 
7764 		pd.sport = &pd.hdr.tcp.th_sport;
7765 		pd.dport = &pd.hdr.tcp.th_dport;
7766 
7767 		/* Respond to SYN with a syncookie. */
7768 		if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_SYN &&
7769 		    pd.dir == PF_IN && pf_synflood_check(&pd)) {
7770 			pf_syncookie_send(m, off, &pd);
7771 			action = PF_DROP;
7772 			break;
7773 		}
7774 
7775 		if ((pd.hdr.tcp.th_flags & TH_ACK) && pd.p_len == 0)
7776 			use_2nd_queue = 1;
7777 		action = pf_normalize_tcp(kif, m, 0, off, h, &pd);
7778 		if (action == PF_DROP)
7779 			goto done;
7780 		action = pf_test_state_tcp(&s, kif, m, off, h, &pd, &reason);
7781 		if (action == PF_PASS) {
7782 			if (V_pfsync_update_state_ptr != NULL)
7783 				V_pfsync_update_state_ptr(s);
7784 			r = s->rule.ptr;
7785 			a = s->anchor.ptr;
7786 		} else if (s == NULL) {
7787 			/* Validate remote SYN|ACK, re-create original SYN if
7788 			 * valid. */
7789 			if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) ==
7790 			    TH_ACK && pf_syncookie_validate(&pd) &&
7791 			    pd.dir == PF_IN) {
7792 				struct mbuf *msyn;
7793 
7794 				msyn = pf_syncookie_recreate_syn(h->ip_ttl, off,
7795 				    &pd);
7796 				if (msyn == NULL) {
7797 					action = PF_DROP;
7798 					break;
7799 				}
7800 
7801 				action = pf_test(dir, pflags, ifp, &msyn, inp,
7802 				    &pd.act);
7803 				m_freem(msyn);
7804 				if (action != PF_PASS)
7805 					break;
7806 
7807 				action = pf_test_state_tcp(&s, kif, m, off, h,
7808 				    &pd, &reason);
7809 				if (action != PF_PASS || s == NULL) {
7810 					action = PF_DROP;
7811 					break;
7812 				}
7813 
7814 				s->src.seqhi = ntohl(pd.hdr.tcp.th_ack) - 1;
7815 				s->src.seqlo = ntohl(pd.hdr.tcp.th_seq) - 1;
7816 				pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_DST);
7817 				action = pf_synproxy(&pd, &s, &reason);
7818 				break;
7819 			} else {
7820 				action = pf_test_rule(&r, &s, kif, m, off, &pd,
7821 				    &a, &ruleset, inp);
7822 			}
7823 		}
7824 		break;
7825 	}
7826 
7827 	case IPPROTO_UDP: {
7828 		if (!pf_pull_hdr(m, off, &pd.hdr.udp, sizeof(pd.hdr.udp),
7829 		    &action, &reason, AF_INET)) {
7830 			if (action != PF_PASS)
7831 				pd.act.log = PF_LOG_FORCE;
7832 			goto done;
7833 		}
7834 		pd.sport = &pd.hdr.udp.uh_sport;
7835 		pd.dport = &pd.hdr.udp.uh_dport;
7836 		if (pd.hdr.udp.uh_dport == 0 ||
7837 		    ntohs(pd.hdr.udp.uh_ulen) > m->m_pkthdr.len - off ||
7838 		    ntohs(pd.hdr.udp.uh_ulen) < sizeof(struct udphdr)) {
7839 			action = PF_DROP;
7840 			REASON_SET(&reason, PFRES_SHORT);
7841 			goto done;
7842 		}
7843 		action = pf_test_state_udp(&s, kif, m, off, h, &pd);
7844 		if (action == PF_PASS) {
7845 			if (V_pfsync_update_state_ptr != NULL)
7846 				V_pfsync_update_state_ptr(s);
7847 			r = s->rule.ptr;
7848 			a = s->anchor.ptr;
7849 		} else if (s == NULL)
7850 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
7851 			    &a, &ruleset, inp);
7852 		break;
7853 	}
7854 
7855 	case IPPROTO_SCTP: {
7856 		if (!pf_pull_hdr(m, off, &pd.hdr.sctp, sizeof(pd.hdr.sctp),
7857 		    &action, &reason, AF_INET)) {
7858 			if (action != PF_PASS)
7859 				pd.act.log |= PF_LOG_FORCE;
7860 			goto done;
7861 		}
7862 		pd.p_len = pd.tot_len - off;
7863 
7864 		pd.sport = &pd.hdr.sctp.src_port;
7865 		pd.dport = &pd.hdr.sctp.dest_port;
7866 		if (pd.hdr.sctp.src_port == 0 || pd.hdr.sctp.dest_port == 0) {
7867 			action = PF_DROP;
7868 			REASON_SET(&reason, PFRES_SHORT);
7869 			goto done;
7870 		}
7871 		action = pf_normalize_sctp(dir, kif, m, 0, off, h, &pd);
7872 		if (action == PF_DROP)
7873 			goto done;
7874 		action = pf_test_state_sctp(&s, kif, m, off, h, &pd,
7875 		    &reason);
7876 		if (action == PF_PASS) {
7877 			if (V_pfsync_update_state_ptr != NULL)
7878 				V_pfsync_update_state_ptr(s);
7879 			r = s->rule.ptr;
7880 			a = s->anchor.ptr;
7881 		} else {
7882 			action = pf_test_rule(&r, &s, kif, m, off,
7883 			    &pd, &a, &ruleset, inp);
7884 		}
7885 		break;
7886 	}
7887 
7888 	case IPPROTO_ICMP: {
7889 		if (!pf_pull_hdr(m, off, &pd.hdr.icmp, ICMP_MINLEN,
7890 		    &action, &reason, AF_INET)) {
7891 			if (action != PF_PASS)
7892 				pd.act.log = PF_LOG_FORCE;
7893 			goto done;
7894 		}
7895 		action = pf_test_state_icmp(&s, kif, m, off, h, &pd, &reason);
7896 		if (action == PF_PASS) {
7897 			if (V_pfsync_update_state_ptr != NULL)
7898 				V_pfsync_update_state_ptr(s);
7899 			r = s->rule.ptr;
7900 			a = s->anchor.ptr;
7901 		} else if (s == NULL)
7902 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
7903 			    &a, &ruleset, inp);
7904 		break;
7905 	}
7906 
7907 #ifdef INET6
7908 	case IPPROTO_ICMPV6: {
7909 		action = PF_DROP;
7910 		DPFPRINTF(PF_DEBUG_MISC,
7911 		    ("pf: dropping IPv4 packet with ICMPv6 payload\n"));
7912 		goto done;
7913 	}
7914 #endif
7915 
7916 	default:
7917 		action = pf_test_state_other(&s, kif, m, &pd);
7918 		if (action == PF_PASS) {
7919 			if (V_pfsync_update_state_ptr != NULL)
7920 				V_pfsync_update_state_ptr(s);
7921 			r = s->rule.ptr;
7922 			a = s->anchor.ptr;
7923 		} else if (s == NULL)
7924 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
7925 			    &a, &ruleset, inp);
7926 		break;
7927 	}
7928 
7929 done:
7930 	PF_RULES_RUNLOCK();
7931 	if (action == PF_PASS && h->ip_hl > 5 &&
7932 	    !((s && s->state_flags & PFSTATE_ALLOWOPTS) || r->allow_opts)) {
7933 		action = PF_DROP;
7934 		REASON_SET(&reason, PFRES_IPOPTIONS);
7935 		pd.act.log = PF_LOG_FORCE;
7936 		DPFPRINTF(PF_DEBUG_MISC,
7937 		    ("pf: dropping packet with ip options\n"));
7938 	}
7939 
7940 	if (s) {
7941 		memcpy(&pd.act, &s->act, sizeof(struct pf_rule_actions));
7942 		tag = s->tag;
7943 		rt = s->rt;
7944 	} else {
7945 		tag = r->tag;
7946 		rt = r->rt;
7947 	}
7948 
7949 	if (tag > 0 && pf_tag_packet(m, &pd, tag)) {
7950 		action = PF_DROP;
7951 		REASON_SET(&reason, PFRES_MEMORY);
7952 	}
7953 
7954 	pf_scrub_ip(&m, &pd);
7955 	if (pd.proto == IPPROTO_TCP && pd.act.max_mss)
7956 		pf_normalize_mss(m, off, &pd);
7957 
7958 	if (pd.act.rtableid >= 0)
7959 		M_SETFIB(m, pd.act.rtableid);
7960 
7961 	if (pd.act.flags & PFSTATE_SETPRIO) {
7962 		if (pd.tos & IPTOS_LOWDELAY)
7963 			use_2nd_queue = 1;
7964 		if (vlan_set_pcp(m, pd.act.set_prio[use_2nd_queue])) {
7965 			action = PF_DROP;
7966 			REASON_SET(&reason, PFRES_MEMORY);
7967 			pd.act.log = PF_LOG_FORCE;
7968 			DPFPRINTF(PF_DEBUG_MISC,
7969 			    ("pf: failed to allocate 802.1q mtag\n"));
7970 		}
7971 	}
7972 
7973 #ifdef ALTQ
7974 	if (action == PF_PASS && pd.act.qid) {
7975 		if (pd.pf_mtag == NULL &&
7976 		    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
7977 			action = PF_DROP;
7978 			REASON_SET(&reason, PFRES_MEMORY);
7979 		} else {
7980 			if (s != NULL)
7981 				pd.pf_mtag->qid_hash = pf_state_hash(s);
7982 			if (use_2nd_queue || (pd.tos & IPTOS_LOWDELAY))
7983 				pd.pf_mtag->qid = pd.act.pqid;
7984 			else
7985 				pd.pf_mtag->qid = pd.act.qid;
7986 			/* Add hints for ecn. */
7987 			pd.pf_mtag->hdr = h;
7988 		}
7989 	}
7990 #endif /* ALTQ */
7991 
7992 	/*
7993 	 * connections redirected to loopback should not match sockets
7994 	 * bound specifically to loopback due to security implications,
7995 	 * see tcp_input() and in_pcblookup_listen().
7996 	 */
7997 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
7998 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL &&
7999 	    (s->nat_rule.ptr->action == PF_RDR ||
8000 	    s->nat_rule.ptr->action == PF_BINAT) &&
8001 	    IN_LOOPBACK(ntohl(pd.dst->v4.s_addr)))
8002 		m->m_flags |= M_SKIP_FIREWALL;
8003 
8004 	if (__predict_false(ip_divert_ptr != NULL) && action == PF_PASS &&
8005 	    r->divert.port && !PACKET_LOOPED(&pd)) {
8006 		ipfwtag = m_tag_alloc(MTAG_IPFW_RULE, 0,
8007 		    sizeof(struct ipfw_rule_ref), M_NOWAIT | M_ZERO);
8008 		if (ipfwtag != NULL) {
8009 			((struct ipfw_rule_ref *)(ipfwtag+1))->info =
8010 			    ntohs(r->divert.port);
8011 			((struct ipfw_rule_ref *)(ipfwtag+1))->rulenum = dir;
8012 
8013 			if (s)
8014 				PF_STATE_UNLOCK(s);
8015 
8016 			m_tag_prepend(m, ipfwtag);
8017 			if (m->m_flags & M_FASTFWD_OURS) {
8018 				if (pd.pf_mtag == NULL &&
8019 				    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
8020 					action = PF_DROP;
8021 					REASON_SET(&reason, PFRES_MEMORY);
8022 					pd.act.log = PF_LOG_FORCE;
8023 					DPFPRINTF(PF_DEBUG_MISC,
8024 					    ("pf: failed to allocate tag\n"));
8025 				} else {
8026 					pd.pf_mtag->flags |=
8027 					    PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
8028 					m->m_flags &= ~M_FASTFWD_OURS;
8029 				}
8030 			}
8031 			ip_divert_ptr(*m0, dir == PF_IN);
8032 			*m0 = NULL;
8033 
8034 			return (action);
8035 		} else {
8036 			/* XXX: ipfw has the same behaviour! */
8037 			action = PF_DROP;
8038 			REASON_SET(&reason, PFRES_MEMORY);
8039 			pd.act.log = PF_LOG_FORCE;
8040 			DPFPRINTF(PF_DEBUG_MISC,
8041 			    ("pf: failed to allocate divert tag\n"));
8042 		}
8043 	}
8044 
8045 	if (pd.act.log) {
8046 		struct pf_krule		*lr;
8047 		struct pf_krule_item	*ri;
8048 
8049 		if (s != NULL && s->nat_rule.ptr != NULL &&
8050 		    s->nat_rule.ptr->log & PF_LOG_ALL)
8051 			lr = s->nat_rule.ptr;
8052 		else
8053 			lr = r;
8054 
8055 		if (pd.act.log & PF_LOG_FORCE || lr->log & PF_LOG_ALL)
8056 			PFLOG_PACKET(kif, m, AF_INET, reason, lr, a, ruleset,
8057 			    &pd, (s == NULL));
8058 		if (s) {
8059 			SLIST_FOREACH(ri, &s->match_rules, entry)
8060 				if (ri->r->log & PF_LOG_ALL)
8061 					PFLOG_PACKET(kif, m, AF_INET, reason,
8062 					    ri->r, a, ruleset, &pd, 0);
8063 		}
8064 	}
8065 
8066 	pf_counter_u64_critical_enter();
8067 	pf_counter_u64_add_protected(&kif->pfik_bytes[0][dir == PF_OUT][action != PF_PASS],
8068 	    pd.tot_len);
8069 	pf_counter_u64_add_protected(&kif->pfik_packets[0][dir == PF_OUT][action != PF_PASS],
8070 	    1);
8071 
8072 	if (action == PF_PASS || r->action == PF_DROP) {
8073 		dirndx = (dir == PF_OUT);
8074 		pf_counter_u64_add_protected(&r->packets[dirndx], 1);
8075 		pf_counter_u64_add_protected(&r->bytes[dirndx], pd.tot_len);
8076 		pf_update_timestamp(r);
8077 
8078 		if (a != NULL) {
8079 			pf_counter_u64_add_protected(&a->packets[dirndx], 1);
8080 			pf_counter_u64_add_protected(&a->bytes[dirndx], pd.tot_len);
8081 		}
8082 		if (s != NULL) {
8083 			struct pf_krule_item	*ri;
8084 
8085 			if (s->nat_rule.ptr != NULL) {
8086 				pf_counter_u64_add_protected(&s->nat_rule.ptr->packets[dirndx],
8087 				    1);
8088 				pf_counter_u64_add_protected(&s->nat_rule.ptr->bytes[dirndx],
8089 				    pd.tot_len);
8090 			}
8091 			if (s->src_node != NULL) {
8092 				counter_u64_add(s->src_node->packets[dirndx],
8093 				    1);
8094 				counter_u64_add(s->src_node->bytes[dirndx],
8095 				    pd.tot_len);
8096 			}
8097 			if (s->nat_src_node != NULL) {
8098 				counter_u64_add(s->nat_src_node->packets[dirndx],
8099 				    1);
8100 				counter_u64_add(s->nat_src_node->bytes[dirndx],
8101 				    pd.tot_len);
8102 			}
8103 			dirndx = (dir == s->direction) ? 0 : 1;
8104 			s->packets[dirndx]++;
8105 			s->bytes[dirndx] += pd.tot_len;
8106 			SLIST_FOREACH(ri, &s->match_rules, entry) {
8107 				pf_counter_u64_add_protected(&ri->r->packets[dirndx], 1);
8108 				pf_counter_u64_add_protected(&ri->r->bytes[dirndx], pd.tot_len);
8109 			}
8110 		}
8111 		tr = r;
8112 		nr = (s != NULL) ? s->nat_rule.ptr : pd.nat_rule;
8113 		if (nr != NULL && r == &V_pf_default_rule)
8114 			tr = nr;
8115 		if (tr->src.addr.type == PF_ADDR_TABLE)
8116 			pfr_update_stats(tr->src.addr.p.tbl,
8117 			    (s == NULL) ? pd.src :
8118 			    &s->key[(s->direction == PF_IN)]->
8119 				addr[(s->direction == PF_OUT)],
8120 			    pd.af, pd.tot_len, dir == PF_OUT,
8121 			    r->action == PF_PASS, tr->src.neg);
8122 		if (tr->dst.addr.type == PF_ADDR_TABLE)
8123 			pfr_update_stats(tr->dst.addr.p.tbl,
8124 			    (s == NULL) ? pd.dst :
8125 			    &s->key[(s->direction == PF_IN)]->
8126 				addr[(s->direction == PF_IN)],
8127 			    pd.af, pd.tot_len, dir == PF_OUT,
8128 			    r->action == PF_PASS, tr->dst.neg);
8129 	}
8130 	pf_counter_u64_critical_exit();
8131 
8132 	switch (action) {
8133 	case PF_SYNPROXY_DROP:
8134 		m_freem(*m0);
8135 	case PF_DEFER:
8136 		*m0 = NULL;
8137 		action = PF_PASS;
8138 		break;
8139 	case PF_DROP:
8140 		m_freem(*m0);
8141 		*m0 = NULL;
8142 		break;
8143 	default:
8144 		/* pf_route() returns unlocked. */
8145 		if (rt) {
8146 			pf_route(m0, r, kif->pfik_ifp, s, &pd, inp);
8147 			goto out;
8148 		}
8149 		if (pf_dummynet(&pd, s, r, m0) != 0) {
8150 			action = PF_DROP;
8151 			REASON_SET(&reason, PFRES_MEMORY);
8152 		}
8153 		break;
8154 	}
8155 
8156 	SDT_PROBE4(pf, ip, test, done, action, reason, r, s);
8157 
8158 	if (s)
8159 		PF_STATE_UNLOCK(s);
8160 
8161 out:
8162 	pf_sctp_multihome_delayed(&pd, off, kif, s, action);
8163 
8164 	return (action);
8165 }
8166 #endif /* INET */
8167 
8168 #ifdef INET6
8169 int
8170 pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp,
8171     struct pf_rule_actions *default_actions)
8172 {
8173 	struct pfi_kkif		*kif;
8174 	u_short			 action, reason = 0;
8175 	struct mbuf		*m = *m0, *n = NULL;
8176 	struct m_tag		*mtag;
8177 	struct ip6_hdr		*h = NULL;
8178 	struct pf_krule		*a = NULL, *r = &V_pf_default_rule, *tr, *nr;
8179 	struct pf_kstate	*s = NULL;
8180 	struct pf_kruleset	*ruleset = NULL;
8181 	struct pf_pdesc		 pd;
8182 	int			 off, terminal = 0, dirndx, rh_cnt = 0, use_2nd_queue = 0;
8183 	uint16_t		 tag;
8184 	uint8_t			 rt;
8185 
8186 	PF_RULES_RLOCK_TRACKER;
8187 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: bad direction %d\n", __func__, dir));
8188 	M_ASSERTPKTHDR(m);
8189 
8190 	if (!V_pf_status.running)
8191 		return (PF_PASS);
8192 
8193 	PF_RULES_RLOCK();
8194 
8195 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
8196 	if (__predict_false(kif == NULL)) {
8197 		DPFPRINTF(PF_DEBUG_URGENT,
8198 		    ("pf_test6: kif == NULL, if_xname %s\n", ifp->if_xname));
8199 		PF_RULES_RUNLOCK();
8200 		return (PF_DROP);
8201 	}
8202 	if (kif->pfik_flags & PFI_IFLAG_SKIP) {
8203 		PF_RULES_RUNLOCK();
8204 		return (PF_PASS);
8205 	}
8206 
8207 	if (m->m_flags & M_SKIP_FIREWALL) {
8208 		PF_RULES_RUNLOCK();
8209 		return (PF_PASS);
8210 	}
8211 
8212 	memset(&pd, 0, sizeof(pd));
8213 	TAILQ_INIT(&pd.sctp_multihome_jobs);
8214 	if (default_actions != NULL)
8215 		memcpy(&pd.act, default_actions, sizeof(pd.act));
8216 	pd.pf_mtag = pf_find_mtag(m);
8217 
8218 	if (pd.pf_mtag != NULL && (pd.pf_mtag->flags & PF_MTAG_FLAG_ROUTE_TO)) {
8219 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
8220 
8221 		ifp = ifnet_byindexgen(pd.pf_mtag->if_index,
8222 		    pd.pf_mtag->if_idxgen);
8223 		if (ifp == NULL || ifp->if_flags & IFF_DYING) {
8224 			PF_RULES_RUNLOCK();
8225 			m_freem(*m0);
8226 			*m0 = NULL;
8227 			return (PF_PASS);
8228 		}
8229 		PF_RULES_RUNLOCK();
8230 		nd6_output_ifp(ifp, ifp, m,
8231                     (struct sockaddr_in6 *)&pd.pf_mtag->dst, NULL);
8232 		*m0 = NULL;
8233 		return (PF_PASS);
8234 	}
8235 
8236 	if (pd.pf_mtag && pd.pf_mtag->dnpipe) {
8237 		pd.act.dnpipe = pd.pf_mtag->dnpipe;
8238 		pd.act.flags = pd.pf_mtag->dnflags;
8239 	}
8240 
8241 	if (ip_dn_io_ptr != NULL && pd.pf_mtag != NULL &&
8242 	    pd.pf_mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
8243 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
8244 		/* Dummynet re-injects packets after they've
8245 		 * completed their delay. We've already
8246 		 * processed them, so pass unconditionally. */
8247 		PF_RULES_RUNLOCK();
8248 		return (PF_PASS);
8249 	}
8250 
8251 	pd.sport = pd.dport = NULL;
8252 	pd.ip_sum = NULL;
8253 	pd.proto_sum = NULL;
8254 	pd.dir = dir;
8255 	pd.sidx = (dir == PF_IN) ? 0 : 1;
8256 	pd.didx = (dir == PF_IN) ? 1 : 0;
8257 	pd.af = AF_INET6;
8258 	pd.act.rtableid = -1;
8259 
8260 	h = mtod(m, struct ip6_hdr *);
8261 	off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr);
8262 
8263 	/* We do IP header normalization and packet reassembly here */
8264 	if (pf_normalize_ip6(m0, kif, &reason, &pd) != PF_PASS) {
8265 		action = PF_DROP;
8266 		goto done;
8267 	}
8268 	m = *m0;	/* pf_normalize messes with m0 */
8269 	h = mtod(m, struct ip6_hdr *);
8270 	off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr);
8271 
8272 	/*
8273 	 * we do not support jumbogram.  if we keep going, zero ip6_plen
8274 	 * will do something bad, so drop the packet for now.
8275 	 */
8276 	if (htons(h->ip6_plen) == 0) {
8277 		action = PF_DROP;
8278 		REASON_SET(&reason, PFRES_NORM);	/*XXX*/
8279 		goto done;
8280 	}
8281 
8282 	pd.src = (struct pf_addr *)&h->ip6_src;
8283 	pd.dst = (struct pf_addr *)&h->ip6_dst;
8284 	pd.tos = IPV6_DSCP(h);
8285 	pd.tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr);
8286 
8287 	pd.proto = h->ip6_nxt;
8288 	do {
8289 		switch (pd.proto) {
8290 		case IPPROTO_FRAGMENT:
8291 			action = pf_test_fragment(&r, kif, m, h, &pd, &a,
8292 			    &ruleset);
8293 			if (action == PF_DROP)
8294 				REASON_SET(&reason, PFRES_FRAG);
8295 			goto done;
8296 		case IPPROTO_ROUTING: {
8297 			struct ip6_rthdr rthdr;
8298 
8299 			if (rh_cnt++) {
8300 				DPFPRINTF(PF_DEBUG_MISC,
8301 				    ("pf: IPv6 more than one rthdr\n"));
8302 				action = PF_DROP;
8303 				REASON_SET(&reason, PFRES_IPOPTIONS);
8304 				pd.act.log = PF_LOG_FORCE;
8305 				goto done;
8306 			}
8307 			if (!pf_pull_hdr(m, off, &rthdr, sizeof(rthdr), NULL,
8308 			    &reason, pd.af)) {
8309 				DPFPRINTF(PF_DEBUG_MISC,
8310 				    ("pf: IPv6 short rthdr\n"));
8311 				action = PF_DROP;
8312 				REASON_SET(&reason, PFRES_SHORT);
8313 				pd.act.log = PF_LOG_FORCE;
8314 				goto done;
8315 			}
8316 			if (rthdr.ip6r_type == IPV6_RTHDR_TYPE_0) {
8317 				DPFPRINTF(PF_DEBUG_MISC,
8318 				    ("pf: IPv6 rthdr0\n"));
8319 				action = PF_DROP;
8320 				REASON_SET(&reason, PFRES_IPOPTIONS);
8321 				pd.act.log = PF_LOG_FORCE;
8322 				goto done;
8323 			}
8324 			/* FALLTHROUGH */
8325 		}
8326 		case IPPROTO_AH:
8327 		case IPPROTO_HOPOPTS:
8328 		case IPPROTO_DSTOPTS: {
8329 			/* get next header and header length */
8330 			struct ip6_ext	opt6;
8331 
8332 			if (!pf_pull_hdr(m, off, &opt6, sizeof(opt6),
8333 			    NULL, &reason, pd.af)) {
8334 				DPFPRINTF(PF_DEBUG_MISC,
8335 				    ("pf: IPv6 short opt\n"));
8336 				action = PF_DROP;
8337 				pd.act.log = PF_LOG_FORCE;
8338 				goto done;
8339 			}
8340 			if (pd.proto == IPPROTO_AH)
8341 				off += (opt6.ip6e_len + 2) * 4;
8342 			else
8343 				off += (opt6.ip6e_len + 1) * 8;
8344 			pd.proto = opt6.ip6e_nxt;
8345 			/* goto the next header */
8346 			break;
8347 		}
8348 		default:
8349 			terminal++;
8350 			break;
8351 		}
8352 	} while (!terminal);
8353 
8354 	/* if there's no routing header, use unmodified mbuf for checksumming */
8355 	if (!n)
8356 		n = m;
8357 
8358 	switch (pd.proto) {
8359 	case IPPROTO_TCP: {
8360 		if (!pf_pull_hdr(m, off, &pd.hdr.tcp, sizeof(pd.hdr.tcp),
8361 		    &action, &reason, AF_INET6)) {
8362 			if (action != PF_PASS)
8363 				pd.act.log |= PF_LOG_FORCE;
8364 			goto done;
8365 		}
8366 		pd.p_len = pd.tot_len - off - (pd.hdr.tcp.th_off << 2);
8367 		pd.sport = &pd.hdr.tcp.th_sport;
8368 		pd.dport = &pd.hdr.tcp.th_dport;
8369 
8370 		/* Respond to SYN with a syncookie. */
8371 		if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_SYN &&
8372 		    pd.dir == PF_IN && pf_synflood_check(&pd)) {
8373 			pf_syncookie_send(m, off, &pd);
8374 			action = PF_DROP;
8375 			break;
8376 		}
8377 
8378 		action = pf_normalize_tcp(kif, m, 0, off, h, &pd);
8379 		if (action == PF_DROP)
8380 			goto done;
8381 		action = pf_test_state_tcp(&s, kif, m, off, h, &pd, &reason);
8382 		if (action == PF_PASS) {
8383 			if (V_pfsync_update_state_ptr != NULL)
8384 				V_pfsync_update_state_ptr(s);
8385 			r = s->rule.ptr;
8386 			a = s->anchor.ptr;
8387 		} else if (s == NULL) {
8388 			/* Validate remote SYN|ACK, re-create original SYN if
8389 			 * valid. */
8390 			if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) ==
8391 			    TH_ACK && pf_syncookie_validate(&pd) &&
8392 			    pd.dir == PF_IN) {
8393 				struct mbuf *msyn;
8394 
8395 				msyn = pf_syncookie_recreate_syn(h->ip6_hlim,
8396 				    off, &pd);
8397 				if (msyn == NULL) {
8398 					action = PF_DROP;
8399 					break;
8400 				}
8401 
8402 				action = pf_test6(dir, pflags, ifp, &msyn, inp,
8403 				    &pd.act);
8404 				m_freem(msyn);
8405 				if (action != PF_PASS)
8406 					break;
8407 
8408 				action = pf_test_state_tcp(&s, kif, m, off, h,
8409 				    &pd, &reason);
8410 				if (action != PF_PASS || s == NULL) {
8411 					action = PF_DROP;
8412 					break;
8413 				}
8414 
8415 				s->src.seqhi = ntohl(pd.hdr.tcp.th_ack) - 1;
8416 				s->src.seqlo = ntohl(pd.hdr.tcp.th_seq) - 1;
8417 				pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_DST);
8418 
8419 				action = pf_synproxy(&pd, &s, &reason);
8420 				break;
8421 			} else {
8422 				action = pf_test_rule(&r, &s, kif, m, off, &pd,
8423 				    &a, &ruleset, inp);
8424 			}
8425 		}
8426 		break;
8427 	}
8428 
8429 	case IPPROTO_UDP: {
8430 		if (!pf_pull_hdr(m, off, &pd.hdr.udp, sizeof(pd.hdr.udp),
8431 		    &action, &reason, AF_INET6)) {
8432 			if (action != PF_PASS)
8433 				pd.act.log |= PF_LOG_FORCE;
8434 			goto done;
8435 		}
8436 		pd.sport = &pd.hdr.udp.uh_sport;
8437 		pd.dport = &pd.hdr.udp.uh_dport;
8438 		if (pd.hdr.udp.uh_dport == 0 ||
8439 		    ntohs(pd.hdr.udp.uh_ulen) > m->m_pkthdr.len - off ||
8440 		    ntohs(pd.hdr.udp.uh_ulen) < sizeof(struct udphdr)) {
8441 			action = PF_DROP;
8442 			REASON_SET(&reason, PFRES_SHORT);
8443 			goto done;
8444 		}
8445 		action = pf_test_state_udp(&s, kif, m, off, h, &pd);
8446 		if (action == PF_PASS) {
8447 			if (V_pfsync_update_state_ptr != NULL)
8448 				V_pfsync_update_state_ptr(s);
8449 			r = s->rule.ptr;
8450 			a = s->anchor.ptr;
8451 		} else if (s == NULL)
8452 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
8453 			    &a, &ruleset, inp);
8454 		break;
8455 	}
8456 
8457 	case IPPROTO_SCTP: {
8458 		if (!pf_pull_hdr(m, off, &pd.hdr.sctp, sizeof(pd.hdr.sctp),
8459 		    &action, &reason, AF_INET6)) {
8460 			if (action != PF_PASS)
8461 				pd.act.log |= PF_LOG_FORCE;
8462 			goto done;
8463 		}
8464 		pd.sport = &pd.hdr.sctp.src_port;
8465 		pd.dport = &pd.hdr.sctp.dest_port;
8466 		if (pd.hdr.sctp.src_port == 0 || pd.hdr.sctp.dest_port == 0) {
8467 			action = PF_DROP;
8468 			REASON_SET(&reason, PFRES_SHORT);
8469 			goto done;
8470 		}
8471 		action = pf_normalize_sctp(dir, kif, m, 0, off, h, &pd);
8472 		if (action == PF_DROP)
8473 			goto done;
8474 		action = pf_test_state_sctp(&s, kif, m, off, h, &pd,
8475 		    &reason);
8476 		if (action == PF_PASS) {
8477 			if (V_pfsync_update_state_ptr != NULL)
8478 				V_pfsync_update_state_ptr(s);
8479 			r = s->rule.ptr;
8480 			a = s->anchor.ptr;
8481 		} else {
8482 			action = pf_test_rule(&r, &s, kif, m, off,
8483 			    &pd, &a, &ruleset, inp);
8484 		}
8485 		break;
8486 	}
8487 
8488 	case IPPROTO_ICMP: {
8489 		action = PF_DROP;
8490 		DPFPRINTF(PF_DEBUG_MISC,
8491 		    ("pf: dropping IPv6 packet with ICMPv4 payload\n"));
8492 		goto done;
8493 	}
8494 
8495 	case IPPROTO_ICMPV6: {
8496 		if (!pf_pull_hdr(m, off, &pd.hdr.icmp6, sizeof(pd.hdr.icmp6),
8497 		    &action, &reason, AF_INET6)) {
8498 			if (action != PF_PASS)
8499 				pd.act.log |= PF_LOG_FORCE;
8500 			goto done;
8501 		}
8502 		action = pf_test_state_icmp(&s, kif, m, off, h, &pd, &reason);
8503 		if (action == PF_PASS) {
8504 			if (V_pfsync_update_state_ptr != NULL)
8505 				V_pfsync_update_state_ptr(s);
8506 			r = s->rule.ptr;
8507 			a = s->anchor.ptr;
8508 		} else if (s == NULL)
8509 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
8510 			    &a, &ruleset, inp);
8511 		break;
8512 	}
8513 
8514 	default:
8515 		action = pf_test_state_other(&s, kif, m, &pd);
8516 		if (action == PF_PASS) {
8517 			if (V_pfsync_update_state_ptr != NULL)
8518 				V_pfsync_update_state_ptr(s);
8519 			r = s->rule.ptr;
8520 			a = s->anchor.ptr;
8521 		} else if (s == NULL)
8522 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
8523 			    &a, &ruleset, inp);
8524 		break;
8525 	}
8526 
8527 done:
8528 	PF_RULES_RUNLOCK();
8529 	if (n != m) {
8530 		m_freem(n);
8531 		n = NULL;
8532 	}
8533 
8534 	/* handle dangerous IPv6 extension headers. */
8535 	if (action == PF_PASS && rh_cnt &&
8536 	    !((s && s->state_flags & PFSTATE_ALLOWOPTS) || r->allow_opts)) {
8537 		action = PF_DROP;
8538 		REASON_SET(&reason, PFRES_IPOPTIONS);
8539 		pd.act.log = r->log;
8540 		DPFPRINTF(PF_DEBUG_MISC,
8541 		    ("pf: dropping packet with dangerous v6 headers\n"));
8542 	}
8543 
8544 	if (s) {
8545 		memcpy(&pd.act, &s->act, sizeof(struct pf_rule_actions));
8546 		tag = s->tag;
8547 		rt = s->rt;
8548 	} else {
8549 		tag = r->tag;
8550 		rt = r->rt;
8551 	}
8552 
8553 	if (tag > 0 && pf_tag_packet(m, &pd, tag)) {
8554 		action = PF_DROP;
8555 		REASON_SET(&reason, PFRES_MEMORY);
8556 	}
8557 
8558 	pf_scrub_ip6(&m, &pd);
8559 	if (pd.proto == IPPROTO_TCP && pd.act.max_mss)
8560 		pf_normalize_mss(m, off, &pd);
8561 
8562 	if (pd.act.rtableid >= 0)
8563 		M_SETFIB(m, pd.act.rtableid);
8564 
8565 	if (pd.act.flags & PFSTATE_SETPRIO) {
8566 		if (pd.tos & IPTOS_LOWDELAY)
8567 			use_2nd_queue = 1;
8568 		if (vlan_set_pcp(m, pd.act.set_prio[use_2nd_queue])) {
8569 			action = PF_DROP;
8570 			REASON_SET(&reason, PFRES_MEMORY);
8571 			pd.act.log = PF_LOG_FORCE;
8572 			DPFPRINTF(PF_DEBUG_MISC,
8573 			    ("pf: failed to allocate 802.1q mtag\n"));
8574 		}
8575 	}
8576 
8577 #ifdef ALTQ
8578 	if (action == PF_PASS && pd.act.qid) {
8579 		if (pd.pf_mtag == NULL &&
8580 		    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
8581 			action = PF_DROP;
8582 			REASON_SET(&reason, PFRES_MEMORY);
8583 		} else {
8584 			if (s != NULL)
8585 				pd.pf_mtag->qid_hash = pf_state_hash(s);
8586 			if (pd.tos & IPTOS_LOWDELAY)
8587 				pd.pf_mtag->qid = pd.act.pqid;
8588 			else
8589 				pd.pf_mtag->qid = pd.act.qid;
8590 			/* Add hints for ecn. */
8591 			pd.pf_mtag->hdr = h;
8592 		}
8593 	}
8594 #endif /* ALTQ */
8595 
8596 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
8597 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL &&
8598 	    (s->nat_rule.ptr->action == PF_RDR ||
8599 	    s->nat_rule.ptr->action == PF_BINAT) &&
8600 	    IN6_IS_ADDR_LOOPBACK(&pd.dst->v6))
8601 		m->m_flags |= M_SKIP_FIREWALL;
8602 
8603 	/* XXX: Anybody working on it?! */
8604 	if (r->divert.port)
8605 		printf("pf: divert(9) is not supported for IPv6\n");
8606 
8607 	if (pd.act.log) {
8608 		struct pf_krule		*lr;
8609 		struct pf_krule_item	*ri;
8610 
8611 		if (s != NULL && s->nat_rule.ptr != NULL &&
8612 		    s->nat_rule.ptr->log & PF_LOG_ALL)
8613 			lr = s->nat_rule.ptr;
8614 		else
8615 			lr = r;
8616 
8617 		if (pd.act.log & PF_LOG_FORCE || lr->log & PF_LOG_ALL)
8618 			PFLOG_PACKET(kif, m, AF_INET6, reason, lr, a, ruleset,
8619 			    &pd, (s == NULL));
8620 		if (s) {
8621 			SLIST_FOREACH(ri, &s->match_rules, entry)
8622 				if (ri->r->log & PF_LOG_ALL)
8623 					PFLOG_PACKET(kif, m, AF_INET6, reason,
8624 					    ri->r, a, ruleset, &pd, 0);
8625 		}
8626 	}
8627 
8628 	pf_counter_u64_critical_enter();
8629 	pf_counter_u64_add_protected(&kif->pfik_bytes[1][dir == PF_OUT][action != PF_PASS],
8630 	    pd.tot_len);
8631 	pf_counter_u64_add_protected(&kif->pfik_packets[1][dir == PF_OUT][action != PF_PASS],
8632 	    1);
8633 
8634 	if (action == PF_PASS || r->action == PF_DROP) {
8635 		dirndx = (dir == PF_OUT);
8636 		pf_counter_u64_add_protected(&r->packets[dirndx], 1);
8637 		pf_counter_u64_add_protected(&r->bytes[dirndx], pd.tot_len);
8638 		if (a != NULL) {
8639 			pf_counter_u64_add_protected(&a->packets[dirndx], 1);
8640 			pf_counter_u64_add_protected(&a->bytes[dirndx], pd.tot_len);
8641 		}
8642 		if (s != NULL) {
8643 			if (s->nat_rule.ptr != NULL) {
8644 				pf_counter_u64_add_protected(&s->nat_rule.ptr->packets[dirndx],
8645 				    1);
8646 				pf_counter_u64_add_protected(&s->nat_rule.ptr->bytes[dirndx],
8647 				    pd.tot_len);
8648 			}
8649 			if (s->src_node != NULL) {
8650 				counter_u64_add(s->src_node->packets[dirndx],
8651 				    1);
8652 				counter_u64_add(s->src_node->bytes[dirndx],
8653 				    pd.tot_len);
8654 			}
8655 			if (s->nat_src_node != NULL) {
8656 				counter_u64_add(s->nat_src_node->packets[dirndx],
8657 				    1);
8658 				counter_u64_add(s->nat_src_node->bytes[dirndx],
8659 				    pd.tot_len);
8660 			}
8661 			dirndx = (dir == s->direction) ? 0 : 1;
8662 			s->packets[dirndx]++;
8663 			s->bytes[dirndx] += pd.tot_len;
8664 		}
8665 		tr = r;
8666 		nr = (s != NULL) ? s->nat_rule.ptr : pd.nat_rule;
8667 		if (nr != NULL && r == &V_pf_default_rule)
8668 			tr = nr;
8669 		if (tr->src.addr.type == PF_ADDR_TABLE)
8670 			pfr_update_stats(tr->src.addr.p.tbl,
8671 			    (s == NULL) ? pd.src :
8672 			    &s->key[(s->direction == PF_IN)]->addr[0],
8673 			    pd.af, pd.tot_len, dir == PF_OUT,
8674 			    r->action == PF_PASS, tr->src.neg);
8675 		if (tr->dst.addr.type == PF_ADDR_TABLE)
8676 			pfr_update_stats(tr->dst.addr.p.tbl,
8677 			    (s == NULL) ? pd.dst :
8678 			    &s->key[(s->direction == PF_IN)]->addr[1],
8679 			    pd.af, pd.tot_len, dir == PF_OUT,
8680 			    r->action == PF_PASS, tr->dst.neg);
8681 	}
8682 	pf_counter_u64_critical_exit();
8683 
8684 	switch (action) {
8685 	case PF_SYNPROXY_DROP:
8686 		m_freem(*m0);
8687 	case PF_DEFER:
8688 		*m0 = NULL;
8689 		action = PF_PASS;
8690 		break;
8691 	case PF_DROP:
8692 		m_freem(*m0);
8693 		*m0 = NULL;
8694 		break;
8695 	default:
8696 		/* pf_route6() returns unlocked. */
8697 		if (rt) {
8698 			pf_route6(m0, r, kif->pfik_ifp, s, &pd, inp);
8699 			goto out;
8700 		}
8701 		if (pf_dummynet(&pd, s, r, m0) != 0) {
8702 			action = PF_DROP;
8703 			REASON_SET(&reason, PFRES_MEMORY);
8704 		}
8705 		break;
8706 	}
8707 
8708 	if (s)
8709 		PF_STATE_UNLOCK(s);
8710 
8711 	/* If reassembled packet passed, create new fragments. */
8712 	if (action == PF_PASS && *m0 && dir == PF_OUT &&
8713 	    (mtag = m_tag_find(m, PACKET_TAG_PF_REASSEMBLED, NULL)) != NULL)
8714 		action = pf_refragment6(ifp, m0, mtag, pflags & PFIL_FWD);
8715 
8716 out:
8717 	SDT_PROBE4(pf, ip, test6, done, action, reason, r, s);
8718 
8719 	pf_sctp_multihome_delayed(&pd, off, kif, s, action);
8720 
8721 	return (action);
8722 }
8723 #endif /* INET6 */
8724