xref: /freebsd/sys/netpfil/pf/pf.c (revision 13de33a5dc2304b13d595d75d48c51793958474f)
1 /*-
2  * Copyright (c) 2001 Daniel Hartmeier
3  * Copyright (c) 2002 - 2008 Henning Brauer
4  * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  *    - Redistributions of source code must retain the above copyright
12  *      notice, this list of conditions and the following disclaimer.
13  *    - Redistributions in binary form must reproduce the above
14  *      copyright notice, this list of conditions and the following
15  *      disclaimer in the documentation and/or other materials provided
16  *      with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
21  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
22  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
24  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  * Effort sponsored in part by the Defense Advanced Research Projects
32  * Agency (DARPA) and Air Force Research Laboratory, Air Force
33  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
34  *
35  *	$OpenBSD: pf.c,v 1.634 2009/02/27 12:37:45 henning Exp $
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include "opt_inet.h"
42 #include "opt_inet6.h"
43 #include "opt_bpf.h"
44 #include "opt_pf.h"
45 
46 #include <sys/param.h>
47 #include <sys/bus.h>
48 #include <sys/endian.h>
49 #include <sys/hash.h>
50 #include <sys/interrupt.h>
51 #include <sys/kernel.h>
52 #include <sys/kthread.h>
53 #include <sys/limits.h>
54 #include <sys/mbuf.h>
55 #include <sys/md5.h>
56 #include <sys/random.h>
57 #include <sys/refcount.h>
58 #include <sys/socket.h>
59 #include <sys/sysctl.h>
60 #include <sys/taskqueue.h>
61 #include <sys/ucred.h>
62 
63 #include <net/if.h>
64 #include <net/if_var.h>
65 #include <net/if_types.h>
66 #include <net/route.h>
67 #include <net/radix_mpath.h>
68 #include <net/vnet.h>
69 
70 #include <net/pfvar.h>
71 #include <net/if_pflog.h>
72 #include <net/if_pfsync.h>
73 
74 #include <netinet/in_pcb.h>
75 #include <netinet/in_var.h>
76 #include <netinet/ip.h>
77 #include <netinet/ip_fw.h>
78 #include <netinet/ip_icmp.h>
79 #include <netinet/icmp_var.h>
80 #include <netinet/ip_var.h>
81 #include <netinet/tcp.h>
82 #include <netinet/tcp_fsm.h>
83 #include <netinet/tcp_seq.h>
84 #include <netinet/tcp_timer.h>
85 #include <netinet/tcp_var.h>
86 #include <netinet/udp.h>
87 #include <netinet/udp_var.h>
88 
89 #include <netpfil/ipfw/ip_fw_private.h> /* XXX: only for DIR_IN/DIR_OUT */
90 
91 #ifdef INET6
92 #include <netinet/ip6.h>
93 #include <netinet/icmp6.h>
94 #include <netinet6/nd6.h>
95 #include <netinet6/ip6_var.h>
96 #include <netinet6/in6_pcb.h>
97 #endif /* INET6 */
98 
99 #include <machine/in_cksum.h>
100 #include <security/mac/mac_framework.h>
101 
102 #define	DPFPRINTF(n, x)	if (V_pf_status.debug >= (n)) printf x
103 
104 /*
105  * Global variables
106  */
107 
108 /* state tables */
109 VNET_DEFINE(struct pf_altqqueue,	 pf_altqs[2]);
110 VNET_DEFINE(struct pf_palist,		 pf_pabuf);
111 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_active);
112 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_inactive);
113 VNET_DEFINE(struct pf_status,		 pf_status);
114 
115 VNET_DEFINE(u_int32_t,			 ticket_altqs_active);
116 VNET_DEFINE(u_int32_t,			 ticket_altqs_inactive);
117 VNET_DEFINE(int,			 altqs_inactive_open);
118 VNET_DEFINE(u_int32_t,			 ticket_pabuf);
119 
120 VNET_DEFINE(MD5_CTX,			 pf_tcp_secret_ctx);
121 #define	V_pf_tcp_secret_ctx		 VNET(pf_tcp_secret_ctx)
122 VNET_DEFINE(u_char,			 pf_tcp_secret[16]);
123 #define	V_pf_tcp_secret			 VNET(pf_tcp_secret)
124 VNET_DEFINE(int,			 pf_tcp_secret_init);
125 #define	V_pf_tcp_secret_init		 VNET(pf_tcp_secret_init)
126 VNET_DEFINE(int,			 pf_tcp_iss_off);
127 #define	V_pf_tcp_iss_off		 VNET(pf_tcp_iss_off)
128 
129 /*
130  * Queue for pf_intr() sends.
131  */
132 static MALLOC_DEFINE(M_PFTEMP, "pf_temp", "pf(4) temporary allocations");
133 struct pf_send_entry {
134 	STAILQ_ENTRY(pf_send_entry)	pfse_next;
135 	struct mbuf			*pfse_m;
136 	enum {
137 		PFSE_IP,
138 		PFSE_IP6,
139 		PFSE_ICMP,
140 		PFSE_ICMP6,
141 	}				pfse_type;
142 	union {
143 		struct route		ro;
144 		struct {
145 			int		type;
146 			int		code;
147 			int		mtu;
148 		} icmpopts;
149 	} u;
150 #define	pfse_ro		u.ro
151 #define	pfse_icmp_type	u.icmpopts.type
152 #define	pfse_icmp_code	u.icmpopts.code
153 #define	pfse_icmp_mtu	u.icmpopts.mtu
154 };
155 
156 STAILQ_HEAD(pf_send_head, pf_send_entry);
157 static VNET_DEFINE(struct pf_send_head, pf_sendqueue);
158 #define	V_pf_sendqueue	VNET(pf_sendqueue)
159 
160 static struct mtx pf_sendqueue_mtx;
161 #define	PF_SENDQ_LOCK()		mtx_lock(&pf_sendqueue_mtx)
162 #define	PF_SENDQ_UNLOCK()	mtx_unlock(&pf_sendqueue_mtx)
163 
164 /*
165  * Queue for pf_overload_task() tasks.
166  */
167 struct pf_overload_entry {
168 	SLIST_ENTRY(pf_overload_entry)	next;
169 	struct pf_addr  		addr;
170 	sa_family_t			af;
171 	uint8_t				dir;
172 	struct pf_rule  		*rule;
173 };
174 
175 SLIST_HEAD(pf_overload_head, pf_overload_entry);
176 static VNET_DEFINE(struct pf_overload_head, pf_overloadqueue);
177 #define V_pf_overloadqueue	VNET(pf_overloadqueue)
178 static VNET_DEFINE(struct task, pf_overloadtask);
179 #define	V_pf_overloadtask	VNET(pf_overloadtask)
180 
181 static struct mtx pf_overloadqueue_mtx;
182 #define	PF_OVERLOADQ_LOCK()	mtx_lock(&pf_overloadqueue_mtx)
183 #define	PF_OVERLOADQ_UNLOCK()	mtx_unlock(&pf_overloadqueue_mtx)
184 
185 VNET_DEFINE(struct pf_rulequeue, pf_unlinked_rules);
186 struct mtx pf_unlnkdrules_mtx;
187 
188 static VNET_DEFINE(uma_zone_t,	pf_sources_z);
189 #define	V_pf_sources_z	VNET(pf_sources_z)
190 static VNET_DEFINE(uma_zone_t,	pf_mtag_z);
191 #define	V_pf_mtag_z	VNET(pf_mtag_z)
192 VNET_DEFINE(uma_zone_t,	 pf_state_z);
193 VNET_DEFINE(uma_zone_t,	 pf_state_key_z);
194 
195 VNET_DEFINE(uint64_t, pf_stateid[MAXCPU]);
196 #define	PFID_CPUBITS	8
197 #define	PFID_CPUSHIFT	(sizeof(uint64_t) * NBBY - PFID_CPUBITS)
198 #define	PFID_CPUMASK	((uint64_t)((1 << PFID_CPUBITS) - 1) <<	PFID_CPUSHIFT)
199 #define	PFID_MAXID	(~PFID_CPUMASK)
200 CTASSERT((1 << PFID_CPUBITS) > MAXCPU);
201 
202 static void		 pf_src_tree_remove_state(struct pf_state *);
203 static void		 pf_init_threshold(struct pf_threshold *, u_int32_t,
204 			    u_int32_t);
205 static void		 pf_add_threshold(struct pf_threshold *);
206 static int		 pf_check_threshold(struct pf_threshold *);
207 
208 static void		 pf_change_ap(struct pf_addr *, u_int16_t *,
209 			    u_int16_t *, u_int16_t *, struct pf_addr *,
210 			    u_int16_t, u_int8_t, sa_family_t);
211 static int		 pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *,
212 			    struct tcphdr *, struct pf_state_peer *);
213 static int		 pf_icmp_mapping(struct pf_pdesc *, uint8_t, int *,
214 			    int *, uint16_t *, uint16_t *);
215 static void		 pf_change_icmp(struct pf_addr *, u_int16_t *,
216 			    struct pf_addr *, struct pf_addr *, u_int16_t,
217 			    u_int16_t *, u_int16_t *, u_int16_t *,
218 			    u_int16_t *, u_int8_t, sa_family_t);
219 static void		 pf_send_tcp(struct mbuf *,
220 			    const struct pf_rule *, sa_family_t,
221 			    const struct pf_addr *, const struct pf_addr *,
222 			    u_int16_t, u_int16_t, u_int32_t, u_int32_t,
223 			    u_int8_t, u_int16_t, u_int16_t, u_int8_t, int,
224 			    u_int16_t, struct ifnet *);
225 static void		 pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t,
226 			    sa_family_t, struct pf_rule *);
227 static void		 pf_detach_state(struct pf_state *);
228 static int		 pf_state_key_attach(struct pf_state_key *,
229 			    struct pf_state_key *, struct pf_state *);
230 static void		 pf_state_key_detach(struct pf_state *, int);
231 static int		 pf_state_key_ctor(void *, int, void *, int);
232 static u_int32_t	 pf_tcp_iss(struct pf_pdesc *);
233 static int		 pf_test_rule(struct pf_rule **, struct pf_state **,
234 			    int, struct pfi_kif *, struct mbuf *, int,
235 			    struct pf_pdesc *, struct pf_rule **,
236 			    struct pf_ruleset **, struct inpcb *);
237 static int		 pf_create_state(struct pf_rule *, struct pf_rule *,
238 			    struct pf_rule *, struct pf_pdesc *,
239 			    struct pf_src_node *, struct pf_state_key *,
240 			    struct pf_state_key *, struct mbuf *, int,
241 			    u_int16_t, u_int16_t, int *, struct pfi_kif *,
242 			    struct pf_state **, int, u_int16_t, u_int16_t,
243 			    int);
244 static int		 pf_test_fragment(struct pf_rule **, int,
245 			    struct pfi_kif *, struct mbuf *, void *,
246 			    struct pf_pdesc *, struct pf_rule **,
247 			    struct pf_ruleset **);
248 static int		 pf_tcp_track_full(struct pf_state_peer *,
249 			    struct pf_state_peer *, struct pf_state **,
250 			    struct pfi_kif *, struct mbuf *, int,
251 			    struct pf_pdesc *, u_short *, int *);
252 static int		 pf_tcp_track_sloppy(struct pf_state_peer *,
253 			    struct pf_state_peer *, struct pf_state **,
254 			    struct pf_pdesc *, u_short *);
255 static int		 pf_test_state_tcp(struct pf_state **, int,
256 			    struct pfi_kif *, struct mbuf *, int,
257 			    void *, struct pf_pdesc *, u_short *);
258 static int		 pf_test_state_udp(struct pf_state **, int,
259 			    struct pfi_kif *, struct mbuf *, int,
260 			    void *, struct pf_pdesc *);
261 static int		 pf_icmp_state_lookup(struct pf_state_key_cmp *,
262 			    struct pf_pdesc *, struct pf_state **, struct mbuf *,
263 			    int, struct pfi_kif *, uint16_t, uint16_t,
264 			    int, int *, int);
265 static int		 pf_test_state_icmp(struct pf_state **, int,
266 			    struct pfi_kif *, struct mbuf *, int,
267 			    void *, struct pf_pdesc *, u_short *);
268 static int		 pf_test_state_other(struct pf_state **, int,
269 			    struct pfi_kif *, struct mbuf *, struct pf_pdesc *);
270 static u_int8_t		 pf_get_wscale(struct mbuf *, int, u_int16_t,
271 			    sa_family_t);
272 static u_int16_t	 pf_get_mss(struct mbuf *, int, u_int16_t,
273 			    sa_family_t);
274 static u_int16_t	 pf_calc_mss(struct pf_addr *, sa_family_t,
275 				int, u_int16_t);
276 static void		 pf_set_rt_ifp(struct pf_state *,
277 			    struct pf_addr *);
278 static int		 pf_check_proto_cksum(struct mbuf *, int, int,
279 			    u_int8_t, sa_family_t);
280 static void		 pf_print_state_parts(struct pf_state *,
281 			    struct pf_state_key *, struct pf_state_key *);
282 static int		 pf_addr_wrap_neq(struct pf_addr_wrap *,
283 			    struct pf_addr_wrap *);
284 static struct pf_state	*pf_find_state(struct pfi_kif *,
285 			    struct pf_state_key_cmp *, u_int);
286 static int		 pf_src_connlimit(struct pf_state **);
287 static void		 pf_overload_task(void *c, int pending);
288 static int		 pf_insert_src_node(struct pf_src_node **,
289 			    struct pf_rule *, struct pf_addr *, sa_family_t);
290 static u_int		 pf_purge_expired_states(u_int, int);
291 static void		 pf_purge_unlinked_rules(void);
292 static int		 pf_mtag_init(void *, int, int);
293 static void		 pf_mtag_free(struct m_tag *);
294 #ifdef INET
295 static void		 pf_route(struct mbuf **, struct pf_rule *, int,
296 			    struct ifnet *, struct pf_state *,
297 			    struct pf_pdesc *);
298 #endif /* INET */
299 #ifdef INET6
300 static void		 pf_change_a6(struct pf_addr *, u_int16_t *,
301 			    struct pf_addr *, u_int8_t);
302 static void		 pf_route6(struct mbuf **, struct pf_rule *, int,
303 			    struct ifnet *, struct pf_state *,
304 			    struct pf_pdesc *);
305 #endif /* INET6 */
306 
307 int in4_cksum(struct mbuf *m, u_int8_t nxt, int off, int len);
308 
309 VNET_DECLARE(int, pf_end_threads);
310 
311 VNET_DEFINE(struct pf_limit, pf_limits[PF_LIMIT_MAX]);
312 
313 enum { PF_ICMP_MULTI_NONE, PF_ICMP_MULTI_SOLICITED, PF_ICMP_MULTI_LINK };
314 
315 #define	PACKET_LOOPED(pd)	((pd)->pf_mtag &&			\
316 				 (pd)->pf_mtag->flags & PF_PACKET_LOOPED)
317 
318 #define	STATE_LOOKUP(i, k, d, s, pd)					\
319 	do {								\
320 		(s) = pf_find_state((i), (k), (d));			\
321 		if ((s) == NULL)					\
322 			return (PF_DROP);				\
323 		if (PACKET_LOOPED(pd))					\
324 			return (PF_PASS);				\
325 		if ((d) == PF_OUT &&					\
326 		    (((s)->rule.ptr->rt == PF_ROUTETO &&		\
327 		    (s)->rule.ptr->direction == PF_OUT) ||		\
328 		    ((s)->rule.ptr->rt == PF_REPLYTO &&			\
329 		    (s)->rule.ptr->direction == PF_IN)) &&		\
330 		    (s)->rt_kif != NULL &&				\
331 		    (s)->rt_kif != (i))					\
332 			return (PF_PASS);				\
333 	} while (0)
334 
335 #define	BOUND_IFACE(r, k) \
336 	((r)->rule_flag & PFRULE_IFBOUND) ? (k) : V_pfi_all
337 
338 #define	STATE_INC_COUNTERS(s)				\
339 	do {						\
340 		s->rule.ptr->states_cur++;		\
341 		s->rule.ptr->states_tot++;		\
342 		if (s->anchor.ptr != NULL) {		\
343 			s->anchor.ptr->states_cur++;	\
344 			s->anchor.ptr->states_tot++;	\
345 		}					\
346 		if (s->nat_rule.ptr != NULL) {		\
347 			s->nat_rule.ptr->states_cur++;	\
348 			s->nat_rule.ptr->states_tot++;	\
349 		}					\
350 	} while (0)
351 
352 #define	STATE_DEC_COUNTERS(s)				\
353 	do {						\
354 		if (s->nat_rule.ptr != NULL)		\
355 			s->nat_rule.ptr->states_cur--;	\
356 		if (s->anchor.ptr != NULL)		\
357 			s->anchor.ptr->states_cur--;	\
358 		s->rule.ptr->states_cur--;		\
359 	} while (0)
360 
361 static MALLOC_DEFINE(M_PFHASH, "pf_hash", "pf(4) hash header structures");
362 VNET_DEFINE(struct pf_keyhash *, pf_keyhash);
363 VNET_DEFINE(struct pf_idhash *, pf_idhash);
364 VNET_DEFINE(u_long, pf_hashmask);
365 VNET_DEFINE(struct pf_srchash *, pf_srchash);
366 VNET_DEFINE(u_long, pf_srchashmask);
367 
368 SYSCTL_NODE(_net, OID_AUTO, pf, CTLFLAG_RW, 0, "pf(4)");
369 
370 VNET_DEFINE(u_long, pf_hashsize);
371 #define	V_pf_hashsize	VNET(pf_hashsize)
372 SYSCTL_VNET_UINT(_net_pf, OID_AUTO, states_hashsize, CTLFLAG_RDTUN,
373     &VNET_NAME(pf_hashsize), 0, "Size of pf(4) states hashtable");
374 
375 VNET_DEFINE(u_long, pf_srchashsize);
376 #define	V_pf_srchashsize	VNET(pf_srchashsize)
377 SYSCTL_VNET_UINT(_net_pf, OID_AUTO, source_nodes_hashsize, CTLFLAG_RDTUN,
378     &VNET_NAME(pf_srchashsize), 0, "Size of pf(4) source nodes hashtable");
379 
380 VNET_DEFINE(void *, pf_swi_cookie);
381 
382 VNET_DEFINE(uint32_t, pf_hashseed);
383 #define	V_pf_hashseed	VNET(pf_hashseed)
384 
385 static __inline uint32_t
386 pf_hashkey(struct pf_state_key *sk)
387 {
388 	uint32_t h;
389 
390 	h = jenkins_hash32((uint32_t *)sk,
391 	    sizeof(struct pf_state_key_cmp)/sizeof(uint32_t),
392 	    V_pf_hashseed);
393 
394 	return (h & V_pf_hashmask);
395 }
396 
397 static __inline uint32_t
398 pf_hashsrc(struct pf_addr *addr, sa_family_t af)
399 {
400 	uint32_t h;
401 
402 	switch (af) {
403 	case AF_INET:
404 		h = jenkins_hash32((uint32_t *)&addr->v4,
405 		    sizeof(addr->v4)/sizeof(uint32_t), V_pf_hashseed);
406 		break;
407 	case AF_INET6:
408 		h = jenkins_hash32((uint32_t *)&addr->v6,
409 		    sizeof(addr->v6)/sizeof(uint32_t), V_pf_hashseed);
410 		break;
411 	default:
412 		panic("%s: unknown address family %u", __func__, af);
413 	}
414 
415 	return (h & V_pf_srchashmask);
416 }
417 
418 #ifdef INET6
419 void
420 pf_addrcpy(struct pf_addr *dst, struct pf_addr *src, sa_family_t af)
421 {
422 	switch (af) {
423 #ifdef INET
424 	case AF_INET:
425 		dst->addr32[0] = src->addr32[0];
426 		break;
427 #endif /* INET */
428 	case AF_INET6:
429 		dst->addr32[0] = src->addr32[0];
430 		dst->addr32[1] = src->addr32[1];
431 		dst->addr32[2] = src->addr32[2];
432 		dst->addr32[3] = src->addr32[3];
433 		break;
434 	}
435 }
436 #endif /* INET6 */
437 
438 static void
439 pf_init_threshold(struct pf_threshold *threshold,
440     u_int32_t limit, u_int32_t seconds)
441 {
442 	threshold->limit = limit * PF_THRESHOLD_MULT;
443 	threshold->seconds = seconds;
444 	threshold->count = 0;
445 	threshold->last = time_uptime;
446 }
447 
448 static void
449 pf_add_threshold(struct pf_threshold *threshold)
450 {
451 	u_int32_t t = time_uptime, diff = t - threshold->last;
452 
453 	if (diff >= threshold->seconds)
454 		threshold->count = 0;
455 	else
456 		threshold->count -= threshold->count * diff /
457 		    threshold->seconds;
458 	threshold->count += PF_THRESHOLD_MULT;
459 	threshold->last = t;
460 }
461 
462 static int
463 pf_check_threshold(struct pf_threshold *threshold)
464 {
465 	return (threshold->count > threshold->limit);
466 }
467 
468 static int
469 pf_src_connlimit(struct pf_state **state)
470 {
471 	struct pf_overload_entry *pfoe;
472 	int bad = 0;
473 
474 	PF_STATE_LOCK_ASSERT(*state);
475 
476 	(*state)->src_node->conn++;
477 	(*state)->src.tcp_est = 1;
478 	pf_add_threshold(&(*state)->src_node->conn_rate);
479 
480 	if ((*state)->rule.ptr->max_src_conn &&
481 	    (*state)->rule.ptr->max_src_conn <
482 	    (*state)->src_node->conn) {
483 		V_pf_status.lcounters[LCNT_SRCCONN]++;
484 		bad++;
485 	}
486 
487 	if ((*state)->rule.ptr->max_src_conn_rate.limit &&
488 	    pf_check_threshold(&(*state)->src_node->conn_rate)) {
489 		V_pf_status.lcounters[LCNT_SRCCONNRATE]++;
490 		bad++;
491 	}
492 
493 	if (!bad)
494 		return (0);
495 
496 	/* Kill this state. */
497 	(*state)->timeout = PFTM_PURGE;
498 	(*state)->src.state = (*state)->dst.state = TCPS_CLOSED;
499 
500 	if ((*state)->rule.ptr->overload_tbl == NULL)
501 		return (1);
502 
503 	/* Schedule overloading and flushing task. */
504 	pfoe = malloc(sizeof(*pfoe), M_PFTEMP, M_NOWAIT);
505 	if (pfoe == NULL)
506 		return (1);	/* too bad :( */
507 
508 	bcopy(&(*state)->src_node->addr, &pfoe->addr, sizeof(pfoe->addr));
509 	pfoe->af = (*state)->key[PF_SK_WIRE]->af;
510 	pfoe->rule = (*state)->rule.ptr;
511 	pfoe->dir = (*state)->direction;
512 	PF_OVERLOADQ_LOCK();
513 	SLIST_INSERT_HEAD(&V_pf_overloadqueue, pfoe, next);
514 	PF_OVERLOADQ_UNLOCK();
515 	taskqueue_enqueue(taskqueue_swi, &V_pf_overloadtask);
516 
517 	return (1);
518 }
519 
520 static void
521 pf_overload_task(void *c, int pending)
522 {
523 	struct pf_overload_head queue;
524 	struct pfr_addr p;
525 	struct pf_overload_entry *pfoe, *pfoe1;
526 	uint32_t killed = 0;
527 
528 	PF_OVERLOADQ_LOCK();
529 	queue = *(struct pf_overload_head *)c;
530 	SLIST_INIT((struct pf_overload_head *)c);
531 	PF_OVERLOADQ_UNLOCK();
532 
533 	bzero(&p, sizeof(p));
534 	SLIST_FOREACH(pfoe, &queue, next) {
535 		V_pf_status.lcounters[LCNT_OVERLOAD_TABLE]++;
536 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
537 			printf("%s: blocking address ", __func__);
538 			pf_print_host(&pfoe->addr, 0, pfoe->af);
539 			printf("\n");
540 		}
541 
542 		p.pfra_af = pfoe->af;
543 		switch (pfoe->af) {
544 #ifdef INET
545 		case AF_INET:
546 			p.pfra_net = 32;
547 			p.pfra_ip4addr = pfoe->addr.v4;
548 			break;
549 #endif
550 #ifdef INET6
551 		case AF_INET6:
552 			p.pfra_net = 128;
553 			p.pfra_ip6addr = pfoe->addr.v6;
554 			break;
555 #endif
556 		}
557 
558 		PF_RULES_WLOCK();
559 		pfr_insert_kentry(pfoe->rule->overload_tbl, &p, time_second);
560 		PF_RULES_WUNLOCK();
561 	}
562 
563 	/*
564 	 * Remove those entries, that don't need flushing.
565 	 */
566 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
567 		if (pfoe->rule->flush == 0) {
568 			SLIST_REMOVE(&queue, pfoe, pf_overload_entry, next);
569 			free(pfoe, M_PFTEMP);
570 		} else
571 			V_pf_status.lcounters[LCNT_OVERLOAD_FLUSH]++;
572 
573 	/* If nothing to flush, return. */
574 	if (SLIST_EMPTY(&queue))
575 		return;
576 
577 	for (int i = 0; i <= V_pf_hashmask; i++) {
578 		struct pf_idhash *ih = &V_pf_idhash[i];
579 		struct pf_state_key *sk;
580 		struct pf_state *s;
581 
582 		PF_HASHROW_LOCK(ih);
583 		LIST_FOREACH(s, &ih->states, entry) {
584 		    sk = s->key[PF_SK_WIRE];
585 		    SLIST_FOREACH(pfoe, &queue, next)
586 			if (sk->af == pfoe->af &&
587 			    ((pfoe->rule->flush & PF_FLUSH_GLOBAL) ||
588 			    pfoe->rule == s->rule.ptr) &&
589 			    ((pfoe->dir == PF_OUT &&
590 			    PF_AEQ(&pfoe->addr, &sk->addr[1], sk->af)) ||
591 			    (pfoe->dir == PF_IN &&
592 			    PF_AEQ(&pfoe->addr, &sk->addr[0], sk->af)))) {
593 				s->timeout = PFTM_PURGE;
594 				s->src.state = s->dst.state = TCPS_CLOSED;
595 				killed++;
596 			}
597 		}
598 		PF_HASHROW_UNLOCK(ih);
599 	}
600 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
601 		free(pfoe, M_PFTEMP);
602 	if (V_pf_status.debug >= PF_DEBUG_MISC)
603 		printf("%s: %u states killed", __func__, killed);
604 }
605 
606 /*
607  * Can return locked on failure, so that we can consistently
608  * allocate and insert a new one.
609  */
610 struct pf_src_node *
611 pf_find_src_node(struct pf_addr *src, struct pf_rule *rule, sa_family_t af,
612 	int returnlocked)
613 {
614 	struct pf_srchash *sh;
615 	struct pf_src_node *n;
616 
617 	V_pf_status.scounters[SCNT_SRC_NODE_SEARCH]++;
618 
619 	sh = &V_pf_srchash[pf_hashsrc(src, af)];
620 	PF_HASHROW_LOCK(sh);
621 	LIST_FOREACH(n, &sh->nodes, entry)
622 		if (n->rule.ptr == rule && n->af == af &&
623 		    ((af == AF_INET && n->addr.v4.s_addr == src->v4.s_addr) ||
624 		    (af == AF_INET6 && bcmp(&n->addr, src, sizeof(*src)) == 0)))
625 			break;
626 	if (n != NULL || returnlocked == 0)
627 		PF_HASHROW_UNLOCK(sh);
628 
629 	return (n);
630 }
631 
632 static int
633 pf_insert_src_node(struct pf_src_node **sn, struct pf_rule *rule,
634     struct pf_addr *src, sa_family_t af)
635 {
636 
637 	KASSERT((rule->rule_flag & PFRULE_RULESRCTRACK ||
638 	    rule->rpool.opts & PF_POOL_STICKYADDR),
639 	    ("%s for non-tracking rule %p", __func__, rule));
640 
641 	if (*sn == NULL)
642 		*sn = pf_find_src_node(src, rule, af, 1);
643 
644 	if (*sn == NULL) {
645 		struct pf_srchash *sh = &V_pf_srchash[pf_hashsrc(src, af)];
646 
647 		PF_HASHROW_ASSERT(sh);
648 
649 		if (!rule->max_src_nodes ||
650 		    rule->src_nodes < rule->max_src_nodes)
651 			(*sn) = uma_zalloc(V_pf_sources_z, M_NOWAIT | M_ZERO);
652 		else
653 			V_pf_status.lcounters[LCNT_SRCNODES]++;
654 		if ((*sn) == NULL) {
655 			PF_HASHROW_UNLOCK(sh);
656 			return (-1);
657 		}
658 
659 		pf_init_threshold(&(*sn)->conn_rate,
660 		    rule->max_src_conn_rate.limit,
661 		    rule->max_src_conn_rate.seconds);
662 
663 		(*sn)->af = af;
664 		(*sn)->rule.ptr = rule;
665 		PF_ACPY(&(*sn)->addr, src, af);
666 		LIST_INSERT_HEAD(&sh->nodes, *sn, entry);
667 		(*sn)->creation = time_uptime;
668 		(*sn)->ruletype = rule->action;
669 		if ((*sn)->rule.ptr != NULL)
670 			(*sn)->rule.ptr->src_nodes++;
671 		PF_HASHROW_UNLOCK(sh);
672 		V_pf_status.scounters[SCNT_SRC_NODE_INSERT]++;
673 		V_pf_status.src_nodes++;
674 	} else {
675 		if (rule->max_src_states &&
676 		    (*sn)->states >= rule->max_src_states) {
677 			V_pf_status.lcounters[LCNT_SRCSTATES]++;
678 			return (-1);
679 		}
680 	}
681 	return (0);
682 }
683 
684 void
685 pf_unlink_src_node_locked(struct pf_src_node *src)
686 {
687 #ifdef INVARIANTS
688 	struct pf_srchash *sh;
689 
690 	sh = &V_pf_srchash[pf_hashsrc(&src->addr, src->af)];
691 	PF_HASHROW_ASSERT(sh);
692 #endif
693 	LIST_REMOVE(src, entry);
694 	if (src->rule.ptr)
695 		src->rule.ptr->src_nodes--;
696 	V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS]++;
697 	V_pf_status.src_nodes--;
698 }
699 
700 void
701 pf_unlink_src_node(struct pf_src_node *src)
702 {
703 	struct pf_srchash *sh;
704 
705 	sh = &V_pf_srchash[pf_hashsrc(&src->addr, src->af)];
706 	PF_HASHROW_LOCK(sh);
707 	pf_unlink_src_node_locked(src);
708 	PF_HASHROW_UNLOCK(sh);
709 }
710 
711 static void
712 pf_free_src_node(struct pf_src_node *sn)
713 {
714 
715 	KASSERT(sn->states == 0, ("%s: %p has refs", __func__, sn));
716 	LIST_REMOVE(sn, entry);
717 	uma_zfree(V_pf_sources_z, sn);
718 }
719 
720 u_int
721 pf_free_src_nodes(struct pf_src_node_list *head)
722 {
723 	struct pf_src_node *sn, *tmp;
724 	u_int count = 0;
725 
726 	LIST_FOREACH_SAFE(sn, head, entry, tmp) {
727 		pf_free_src_node(sn);
728 		count++;
729 	}
730 
731 	return (count);
732 }
733 
734 /* Data storage structures initialization. */
735 void
736 pf_initialize()
737 {
738 	struct pf_keyhash	*kh;
739 	struct pf_idhash	*ih;
740 	struct pf_srchash	*sh;
741 	u_int i;
742 
743 	TUNABLE_ULONG_FETCH("net.pf.states_hashsize", &V_pf_hashsize);
744 	if (V_pf_hashsize == 0 || !powerof2(V_pf_hashsize))
745 		V_pf_hashsize = PF_HASHSIZ;
746 	TUNABLE_ULONG_FETCH("net.pf.source_nodes_hashsize", &V_pf_srchashsize);
747 	if (V_pf_srchashsize == 0 || !powerof2(V_pf_srchashsize))
748 		V_pf_srchashsize = PF_HASHSIZ / 4;
749 
750 	V_pf_hashseed = arc4random();
751 
752 	/* States and state keys storage. */
753 	V_pf_state_z = uma_zcreate("pf states", sizeof(struct pf_state),
754 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
755 	V_pf_limits[PF_LIMIT_STATES].zone = V_pf_state_z;
756 	uma_zone_set_max(V_pf_state_z, PFSTATE_HIWAT);
757 	uma_zone_set_warning(V_pf_state_z, "PF states limit reached");
758 
759 	V_pf_state_key_z = uma_zcreate("pf state keys",
760 	    sizeof(struct pf_state_key), pf_state_key_ctor, NULL, NULL, NULL,
761 	    UMA_ALIGN_PTR, 0);
762 	V_pf_keyhash = malloc(V_pf_hashsize * sizeof(struct pf_keyhash),
763 	    M_PFHASH, M_WAITOK | M_ZERO);
764 	V_pf_idhash = malloc(V_pf_hashsize * sizeof(struct pf_idhash),
765 	    M_PFHASH, M_WAITOK | M_ZERO);
766 	V_pf_hashmask = V_pf_hashsize - 1;
767 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= V_pf_hashmask;
768 	    i++, kh++, ih++) {
769 		mtx_init(&kh->lock, "pf_keyhash", NULL, MTX_DEF | MTX_DUPOK);
770 		mtx_init(&ih->lock, "pf_idhash", NULL, MTX_DEF);
771 	}
772 
773 	/* Source nodes. */
774 	V_pf_sources_z = uma_zcreate("pf source nodes",
775 	    sizeof(struct pf_src_node), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
776 	    0);
777 	V_pf_limits[PF_LIMIT_SRC_NODES].zone = V_pf_sources_z;
778 	uma_zone_set_max(V_pf_sources_z, PFSNODE_HIWAT);
779 	uma_zone_set_warning(V_pf_sources_z, "PF source nodes limit reached");
780 	V_pf_srchash = malloc(V_pf_srchashsize * sizeof(struct pf_srchash),
781 	  M_PFHASH, M_WAITOK|M_ZERO);
782 	V_pf_srchashmask = V_pf_srchashsize - 1;
783 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++)
784 		mtx_init(&sh->lock, "pf_srchash", NULL, MTX_DEF);
785 
786 	/* ALTQ */
787 	TAILQ_INIT(&V_pf_altqs[0]);
788 	TAILQ_INIT(&V_pf_altqs[1]);
789 	TAILQ_INIT(&V_pf_pabuf);
790 	V_pf_altqs_active = &V_pf_altqs[0];
791 	V_pf_altqs_inactive = &V_pf_altqs[1];
792 
793 	/* Mbuf tags */
794 	V_pf_mtag_z = uma_zcreate("pf mtags", sizeof(struct m_tag) +
795 	    sizeof(struct pf_mtag), NULL, NULL, pf_mtag_init, NULL,
796 	    UMA_ALIGN_PTR, 0);
797 
798 	/* Send & overload+flush queues. */
799 	STAILQ_INIT(&V_pf_sendqueue);
800 	SLIST_INIT(&V_pf_overloadqueue);
801 	TASK_INIT(&V_pf_overloadtask, 0, pf_overload_task, &V_pf_overloadqueue);
802 	mtx_init(&pf_sendqueue_mtx, "pf send queue", NULL, MTX_DEF);
803 	mtx_init(&pf_overloadqueue_mtx, "pf overload/flush queue", NULL,
804 	    MTX_DEF);
805 
806 	/* Unlinked, but may be referenced rules. */
807 	TAILQ_INIT(&V_pf_unlinked_rules);
808 	mtx_init(&pf_unlnkdrules_mtx, "pf unlinked rules", NULL, MTX_DEF);
809 }
810 
811 void
812 pf_cleanup()
813 {
814 	struct pf_keyhash	*kh;
815 	struct pf_idhash	*ih;
816 	struct pf_srchash	*sh;
817 	struct pf_send_entry	*pfse, *next;
818 	u_int i;
819 
820 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= V_pf_hashmask;
821 	    i++, kh++, ih++) {
822 		KASSERT(LIST_EMPTY(&kh->keys), ("%s: key hash not empty",
823 		    __func__));
824 		KASSERT(LIST_EMPTY(&ih->states), ("%s: id hash not empty",
825 		    __func__));
826 		mtx_destroy(&kh->lock);
827 		mtx_destroy(&ih->lock);
828 	}
829 	free(V_pf_keyhash, M_PFHASH);
830 	free(V_pf_idhash, M_PFHASH);
831 
832 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
833 		KASSERT(LIST_EMPTY(&sh->nodes),
834 		    ("%s: source node hash not empty", __func__));
835 		mtx_destroy(&sh->lock);
836 	}
837 	free(V_pf_srchash, M_PFHASH);
838 
839 	STAILQ_FOREACH_SAFE(pfse, &V_pf_sendqueue, pfse_next, next) {
840 		m_freem(pfse->pfse_m);
841 		free(pfse, M_PFTEMP);
842 	}
843 
844 	mtx_destroy(&pf_sendqueue_mtx);
845 	mtx_destroy(&pf_overloadqueue_mtx);
846 	mtx_destroy(&pf_unlnkdrules_mtx);
847 
848 	uma_zdestroy(V_pf_mtag_z);
849 	uma_zdestroy(V_pf_sources_z);
850 	uma_zdestroy(V_pf_state_z);
851 	uma_zdestroy(V_pf_state_key_z);
852 }
853 
854 static int
855 pf_mtag_init(void *mem, int size, int how)
856 {
857 	struct m_tag *t;
858 
859 	t = (struct m_tag *)mem;
860 	t->m_tag_cookie = MTAG_ABI_COMPAT;
861 	t->m_tag_id = PACKET_TAG_PF;
862 	t->m_tag_len = sizeof(struct pf_mtag);
863 	t->m_tag_free = pf_mtag_free;
864 
865 	return (0);
866 }
867 
868 static void
869 pf_mtag_free(struct m_tag *t)
870 {
871 
872 	uma_zfree(V_pf_mtag_z, t);
873 }
874 
875 struct pf_mtag *
876 pf_get_mtag(struct mbuf *m)
877 {
878 	struct m_tag *mtag;
879 
880 	if ((mtag = m_tag_find(m, PACKET_TAG_PF, NULL)) != NULL)
881 		return ((struct pf_mtag *)(mtag + 1));
882 
883 	mtag = uma_zalloc(V_pf_mtag_z, M_NOWAIT);
884 	if (mtag == NULL)
885 		return (NULL);
886 	bzero(mtag + 1, sizeof(struct pf_mtag));
887 	m_tag_prepend(m, mtag);
888 
889 	return ((struct pf_mtag *)(mtag + 1));
890 }
891 
892 static int
893 pf_state_key_attach(struct pf_state_key *skw, struct pf_state_key *sks,
894     struct pf_state *s)
895 {
896 	struct pf_keyhash	*khs, *khw, *kh;
897 	struct pf_state_key	*sk, *cur;
898 	struct pf_state		*si, *olds = NULL;
899 	int idx;
900 
901 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
902 	KASSERT(s->key[PF_SK_WIRE] == NULL, ("%s: state has key", __func__));
903 	KASSERT(s->key[PF_SK_STACK] == NULL, ("%s: state has key", __func__));
904 
905 	/*
906 	 * We need to lock hash slots of both keys. To avoid deadlock
907 	 * we always lock the slot with lower address first. Unlock order
908 	 * isn't important.
909 	 *
910 	 * We also need to lock ID hash slot before dropping key
911 	 * locks. On success we return with ID hash slot locked.
912 	 */
913 
914 	if (skw == sks) {
915 		khs = khw = &V_pf_keyhash[pf_hashkey(skw)];
916 		PF_HASHROW_LOCK(khs);
917 	} else {
918 		khs = &V_pf_keyhash[pf_hashkey(sks)];
919 		khw = &V_pf_keyhash[pf_hashkey(skw)];
920 		if (khs == khw) {
921 			PF_HASHROW_LOCK(khs);
922 		} else if (khs < khw) {
923 			PF_HASHROW_LOCK(khs);
924 			PF_HASHROW_LOCK(khw);
925 		} else {
926 			PF_HASHROW_LOCK(khw);
927 			PF_HASHROW_LOCK(khs);
928 		}
929 	}
930 
931 #define	KEYS_UNLOCK()	do {			\
932 	if (khs != khw) {			\
933 		PF_HASHROW_UNLOCK(khs);		\
934 		PF_HASHROW_UNLOCK(khw);		\
935 	} else					\
936 		PF_HASHROW_UNLOCK(khs);		\
937 } while (0)
938 
939 	/*
940 	 * First run: start with wire key.
941 	 */
942 	sk = skw;
943 	kh = khw;
944 	idx = PF_SK_WIRE;
945 
946 keyattach:
947 	LIST_FOREACH(cur, &kh->keys, entry)
948 		if (bcmp(cur, sk, sizeof(struct pf_state_key_cmp)) == 0)
949 			break;
950 
951 	if (cur != NULL) {
952 		/* Key exists. Check for same kif, if none, add to key. */
953 		TAILQ_FOREACH(si, &cur->states[idx], key_list[idx]) {
954 			struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(si)];
955 
956 			PF_HASHROW_LOCK(ih);
957 			if (si->kif == s->kif &&
958 			    si->direction == s->direction) {
959 				if (sk->proto == IPPROTO_TCP &&
960 				    si->src.state >= TCPS_FIN_WAIT_2 &&
961 				    si->dst.state >= TCPS_FIN_WAIT_2) {
962 					/*
963 					 * New state matches an old >FIN_WAIT_2
964 					 * state. We can't drop key hash locks,
965 					 * thus we can't unlink it properly.
966 					 *
967 					 * As a workaround we drop it into
968 					 * TCPS_CLOSED state, schedule purge
969 					 * ASAP and push it into the very end
970 					 * of the slot TAILQ, so that it won't
971 					 * conflict with our new state.
972 					 */
973 					si->src.state = si->dst.state =
974 					    TCPS_CLOSED;
975 					si->timeout = PFTM_PURGE;
976 					olds = si;
977 				} else {
978 					if (V_pf_status.debug >= PF_DEBUG_MISC) {
979 						printf("pf: %s key attach "
980 						    "failed on %s: ",
981 						    (idx == PF_SK_WIRE) ?
982 						    "wire" : "stack",
983 						    s->kif->pfik_name);
984 						pf_print_state_parts(s,
985 						    (idx == PF_SK_WIRE) ?
986 						    sk : NULL,
987 						    (idx == PF_SK_STACK) ?
988 						    sk : NULL);
989 						printf(", existing: ");
990 						pf_print_state_parts(si,
991 						    (idx == PF_SK_WIRE) ?
992 						    sk : NULL,
993 						    (idx == PF_SK_STACK) ?
994 						    sk : NULL);
995 						printf("\n");
996 					}
997 					PF_HASHROW_UNLOCK(ih);
998 					KEYS_UNLOCK();
999 					uma_zfree(V_pf_state_key_z, sk);
1000 					if (idx == PF_SK_STACK)
1001 						pf_detach_state(s);
1002 					return (EEXIST); /* collision! */
1003 				}
1004 			}
1005 			PF_HASHROW_UNLOCK(ih);
1006 		}
1007 		uma_zfree(V_pf_state_key_z, sk);
1008 		s->key[idx] = cur;
1009 	} else {
1010 		LIST_INSERT_HEAD(&kh->keys, sk, entry);
1011 		s->key[idx] = sk;
1012 	}
1013 
1014 stateattach:
1015 	/* List is sorted, if-bound states before floating. */
1016 	if (s->kif == V_pfi_all)
1017 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], s, key_list[idx]);
1018 	else
1019 		TAILQ_INSERT_HEAD(&s->key[idx]->states[idx], s, key_list[idx]);
1020 
1021 	if (olds) {
1022 		TAILQ_REMOVE(&s->key[idx]->states[idx], olds, key_list[idx]);
1023 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], olds,
1024 		    key_list[idx]);
1025 		olds = NULL;
1026 	}
1027 
1028 	/*
1029 	 * Attach done. See how should we (or should not?)
1030 	 * attach a second key.
1031 	 */
1032 	if (sks == skw) {
1033 		s->key[PF_SK_STACK] = s->key[PF_SK_WIRE];
1034 		idx = PF_SK_STACK;
1035 		sks = NULL;
1036 		goto stateattach;
1037 	} else if (sks != NULL) {
1038 		/*
1039 		 * Continue attaching with stack key.
1040 		 */
1041 		sk = sks;
1042 		kh = khs;
1043 		idx = PF_SK_STACK;
1044 		sks = NULL;
1045 		goto keyattach;
1046 	}
1047 
1048 	PF_STATE_LOCK(s);
1049 	KEYS_UNLOCK();
1050 
1051 	KASSERT(s->key[PF_SK_WIRE] != NULL && s->key[PF_SK_STACK] != NULL,
1052 	    ("%s failure", __func__));
1053 
1054 	return (0);
1055 #undef	KEYS_UNLOCK
1056 }
1057 
1058 static void
1059 pf_detach_state(struct pf_state *s)
1060 {
1061 	struct pf_state_key *sks = s->key[PF_SK_STACK];
1062 	struct pf_keyhash *kh;
1063 
1064 	if (sks != NULL) {
1065 		kh = &V_pf_keyhash[pf_hashkey(sks)];
1066 		PF_HASHROW_LOCK(kh);
1067 		if (s->key[PF_SK_STACK] != NULL)
1068 			pf_state_key_detach(s, PF_SK_STACK);
1069 		/*
1070 		 * If both point to same key, then we are done.
1071 		 */
1072 		if (sks == s->key[PF_SK_WIRE]) {
1073 			pf_state_key_detach(s, PF_SK_WIRE);
1074 			PF_HASHROW_UNLOCK(kh);
1075 			return;
1076 		}
1077 		PF_HASHROW_UNLOCK(kh);
1078 	}
1079 
1080 	if (s->key[PF_SK_WIRE] != NULL) {
1081 		kh = &V_pf_keyhash[pf_hashkey(s->key[PF_SK_WIRE])];
1082 		PF_HASHROW_LOCK(kh);
1083 		if (s->key[PF_SK_WIRE] != NULL)
1084 			pf_state_key_detach(s, PF_SK_WIRE);
1085 		PF_HASHROW_UNLOCK(kh);
1086 	}
1087 }
1088 
1089 static void
1090 pf_state_key_detach(struct pf_state *s, int idx)
1091 {
1092 	struct pf_state_key *sk = s->key[idx];
1093 #ifdef INVARIANTS
1094 	struct pf_keyhash *kh = &V_pf_keyhash[pf_hashkey(sk)];
1095 
1096 	PF_HASHROW_ASSERT(kh);
1097 #endif
1098 	TAILQ_REMOVE(&sk->states[idx], s, key_list[idx]);
1099 	s->key[idx] = NULL;
1100 
1101 	if (TAILQ_EMPTY(&sk->states[0]) && TAILQ_EMPTY(&sk->states[1])) {
1102 		LIST_REMOVE(sk, entry);
1103 		uma_zfree(V_pf_state_key_z, sk);
1104 	}
1105 }
1106 
1107 static int
1108 pf_state_key_ctor(void *mem, int size, void *arg, int flags)
1109 {
1110 	struct pf_state_key *sk = mem;
1111 
1112 	bzero(sk, sizeof(struct pf_state_key_cmp));
1113 	TAILQ_INIT(&sk->states[PF_SK_WIRE]);
1114 	TAILQ_INIT(&sk->states[PF_SK_STACK]);
1115 
1116 	return (0);
1117 }
1118 
1119 struct pf_state_key *
1120 pf_state_key_setup(struct pf_pdesc *pd, struct pf_addr *saddr,
1121 	struct pf_addr *daddr, u_int16_t sport, u_int16_t dport)
1122 {
1123 	struct pf_state_key *sk;
1124 
1125 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1126 	if (sk == NULL)
1127 		return (NULL);
1128 
1129 	PF_ACPY(&sk->addr[pd->sidx], saddr, pd->af);
1130 	PF_ACPY(&sk->addr[pd->didx], daddr, pd->af);
1131 	sk->port[pd->sidx] = sport;
1132 	sk->port[pd->didx] = dport;
1133 	sk->proto = pd->proto;
1134 	sk->af = pd->af;
1135 
1136 	return (sk);
1137 }
1138 
1139 struct pf_state_key *
1140 pf_state_key_clone(struct pf_state_key *orig)
1141 {
1142 	struct pf_state_key *sk;
1143 
1144 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1145 	if (sk == NULL)
1146 		return (NULL);
1147 
1148 	bcopy(orig, sk, sizeof(struct pf_state_key_cmp));
1149 
1150 	return (sk);
1151 }
1152 
1153 int
1154 pf_state_insert(struct pfi_kif *kif, struct pf_state_key *skw,
1155     struct pf_state_key *sks, struct pf_state *s)
1156 {
1157 	struct pf_idhash *ih;
1158 	struct pf_state *cur;
1159 	int error;
1160 
1161 	KASSERT(TAILQ_EMPTY(&sks->states[0]) && TAILQ_EMPTY(&sks->states[1]),
1162 	    ("%s: sks not pristine", __func__));
1163 	KASSERT(TAILQ_EMPTY(&skw->states[0]) && TAILQ_EMPTY(&skw->states[1]),
1164 	    ("%s: skw not pristine", __func__));
1165 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1166 
1167 	s->kif = kif;
1168 
1169 	if (s->id == 0 && s->creatorid == 0) {
1170 		/* XXX: should be atomic, but probability of collision low */
1171 		if ((s->id = V_pf_stateid[curcpu]++) == PFID_MAXID)
1172 			V_pf_stateid[curcpu] = 1;
1173 		s->id |= (uint64_t )curcpu << PFID_CPUSHIFT;
1174 		s->id = htobe64(s->id);
1175 		s->creatorid = V_pf_status.hostid;
1176 	}
1177 
1178 	/* Returns with ID locked on success. */
1179 	if ((error = pf_state_key_attach(skw, sks, s)) != 0)
1180 		return (error);
1181 
1182 	ih = &V_pf_idhash[PF_IDHASH(s)];
1183 	PF_HASHROW_ASSERT(ih);
1184 	LIST_FOREACH(cur, &ih->states, entry)
1185 		if (cur->id == s->id && cur->creatorid == s->creatorid)
1186 			break;
1187 
1188 	if (cur != NULL) {
1189 		PF_HASHROW_UNLOCK(ih);
1190 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
1191 			printf("pf: state ID collision: "
1192 			    "id: %016llx creatorid: %08x\n",
1193 			    (unsigned long long)be64toh(s->id),
1194 			    ntohl(s->creatorid));
1195 		}
1196 		pf_detach_state(s);
1197 		return (EEXIST);
1198 	}
1199 	LIST_INSERT_HEAD(&ih->states, s, entry);
1200 	/* One for keys, one for ID hash. */
1201 	refcount_init(&s->refs, 2);
1202 
1203 	V_pf_status.fcounters[FCNT_STATE_INSERT]++;
1204 	if (pfsync_insert_state_ptr != NULL)
1205 		pfsync_insert_state_ptr(s);
1206 
1207 	/* Returns locked. */
1208 	return (0);
1209 }
1210 
1211 /*
1212  * Find state by ID: returns with locked row on success.
1213  */
1214 struct pf_state *
1215 pf_find_state_byid(uint64_t id, uint32_t creatorid)
1216 {
1217 	struct pf_idhash *ih;
1218 	struct pf_state *s;
1219 
1220 	V_pf_status.fcounters[FCNT_STATE_SEARCH]++;
1221 
1222 	ih = &V_pf_idhash[(be64toh(id) % (V_pf_hashmask + 1))];
1223 
1224 	PF_HASHROW_LOCK(ih);
1225 	LIST_FOREACH(s, &ih->states, entry)
1226 		if (s->id == id && s->creatorid == creatorid)
1227 			break;
1228 
1229 	if (s == NULL)
1230 		PF_HASHROW_UNLOCK(ih);
1231 
1232 	return (s);
1233 }
1234 
1235 /*
1236  * Find state by key.
1237  * Returns with ID hash slot locked on success.
1238  */
1239 static struct pf_state *
1240 pf_find_state(struct pfi_kif *kif, struct pf_state_key_cmp *key, u_int dir)
1241 {
1242 	struct pf_keyhash	*kh;
1243 	struct pf_state_key	*sk;
1244 	struct pf_state		*s;
1245 	int idx;
1246 
1247 	V_pf_status.fcounters[FCNT_STATE_SEARCH]++;
1248 
1249 	kh = &V_pf_keyhash[pf_hashkey((struct pf_state_key *)key)];
1250 
1251 	PF_HASHROW_LOCK(kh);
1252 	LIST_FOREACH(sk, &kh->keys, entry)
1253 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1254 			break;
1255 	if (sk == NULL) {
1256 		PF_HASHROW_UNLOCK(kh);
1257 		return (NULL);
1258 	}
1259 
1260 	idx = (dir == PF_IN ? PF_SK_WIRE : PF_SK_STACK);
1261 
1262 	/* List is sorted, if-bound states before floating ones. */
1263 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx])
1264 		if (s->kif == V_pfi_all || s->kif == kif) {
1265 			PF_STATE_LOCK(s);
1266 			PF_HASHROW_UNLOCK(kh);
1267 			if (s->timeout >= PFTM_MAX) {
1268 				/*
1269 				 * State is either being processed by
1270 				 * pf_unlink_state() in an other thread, or
1271 				 * is scheduled for immediate expiry.
1272 				 */
1273 				PF_STATE_UNLOCK(s);
1274 				return (NULL);
1275 			}
1276 			return (s);
1277 		}
1278 	PF_HASHROW_UNLOCK(kh);
1279 
1280 	return (NULL);
1281 }
1282 
1283 struct pf_state *
1284 pf_find_state_all(struct pf_state_key_cmp *key, u_int dir, int *more)
1285 {
1286 	struct pf_keyhash	*kh;
1287 	struct pf_state_key	*sk;
1288 	struct pf_state		*s, *ret = NULL;
1289 	int			 idx, inout = 0;
1290 
1291 	V_pf_status.fcounters[FCNT_STATE_SEARCH]++;
1292 
1293 	kh = &V_pf_keyhash[pf_hashkey((struct pf_state_key *)key)];
1294 
1295 	PF_HASHROW_LOCK(kh);
1296 	LIST_FOREACH(sk, &kh->keys, entry)
1297 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1298 			break;
1299 	if (sk == NULL) {
1300 		PF_HASHROW_UNLOCK(kh);
1301 		return (NULL);
1302 	}
1303 	switch (dir) {
1304 	case PF_IN:
1305 		idx = PF_SK_WIRE;
1306 		break;
1307 	case PF_OUT:
1308 		idx = PF_SK_STACK;
1309 		break;
1310 	case PF_INOUT:
1311 		idx = PF_SK_WIRE;
1312 		inout = 1;
1313 		break;
1314 	default:
1315 		panic("%s: dir %u", __func__, dir);
1316 	}
1317 second_run:
1318 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx]) {
1319 		if (more == NULL) {
1320 			PF_HASHROW_UNLOCK(kh);
1321 			return (s);
1322 		}
1323 
1324 		if (ret)
1325 			(*more)++;
1326 		else
1327 			ret = s;
1328 	}
1329 	if (inout == 1) {
1330 		inout = 0;
1331 		idx = PF_SK_STACK;
1332 		goto second_run;
1333 	}
1334 	PF_HASHROW_UNLOCK(kh);
1335 
1336 	return (ret);
1337 }
1338 
1339 /* END state table stuff */
1340 
1341 static void
1342 pf_send(struct pf_send_entry *pfse)
1343 {
1344 
1345 	PF_SENDQ_LOCK();
1346 	STAILQ_INSERT_TAIL(&V_pf_sendqueue, pfse, pfse_next);
1347 	PF_SENDQ_UNLOCK();
1348 	swi_sched(V_pf_swi_cookie, 0);
1349 }
1350 
1351 void
1352 pf_intr(void *v)
1353 {
1354 	struct pf_send_head queue;
1355 	struct pf_send_entry *pfse, *next;
1356 
1357 	CURVNET_SET((struct vnet *)v);
1358 
1359 	PF_SENDQ_LOCK();
1360 	queue = V_pf_sendqueue;
1361 	STAILQ_INIT(&V_pf_sendqueue);
1362 	PF_SENDQ_UNLOCK();
1363 
1364 	STAILQ_FOREACH_SAFE(pfse, &queue, pfse_next, next) {
1365 		switch (pfse->pfse_type) {
1366 #ifdef INET
1367 		case PFSE_IP:
1368 			ip_output(pfse->pfse_m, NULL, NULL, 0, NULL, NULL);
1369 			break;
1370 		case PFSE_ICMP:
1371 			icmp_error(pfse->pfse_m, pfse->pfse_icmp_type,
1372 			    pfse->pfse_icmp_code, 0, pfse->pfse_icmp_mtu);
1373 			break;
1374 #endif /* INET */
1375 #ifdef INET6
1376 		case PFSE_IP6:
1377 			ip6_output(pfse->pfse_m, NULL, NULL, 0, NULL, NULL,
1378 			    NULL);
1379 			break;
1380 		case PFSE_ICMP6:
1381 			icmp6_error(pfse->pfse_m, pfse->pfse_icmp_type,
1382 			    pfse->pfse_icmp_code, pfse->pfse_icmp_mtu);
1383 			break;
1384 #endif /* INET6 */
1385 		default:
1386 			panic("%s: unknown type", __func__);
1387 		}
1388 		free(pfse, M_PFTEMP);
1389 	}
1390 	CURVNET_RESTORE();
1391 }
1392 
1393 void
1394 pf_purge_thread(void *v)
1395 {
1396 	u_int idx = 0;
1397 
1398 	CURVNET_SET((struct vnet *)v);
1399 
1400 	for (;;) {
1401 		PF_RULES_RLOCK();
1402 		rw_sleep(pf_purge_thread, &pf_rules_lock, 0, "pftm", hz / 10);
1403 
1404 		if (V_pf_end_threads) {
1405 			/*
1406 			 * To cleanse up all kifs and rules we need
1407 			 * two runs: first one clears reference flags,
1408 			 * then pf_purge_expired_states() doesn't
1409 			 * raise them, and then second run frees.
1410 			 */
1411 			PF_RULES_RUNLOCK();
1412 			pf_purge_unlinked_rules();
1413 			pfi_kif_purge();
1414 
1415 			/*
1416 			 * Now purge everything.
1417 			 */
1418 			pf_purge_expired_states(0, V_pf_hashmask);
1419 			pf_purge_expired_fragments();
1420 			pf_purge_expired_src_nodes();
1421 
1422 			/*
1423 			 * Now all kifs & rules should be unreferenced,
1424 			 * thus should be successfully freed.
1425 			 */
1426 			pf_purge_unlinked_rules();
1427 			pfi_kif_purge();
1428 
1429 			/*
1430 			 * Announce success and exit.
1431 			 */
1432 			PF_RULES_RLOCK();
1433 			V_pf_end_threads++;
1434 			PF_RULES_RUNLOCK();
1435 			wakeup(pf_purge_thread);
1436 			kproc_exit(0);
1437 		}
1438 		PF_RULES_RUNLOCK();
1439 
1440 		/* Process 1/interval fraction of the state table every run. */
1441 		idx = pf_purge_expired_states(idx, V_pf_hashmask /
1442 			    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
1443 
1444 		/* Purge other expired types every PFTM_INTERVAL seconds. */
1445 		if (idx == 0) {
1446 			/*
1447 			 * Order is important:
1448 			 * - states and src nodes reference rules
1449 			 * - states and rules reference kifs
1450 			 */
1451 			pf_purge_expired_fragments();
1452 			pf_purge_expired_src_nodes();
1453 			pf_purge_unlinked_rules();
1454 			pfi_kif_purge();
1455 		}
1456 	}
1457 	/* not reached */
1458 	CURVNET_RESTORE();
1459 }
1460 
1461 u_int32_t
1462 pf_state_expires(const struct pf_state *state)
1463 {
1464 	u_int32_t	timeout;
1465 	u_int32_t	start;
1466 	u_int32_t	end;
1467 	u_int32_t	states;
1468 
1469 	/* handle all PFTM_* > PFTM_MAX here */
1470 	if (state->timeout == PFTM_PURGE)
1471 		return (time_uptime);
1472 	KASSERT(state->timeout != PFTM_UNLINKED,
1473 	    ("pf_state_expires: timeout == PFTM_UNLINKED"));
1474 	KASSERT((state->timeout < PFTM_MAX),
1475 	    ("pf_state_expires: timeout > PFTM_MAX"));
1476 	timeout = state->rule.ptr->timeout[state->timeout];
1477 	if (!timeout)
1478 		timeout = V_pf_default_rule.timeout[state->timeout];
1479 	start = state->rule.ptr->timeout[PFTM_ADAPTIVE_START];
1480 	if (start) {
1481 		end = state->rule.ptr->timeout[PFTM_ADAPTIVE_END];
1482 		states = state->rule.ptr->states_cur;	/* XXXGL */
1483 	} else {
1484 		start = V_pf_default_rule.timeout[PFTM_ADAPTIVE_START];
1485 		end = V_pf_default_rule.timeout[PFTM_ADAPTIVE_END];
1486 		states = V_pf_status.states;
1487 	}
1488 	if (end && states > start && start < end) {
1489 		if (states < end)
1490 			return (state->expire + timeout * (end - states) /
1491 			    (end - start));
1492 		else
1493 			return (time_uptime);
1494 	}
1495 	return (state->expire + timeout);
1496 }
1497 
1498 void
1499 pf_purge_expired_src_nodes()
1500 {
1501 	struct pf_src_node_list	 freelist;
1502 	struct pf_srchash	*sh;
1503 	struct pf_src_node	*cur, *next;
1504 	int i;
1505 
1506 	LIST_INIT(&freelist);
1507 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
1508 	    PF_HASHROW_LOCK(sh);
1509 	    LIST_FOREACH_SAFE(cur, &sh->nodes, entry, next)
1510 		if (cur->states == 0 && cur->expire <= time_uptime) {
1511 			pf_unlink_src_node_locked(cur);
1512 			LIST_INSERT_HEAD(&freelist, cur, entry);
1513 		} else if (cur->rule.ptr != NULL)
1514 			cur->rule.ptr->rule_flag |= PFRULE_REFS;
1515 	    PF_HASHROW_UNLOCK(sh);
1516 	}
1517 
1518 	pf_free_src_nodes(&freelist);
1519 }
1520 
1521 static void
1522 pf_src_tree_remove_state(struct pf_state *s)
1523 {
1524 	u_int32_t timeout;
1525 
1526 	if (s->src_node != NULL) {
1527 		if (s->src.tcp_est)
1528 			--s->src_node->conn;
1529 		if (--s->src_node->states == 0) {
1530 			timeout = s->rule.ptr->timeout[PFTM_SRC_NODE];
1531 			if (!timeout)
1532 				timeout =
1533 				    V_pf_default_rule.timeout[PFTM_SRC_NODE];
1534 			s->src_node->expire = time_uptime + timeout;
1535 		}
1536 	}
1537 	if (s->nat_src_node != s->src_node && s->nat_src_node != NULL) {
1538 		if (--s->nat_src_node->states == 0) {
1539 			timeout = s->rule.ptr->timeout[PFTM_SRC_NODE];
1540 			if (!timeout)
1541 				timeout =
1542 				    V_pf_default_rule.timeout[PFTM_SRC_NODE];
1543 			s->nat_src_node->expire = time_uptime + timeout;
1544 		}
1545 	}
1546 	s->src_node = s->nat_src_node = NULL;
1547 }
1548 
1549 /*
1550  * Unlink and potentilly free a state. Function may be
1551  * called with ID hash row locked, but always returns
1552  * unlocked, since it needs to go through key hash locking.
1553  */
1554 int
1555 pf_unlink_state(struct pf_state *s, u_int flags)
1556 {
1557 	struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)];
1558 
1559 	if ((flags & PF_ENTER_LOCKED) == 0)
1560 		PF_HASHROW_LOCK(ih);
1561 	else
1562 		PF_HASHROW_ASSERT(ih);
1563 
1564 	if (s->timeout == PFTM_UNLINKED) {
1565 		/*
1566 		 * State is being processed
1567 		 * by pf_unlink_state() in
1568 		 * an other thread.
1569 		 */
1570 		PF_HASHROW_UNLOCK(ih);
1571 		return (0);	/* XXXGL: undefined actually */
1572 	}
1573 
1574 	if (s->src.state == PF_TCPS_PROXY_DST) {
1575 		/* XXX wire key the right one? */
1576 		pf_send_tcp(NULL, s->rule.ptr, s->key[PF_SK_WIRE]->af,
1577 		    &s->key[PF_SK_WIRE]->addr[1],
1578 		    &s->key[PF_SK_WIRE]->addr[0],
1579 		    s->key[PF_SK_WIRE]->port[1],
1580 		    s->key[PF_SK_WIRE]->port[0],
1581 		    s->src.seqhi, s->src.seqlo + 1,
1582 		    TH_RST|TH_ACK, 0, 0, 0, 1, s->tag, NULL);
1583 	}
1584 
1585 	LIST_REMOVE(s, entry);
1586 	pf_src_tree_remove_state(s);
1587 
1588 	if (pfsync_delete_state_ptr != NULL)
1589 		pfsync_delete_state_ptr(s);
1590 
1591 	--s->rule.ptr->states_cur;
1592 	if (s->nat_rule.ptr != NULL)
1593 		--s->nat_rule.ptr->states_cur;
1594 	if (s->anchor.ptr != NULL)
1595 		--s->anchor.ptr->states_cur;
1596 
1597 	s->timeout = PFTM_UNLINKED;
1598 
1599 	PF_HASHROW_UNLOCK(ih);
1600 
1601 	pf_detach_state(s);
1602 	refcount_release(&s->refs);
1603 
1604 	return (pf_release_state(s));
1605 }
1606 
1607 void
1608 pf_free_state(struct pf_state *cur)
1609 {
1610 
1611 	KASSERT(cur->refs == 0, ("%s: %p has refs", __func__, cur));
1612 	KASSERT(cur->timeout == PFTM_UNLINKED, ("%s: timeout %u", __func__,
1613 	    cur->timeout));
1614 
1615 	pf_normalize_tcp_cleanup(cur);
1616 	uma_zfree(V_pf_state_z, cur);
1617 	V_pf_status.fcounters[FCNT_STATE_REMOVALS]++;
1618 }
1619 
1620 /*
1621  * Called only from pf_purge_thread(), thus serialized.
1622  */
1623 static u_int
1624 pf_purge_expired_states(u_int i, int maxcheck)
1625 {
1626 	struct pf_idhash *ih;
1627 	struct pf_state *s;
1628 
1629 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
1630 
1631 	/*
1632 	 * Go through hash and unlink states that expire now.
1633 	 */
1634 	while (maxcheck > 0) {
1635 
1636 		ih = &V_pf_idhash[i];
1637 relock:
1638 		PF_HASHROW_LOCK(ih);
1639 		LIST_FOREACH(s, &ih->states, entry) {
1640 			if (pf_state_expires(s) <= time_uptime) {
1641 				V_pf_status.states -=
1642 				    pf_unlink_state(s, PF_ENTER_LOCKED);
1643 				goto relock;
1644 			}
1645 			s->rule.ptr->rule_flag |= PFRULE_REFS;
1646 			if (s->nat_rule.ptr != NULL)
1647 				s->nat_rule.ptr->rule_flag |= PFRULE_REFS;
1648 			if (s->anchor.ptr != NULL)
1649 				s->anchor.ptr->rule_flag |= PFRULE_REFS;
1650 			s->kif->pfik_flags |= PFI_IFLAG_REFS;
1651 			if (s->rt_kif)
1652 				s->rt_kif->pfik_flags |= PFI_IFLAG_REFS;
1653 		}
1654 		PF_HASHROW_UNLOCK(ih);
1655 
1656 		/* Return when we hit end of hash. */
1657 		if (++i > V_pf_hashmask) {
1658 			V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
1659 			return (0);
1660 		}
1661 
1662 		maxcheck--;
1663 	}
1664 
1665 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
1666 
1667 	return (i);
1668 }
1669 
1670 static void
1671 pf_purge_unlinked_rules()
1672 {
1673 	struct pf_rulequeue tmpq;
1674 	struct pf_rule *r, *r1;
1675 
1676 	/*
1677 	 * If we have overloading task pending, then we'd
1678 	 * better skip purging this time. There is a tiny
1679 	 * probability that overloading task references
1680 	 * an already unlinked rule.
1681 	 */
1682 	PF_OVERLOADQ_LOCK();
1683 	if (!SLIST_EMPTY(&V_pf_overloadqueue)) {
1684 		PF_OVERLOADQ_UNLOCK();
1685 		return;
1686 	}
1687 	PF_OVERLOADQ_UNLOCK();
1688 
1689 	/*
1690 	 * Do naive mark-and-sweep garbage collecting of old rules.
1691 	 * Reference flag is raised by pf_purge_expired_states()
1692 	 * and pf_purge_expired_src_nodes().
1693 	 *
1694 	 * To avoid LOR between PF_UNLNKDRULES_LOCK/PF_RULES_WLOCK,
1695 	 * use a temporary queue.
1696 	 */
1697 	TAILQ_INIT(&tmpq);
1698 	PF_UNLNKDRULES_LOCK();
1699 	TAILQ_FOREACH_SAFE(r, &V_pf_unlinked_rules, entries, r1) {
1700 		if (!(r->rule_flag & PFRULE_REFS)) {
1701 			TAILQ_REMOVE(&V_pf_unlinked_rules, r, entries);
1702 			TAILQ_INSERT_TAIL(&tmpq, r, entries);
1703 		} else
1704 			r->rule_flag &= ~PFRULE_REFS;
1705 	}
1706 	PF_UNLNKDRULES_UNLOCK();
1707 
1708 	if (!TAILQ_EMPTY(&tmpq)) {
1709 		PF_RULES_WLOCK();
1710 		TAILQ_FOREACH_SAFE(r, &tmpq, entries, r1) {
1711 			TAILQ_REMOVE(&tmpq, r, entries);
1712 			pf_free_rule(r);
1713 		}
1714 		PF_RULES_WUNLOCK();
1715 	}
1716 }
1717 
1718 void
1719 pf_print_host(struct pf_addr *addr, u_int16_t p, sa_family_t af)
1720 {
1721 	switch (af) {
1722 #ifdef INET
1723 	case AF_INET: {
1724 		u_int32_t a = ntohl(addr->addr32[0]);
1725 		printf("%u.%u.%u.%u", (a>>24)&255, (a>>16)&255,
1726 		    (a>>8)&255, a&255);
1727 		if (p) {
1728 			p = ntohs(p);
1729 			printf(":%u", p);
1730 		}
1731 		break;
1732 	}
1733 #endif /* INET */
1734 #ifdef INET6
1735 	case AF_INET6: {
1736 		u_int16_t b;
1737 		u_int8_t i, curstart, curend, maxstart, maxend;
1738 		curstart = curend = maxstart = maxend = 255;
1739 		for (i = 0; i < 8; i++) {
1740 			if (!addr->addr16[i]) {
1741 				if (curstart == 255)
1742 					curstart = i;
1743 				curend = i;
1744 			} else {
1745 				if ((curend - curstart) >
1746 				    (maxend - maxstart)) {
1747 					maxstart = curstart;
1748 					maxend = curend;
1749 				}
1750 				curstart = curend = 255;
1751 			}
1752 		}
1753 		if ((curend - curstart) >
1754 		    (maxend - maxstart)) {
1755 			maxstart = curstart;
1756 			maxend = curend;
1757 		}
1758 		for (i = 0; i < 8; i++) {
1759 			if (i >= maxstart && i <= maxend) {
1760 				if (i == 0)
1761 					printf(":");
1762 				if (i == maxend)
1763 					printf(":");
1764 			} else {
1765 				b = ntohs(addr->addr16[i]);
1766 				printf("%x", b);
1767 				if (i < 7)
1768 					printf(":");
1769 			}
1770 		}
1771 		if (p) {
1772 			p = ntohs(p);
1773 			printf("[%u]", p);
1774 		}
1775 		break;
1776 	}
1777 #endif /* INET6 */
1778 	}
1779 }
1780 
1781 void
1782 pf_print_state(struct pf_state *s)
1783 {
1784 	pf_print_state_parts(s, NULL, NULL);
1785 }
1786 
1787 static void
1788 pf_print_state_parts(struct pf_state *s,
1789     struct pf_state_key *skwp, struct pf_state_key *sksp)
1790 {
1791 	struct pf_state_key *skw, *sks;
1792 	u_int8_t proto, dir;
1793 
1794 	/* Do our best to fill these, but they're skipped if NULL */
1795 	skw = skwp ? skwp : (s ? s->key[PF_SK_WIRE] : NULL);
1796 	sks = sksp ? sksp : (s ? s->key[PF_SK_STACK] : NULL);
1797 	proto = skw ? skw->proto : (sks ? sks->proto : 0);
1798 	dir = s ? s->direction : 0;
1799 
1800 	switch (proto) {
1801 	case IPPROTO_IPV4:
1802 		printf("IPv4");
1803 		break;
1804 	case IPPROTO_IPV6:
1805 		printf("IPv6");
1806 		break;
1807 	case IPPROTO_TCP:
1808 		printf("TCP");
1809 		break;
1810 	case IPPROTO_UDP:
1811 		printf("UDP");
1812 		break;
1813 	case IPPROTO_ICMP:
1814 		printf("ICMP");
1815 		break;
1816 	case IPPROTO_ICMPV6:
1817 		printf("ICMPv6");
1818 		break;
1819 	default:
1820 		printf("%u", skw->proto);
1821 		break;
1822 	}
1823 	switch (dir) {
1824 	case PF_IN:
1825 		printf(" in");
1826 		break;
1827 	case PF_OUT:
1828 		printf(" out");
1829 		break;
1830 	}
1831 	if (skw) {
1832 		printf(" wire: ");
1833 		pf_print_host(&skw->addr[0], skw->port[0], skw->af);
1834 		printf(" ");
1835 		pf_print_host(&skw->addr[1], skw->port[1], skw->af);
1836 	}
1837 	if (sks) {
1838 		printf(" stack: ");
1839 		if (sks != skw) {
1840 			pf_print_host(&sks->addr[0], sks->port[0], sks->af);
1841 			printf(" ");
1842 			pf_print_host(&sks->addr[1], sks->port[1], sks->af);
1843 		} else
1844 			printf("-");
1845 	}
1846 	if (s) {
1847 		if (proto == IPPROTO_TCP) {
1848 			printf(" [lo=%u high=%u win=%u modulator=%u",
1849 			    s->src.seqlo, s->src.seqhi,
1850 			    s->src.max_win, s->src.seqdiff);
1851 			if (s->src.wscale && s->dst.wscale)
1852 				printf(" wscale=%u",
1853 				    s->src.wscale & PF_WSCALE_MASK);
1854 			printf("]");
1855 			printf(" [lo=%u high=%u win=%u modulator=%u",
1856 			    s->dst.seqlo, s->dst.seqhi,
1857 			    s->dst.max_win, s->dst.seqdiff);
1858 			if (s->src.wscale && s->dst.wscale)
1859 				printf(" wscale=%u",
1860 				s->dst.wscale & PF_WSCALE_MASK);
1861 			printf("]");
1862 		}
1863 		printf(" %u:%u", s->src.state, s->dst.state);
1864 	}
1865 }
1866 
1867 void
1868 pf_print_flags(u_int8_t f)
1869 {
1870 	if (f)
1871 		printf(" ");
1872 	if (f & TH_FIN)
1873 		printf("F");
1874 	if (f & TH_SYN)
1875 		printf("S");
1876 	if (f & TH_RST)
1877 		printf("R");
1878 	if (f & TH_PUSH)
1879 		printf("P");
1880 	if (f & TH_ACK)
1881 		printf("A");
1882 	if (f & TH_URG)
1883 		printf("U");
1884 	if (f & TH_ECE)
1885 		printf("E");
1886 	if (f & TH_CWR)
1887 		printf("W");
1888 }
1889 
1890 #define	PF_SET_SKIP_STEPS(i)					\
1891 	do {							\
1892 		while (head[i] != cur) {			\
1893 			head[i]->skip[i].ptr = cur;		\
1894 			head[i] = TAILQ_NEXT(head[i], entries);	\
1895 		}						\
1896 	} while (0)
1897 
1898 void
1899 pf_calc_skip_steps(struct pf_rulequeue *rules)
1900 {
1901 	struct pf_rule *cur, *prev, *head[PF_SKIP_COUNT];
1902 	int i;
1903 
1904 	cur = TAILQ_FIRST(rules);
1905 	prev = cur;
1906 	for (i = 0; i < PF_SKIP_COUNT; ++i)
1907 		head[i] = cur;
1908 	while (cur != NULL) {
1909 
1910 		if (cur->kif != prev->kif || cur->ifnot != prev->ifnot)
1911 			PF_SET_SKIP_STEPS(PF_SKIP_IFP);
1912 		if (cur->direction != prev->direction)
1913 			PF_SET_SKIP_STEPS(PF_SKIP_DIR);
1914 		if (cur->af != prev->af)
1915 			PF_SET_SKIP_STEPS(PF_SKIP_AF);
1916 		if (cur->proto != prev->proto)
1917 			PF_SET_SKIP_STEPS(PF_SKIP_PROTO);
1918 		if (cur->src.neg != prev->src.neg ||
1919 		    pf_addr_wrap_neq(&cur->src.addr, &prev->src.addr))
1920 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_ADDR);
1921 		if (cur->src.port[0] != prev->src.port[0] ||
1922 		    cur->src.port[1] != prev->src.port[1] ||
1923 		    cur->src.port_op != prev->src.port_op)
1924 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_PORT);
1925 		if (cur->dst.neg != prev->dst.neg ||
1926 		    pf_addr_wrap_neq(&cur->dst.addr, &prev->dst.addr))
1927 			PF_SET_SKIP_STEPS(PF_SKIP_DST_ADDR);
1928 		if (cur->dst.port[0] != prev->dst.port[0] ||
1929 		    cur->dst.port[1] != prev->dst.port[1] ||
1930 		    cur->dst.port_op != prev->dst.port_op)
1931 			PF_SET_SKIP_STEPS(PF_SKIP_DST_PORT);
1932 
1933 		prev = cur;
1934 		cur = TAILQ_NEXT(cur, entries);
1935 	}
1936 	for (i = 0; i < PF_SKIP_COUNT; ++i)
1937 		PF_SET_SKIP_STEPS(i);
1938 }
1939 
1940 static int
1941 pf_addr_wrap_neq(struct pf_addr_wrap *aw1, struct pf_addr_wrap *aw2)
1942 {
1943 	if (aw1->type != aw2->type)
1944 		return (1);
1945 	switch (aw1->type) {
1946 	case PF_ADDR_ADDRMASK:
1947 	case PF_ADDR_RANGE:
1948 		if (PF_ANEQ(&aw1->v.a.addr, &aw2->v.a.addr, 0))
1949 			return (1);
1950 		if (PF_ANEQ(&aw1->v.a.mask, &aw2->v.a.mask, 0))
1951 			return (1);
1952 		return (0);
1953 	case PF_ADDR_DYNIFTL:
1954 		return (aw1->p.dyn->pfid_kt != aw2->p.dyn->pfid_kt);
1955 	case PF_ADDR_NOROUTE:
1956 	case PF_ADDR_URPFFAILED:
1957 		return (0);
1958 	case PF_ADDR_TABLE:
1959 		return (aw1->p.tbl != aw2->p.tbl);
1960 	default:
1961 		printf("invalid address type: %d\n", aw1->type);
1962 		return (1);
1963 	}
1964 }
1965 
1966 u_int16_t
1967 pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
1968 {
1969 	u_int32_t	l;
1970 
1971 	if (udp && !cksum)
1972 		return (0x0000);
1973 	l = cksum + old - new;
1974 	l = (l >> 16) + (l & 65535);
1975 	l = l & 65535;
1976 	if (udp && !l)
1977 		return (0xFFFF);
1978 	return (l);
1979 }
1980 
1981 static void
1982 pf_change_ap(struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc,
1983     struct pf_addr *an, u_int16_t pn, u_int8_t u, sa_family_t af)
1984 {
1985 	struct pf_addr	ao;
1986 	u_int16_t	po = *p;
1987 
1988 	PF_ACPY(&ao, a, af);
1989 	PF_ACPY(a, an, af);
1990 
1991 	*p = pn;
1992 
1993 	switch (af) {
1994 #ifdef INET
1995 	case AF_INET:
1996 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
1997 		    ao.addr16[0], an->addr16[0], 0),
1998 		    ao.addr16[1], an->addr16[1], 0);
1999 		*p = pn;
2000 		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(*pc,
2001 		    ao.addr16[0], an->addr16[0], u),
2002 		    ao.addr16[1], an->addr16[1], u),
2003 		    po, pn, u);
2004 		break;
2005 #endif /* INET */
2006 #ifdef INET6
2007 	case AF_INET6:
2008 		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2009 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2010 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(*pc,
2011 		    ao.addr16[0], an->addr16[0], u),
2012 		    ao.addr16[1], an->addr16[1], u),
2013 		    ao.addr16[2], an->addr16[2], u),
2014 		    ao.addr16[3], an->addr16[3], u),
2015 		    ao.addr16[4], an->addr16[4], u),
2016 		    ao.addr16[5], an->addr16[5], u),
2017 		    ao.addr16[6], an->addr16[6], u),
2018 		    ao.addr16[7], an->addr16[7], u),
2019 		    po, pn, u);
2020 		break;
2021 #endif /* INET6 */
2022 	}
2023 }
2024 
2025 
2026 /* Changes a u_int32_t.  Uses a void * so there are no align restrictions */
2027 void
2028 pf_change_a(void *a, u_int16_t *c, u_int32_t an, u_int8_t u)
2029 {
2030 	u_int32_t	ao;
2031 
2032 	memcpy(&ao, a, sizeof(ao));
2033 	memcpy(a, &an, sizeof(u_int32_t));
2034 	*c = pf_cksum_fixup(pf_cksum_fixup(*c, ao / 65536, an / 65536, u),
2035 	    ao % 65536, an % 65536, u);
2036 }
2037 
2038 #ifdef INET6
2039 static void
2040 pf_change_a6(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u)
2041 {
2042 	struct pf_addr	ao;
2043 
2044 	PF_ACPY(&ao, a, AF_INET6);
2045 	PF_ACPY(a, an, AF_INET6);
2046 
2047 	*c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2048 	    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2049 	    pf_cksum_fixup(pf_cksum_fixup(*c,
2050 	    ao.addr16[0], an->addr16[0], u),
2051 	    ao.addr16[1], an->addr16[1], u),
2052 	    ao.addr16[2], an->addr16[2], u),
2053 	    ao.addr16[3], an->addr16[3], u),
2054 	    ao.addr16[4], an->addr16[4], u),
2055 	    ao.addr16[5], an->addr16[5], u),
2056 	    ao.addr16[6], an->addr16[6], u),
2057 	    ao.addr16[7], an->addr16[7], u);
2058 }
2059 #endif /* INET6 */
2060 
2061 static int
2062 pf_icmp_mapping(struct pf_pdesc *pd, uint8_t type,
2063     int *icmp_dir, int *multi, uint16_t *icmpid, uint16_t *icmptype)
2064 {
2065 	/*
2066 	 * ICMP types marked with PF_OUT are typically responses to
2067 	 * PF_IN, and will match states in the opposite direction.
2068 	 * PF_IN ICMP types need to match a state with that type.
2069 	 */
2070 	*icmp_dir = PF_OUT;
2071 	*multi = PF_ICMP_MULTI_LINK;
2072 	/* Queries (and responses) */
2073 	switch (type) {
2074 	case ICMP_ECHO:
2075 		*icmp_dir = PF_IN;
2076 	case ICMP_ECHOREPLY:
2077 		*icmptype = ICMP_ECHO;
2078 		*icmpid = pd->hdr.icmp->icmp_id;
2079 		break;
2080 
2081 	case ICMP_TSTAMP:
2082 		*icmp_dir = PF_IN;
2083 	case ICMP_TSTAMPREPLY:
2084 		*icmptype = ICMP_TSTAMP;
2085 		*icmpid = pd->hdr.icmp->icmp_id;
2086 		break;
2087 
2088 	case ICMP_IREQ:
2089 		*icmp_dir = PF_IN;
2090 	case ICMP_IREQREPLY:
2091 		*icmptype = ICMP_IREQ;
2092 		*icmpid = pd->hdr.icmp->icmp_id;
2093 		break;
2094 
2095 	case ICMP_MASKREQ:
2096 		*icmp_dir = PF_IN;
2097 	case ICMP_MASKREPLY:
2098 		*icmptype = ICMP_MASKREQ;
2099 		*icmpid = pd->hdr.icmp->icmp_id;
2100 		break;
2101 
2102 	case ICMP_IPV6_WHEREAREYOU:
2103 		*icmp_dir = PF_IN;
2104 	case ICMP_IPV6_IAMHERE:
2105 		*icmptype = ICMP_IPV6_WHEREAREYOU;
2106 		*icmpid = 0; /* Nothing sane to match on! */
2107 		break;
2108 
2109 	case ICMP_MOBILE_REGREQUEST:
2110 		*icmp_dir = PF_IN;
2111 	case ICMP_MOBILE_REGREPLY:
2112 		*icmptype = ICMP_MOBILE_REGREQUEST;
2113 		*icmpid = 0; /* Nothing sane to match on! */
2114 		break;
2115 
2116 	case ICMP_ROUTERSOLICIT:
2117 		*icmp_dir = PF_IN;
2118 	case ICMP_ROUTERADVERT:
2119 		*icmptype = ICMP_MOBILE_REGREQUEST;
2120 		*icmpid = 0; /* Nothing sane to match on! */
2121 		break;
2122 
2123 #ifdef INET6
2124 	case ICMP6_ECHO_REQUEST:
2125 		*icmp_dir = PF_IN;
2126 	case ICMP6_ECHO_REPLY:
2127 		*icmptype = ICMP6_ECHO_REPLY;
2128 		*icmpid = 0; /* Nothing sane to match on! */
2129 		break;
2130 
2131 	case MLD_LISTENER_QUERY:
2132 		*icmp_dir = PF_IN;
2133 	case MLD_LISTENER_REPORT: {
2134 		struct mld_hdr *mld = (void *)pd->hdr.icmp6;
2135 
2136 		*icmptype = MLD_LISTENER_QUERY;
2137 		/* generate fake id for these messages */
2138 		*icmpid = (mld->mld_addr.s6_addr32[0] ^
2139 			mld->mld_addr.s6_addr32[1] ^
2140 			mld->mld_addr.s6_addr32[2] ^
2141 			mld->mld_addr.s6_addr32[3]) & 0xffff;
2142 		break;
2143 	}
2144 
2145 	/* ICMP6_FQDN and ICMP6_NI query/reply are the same type as ICMP6_WRU */
2146 	case ICMP6_WRUREQUEST:
2147 		*icmp_dir = PF_IN;
2148 	case ICMP6_WRUREPLY:
2149 		*icmptype = ICMP6_WRUREQUEST;
2150 		*icmpid = 0; /* Nothing sane to match on! */
2151 		break;
2152 
2153 	case MLD_MTRACE:
2154 		*icmp_dir = PF_IN;
2155 	case MLD_MTRACE_RESP:
2156 		*icmptype = MLD_MTRACE;
2157 		*icmpid = 0; /* Nothing sane to match on! */
2158 		break;
2159 
2160 	case ND_NEIGHBOR_SOLICIT:
2161 		*icmp_dir = PF_IN;
2162 	case ND_NEIGHBOR_ADVERT: {
2163 		struct nd_neighbor_solicit *nd = (void *)pd->hdr.icmp6;
2164 
2165 		*icmptype = ND_NEIGHBOR_SOLICIT;
2166 		*multi = PF_ICMP_MULTI_SOLICITED;
2167 		/* generate fake id for these messages */
2168 		*icmpid = (nd->nd_ns_target.s6_addr32[0] ^
2169 			nd->nd_ns_target.s6_addr32[1] ^
2170 			nd->nd_ns_target.s6_addr32[2] ^
2171 			nd->nd_ns_target.s6_addr32[3]) & 0xffff;
2172 		break;
2173 	}
2174 
2175 #endif /* INET6 */
2176 	/* These ICMP types map to other connections */
2177 	case ICMP_UNREACH:
2178 	case ICMP_SOURCEQUENCH:
2179 	case ICMP_REDIRECT:
2180 	case ICMP_TIMXCEED:
2181 	case ICMP_PARAMPROB:
2182 #ifdef INET6
2183 	/*
2184 	 * ICMP6_TIME_EXCEEDED is the same type as ICMP_UNREACH
2185 	 * ND_REDIRECT can't be in this list because the triggering packet
2186 	 * header is optional.
2187 	 */
2188 	case ICMP6_PACKET_TOO_BIG:
2189 #endif /* INET6 */
2190 		/* These will not be used, but set them anyways */
2191 		*icmp_dir = PF_IN;
2192 		*icmptype = htons(type);
2193 		*icmpid = 0;
2194 		return (1);	/* These types are matched to other state */
2195 	/*
2196 	 * All remaining ICMP types get their own states,
2197 	 * and will only match in one direction.
2198 	 */
2199 	default:
2200 		*icmp_dir = PF_IN;
2201 		*icmptype = type;
2202 		*icmpid = 0;
2203 		break;
2204 	}
2205 	*icmptype = htons(*icmptype);
2206 
2207 	return (0);
2208 }
2209 
2210 static void
2211 pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa,
2212     struct pf_addr *na, u_int16_t np, u_int16_t *pc, u_int16_t *h2c,
2213     u_int16_t *ic, u_int16_t *hc, u_int8_t u, sa_family_t af)
2214 {
2215 	struct pf_addr	oia, ooa;
2216 
2217 	PF_ACPY(&oia, ia, af);
2218 	if (oa)
2219 		PF_ACPY(&ooa, oa, af);
2220 
2221 	/* Change inner protocol port, fix inner protocol checksum. */
2222 	if (ip != NULL) {
2223 		u_int16_t	oip = *ip;
2224 		u_int32_t	opc;
2225 
2226 		if (pc != NULL)
2227 			opc = *pc;
2228 		*ip = np;
2229 		if (pc != NULL)
2230 			*pc = pf_cksum_fixup(*pc, oip, *ip, u);
2231 		*ic = pf_cksum_fixup(*ic, oip, *ip, 0);
2232 		if (pc != NULL)
2233 			*ic = pf_cksum_fixup(*ic, opc, *pc, 0);
2234 	}
2235 	/* Change inner ip address, fix inner ip and icmp checksums. */
2236 	PF_ACPY(ia, na, af);
2237 	switch (af) {
2238 #ifdef INET
2239 	case AF_INET: {
2240 		u_int32_t	 oh2c = *h2c;
2241 
2242 		*h2c = pf_cksum_fixup(pf_cksum_fixup(*h2c,
2243 		    oia.addr16[0], ia->addr16[0], 0),
2244 		    oia.addr16[1], ia->addr16[1], 0);
2245 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
2246 		    oia.addr16[0], ia->addr16[0], 0),
2247 		    oia.addr16[1], ia->addr16[1], 0);
2248 		*ic = pf_cksum_fixup(*ic, oh2c, *h2c, 0);
2249 		break;
2250 	}
2251 #endif /* INET */
2252 #ifdef INET6
2253 	case AF_INET6:
2254 		*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2255 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2256 		    pf_cksum_fixup(pf_cksum_fixup(*ic,
2257 		    oia.addr16[0], ia->addr16[0], u),
2258 		    oia.addr16[1], ia->addr16[1], u),
2259 		    oia.addr16[2], ia->addr16[2], u),
2260 		    oia.addr16[3], ia->addr16[3], u),
2261 		    oia.addr16[4], ia->addr16[4], u),
2262 		    oia.addr16[5], ia->addr16[5], u),
2263 		    oia.addr16[6], ia->addr16[6], u),
2264 		    oia.addr16[7], ia->addr16[7], u);
2265 		break;
2266 #endif /* INET6 */
2267 	}
2268 	/* Outer ip address, fix outer ip or icmpv6 checksum, if necessary. */
2269 	if (oa) {
2270 		PF_ACPY(oa, na, af);
2271 		switch (af) {
2272 #ifdef INET
2273 		case AF_INET:
2274 			*hc = pf_cksum_fixup(pf_cksum_fixup(*hc,
2275 			    ooa.addr16[0], oa->addr16[0], 0),
2276 			    ooa.addr16[1], oa->addr16[1], 0);
2277 			break;
2278 #endif /* INET */
2279 #ifdef INET6
2280 		case AF_INET6:
2281 			*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2282 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
2283 			    pf_cksum_fixup(pf_cksum_fixup(*ic,
2284 			    ooa.addr16[0], oa->addr16[0], u),
2285 			    ooa.addr16[1], oa->addr16[1], u),
2286 			    ooa.addr16[2], oa->addr16[2], u),
2287 			    ooa.addr16[3], oa->addr16[3], u),
2288 			    ooa.addr16[4], oa->addr16[4], u),
2289 			    ooa.addr16[5], oa->addr16[5], u),
2290 			    ooa.addr16[6], oa->addr16[6], u),
2291 			    ooa.addr16[7], oa->addr16[7], u);
2292 			break;
2293 #endif /* INET6 */
2294 		}
2295 	}
2296 }
2297 
2298 
2299 /*
2300  * Need to modulate the sequence numbers in the TCP SACK option
2301  * (credits to Krzysztof Pfaff for report and patch)
2302  */
2303 static int
2304 pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd,
2305     struct tcphdr *th, struct pf_state_peer *dst)
2306 {
2307 	int hlen = (th->th_off << 2) - sizeof(*th), thoptlen = hlen;
2308 	u_int8_t opts[TCP_MAXOLEN], *opt = opts;
2309 	int copyback = 0, i, olen;
2310 	struct sackblk sack;
2311 
2312 #define	TCPOLEN_SACKLEN	(TCPOLEN_SACK + 2)
2313 	if (hlen < TCPOLEN_SACKLEN ||
2314 	    !pf_pull_hdr(m, off + sizeof(*th), opts, hlen, NULL, NULL, pd->af))
2315 		return 0;
2316 
2317 	while (hlen >= TCPOLEN_SACKLEN) {
2318 		olen = opt[1];
2319 		switch (*opt) {
2320 		case TCPOPT_EOL:	/* FALLTHROUGH */
2321 		case TCPOPT_NOP:
2322 			opt++;
2323 			hlen--;
2324 			break;
2325 		case TCPOPT_SACK:
2326 			if (olen > hlen)
2327 				olen = hlen;
2328 			if (olen >= TCPOLEN_SACKLEN) {
2329 				for (i = 2; i + TCPOLEN_SACK <= olen;
2330 				    i += TCPOLEN_SACK) {
2331 					memcpy(&sack, &opt[i], sizeof(sack));
2332 					pf_change_a(&sack.start, &th->th_sum,
2333 					    htonl(ntohl(sack.start) -
2334 					    dst->seqdiff), 0);
2335 					pf_change_a(&sack.end, &th->th_sum,
2336 					    htonl(ntohl(sack.end) -
2337 					    dst->seqdiff), 0);
2338 					memcpy(&opt[i], &sack, sizeof(sack));
2339 				}
2340 				copyback = 1;
2341 			}
2342 			/* FALLTHROUGH */
2343 		default:
2344 			if (olen < 2)
2345 				olen = 2;
2346 			hlen -= olen;
2347 			opt += olen;
2348 		}
2349 	}
2350 
2351 	if (copyback)
2352 		m_copyback(m, off + sizeof(*th), thoptlen, (caddr_t)opts);
2353 	return (copyback);
2354 }
2355 
2356 static void
2357 pf_send_tcp(struct mbuf *replyto, const struct pf_rule *r, sa_family_t af,
2358     const struct pf_addr *saddr, const struct pf_addr *daddr,
2359     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
2360     u_int8_t flags, u_int16_t win, u_int16_t mss, u_int8_t ttl, int tag,
2361     u_int16_t rtag, struct ifnet *ifp)
2362 {
2363 	struct pf_send_entry *pfse;
2364 	struct mbuf	*m;
2365 	int		 len, tlen;
2366 #ifdef INET
2367 	struct ip	*h = NULL;
2368 #endif /* INET */
2369 #ifdef INET6
2370 	struct ip6_hdr	*h6 = NULL;
2371 #endif /* INET6 */
2372 	struct tcphdr	*th;
2373 	char		*opt;
2374 	struct pf_mtag  *pf_mtag;
2375 
2376 	len = 0;
2377 	th = NULL;
2378 
2379 	/* maximum segment size tcp option */
2380 	tlen = sizeof(struct tcphdr);
2381 	if (mss)
2382 		tlen += 4;
2383 
2384 	switch (af) {
2385 #ifdef INET
2386 	case AF_INET:
2387 		len = sizeof(struct ip) + tlen;
2388 		break;
2389 #endif /* INET */
2390 #ifdef INET6
2391 	case AF_INET6:
2392 		len = sizeof(struct ip6_hdr) + tlen;
2393 		break;
2394 #endif /* INET6 */
2395 	default:
2396 		panic("%s: unsupported af %d", __func__, af);
2397 	}
2398 
2399 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
2400 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
2401 	if (pfse == NULL)
2402 		return;
2403 	m = m_gethdr(M_NOWAIT, MT_DATA);
2404 	if (m == NULL) {
2405 		free(pfse, M_PFTEMP);
2406 		return;
2407 	}
2408 #ifdef MAC
2409 	mac_netinet_firewall_send(m);
2410 #endif
2411 	if ((pf_mtag = pf_get_mtag(m)) == NULL) {
2412 		free(pfse, M_PFTEMP);
2413 		m_freem(m);
2414 		return;
2415 	}
2416 	if (tag)
2417 		m->m_flags |= M_SKIP_FIREWALL;
2418 	pf_mtag->tag = rtag;
2419 
2420 	if (r != NULL && r->rtableid >= 0)
2421 		M_SETFIB(m, r->rtableid);
2422 
2423 #ifdef ALTQ
2424 	if (r != NULL && r->qid) {
2425 		pf_mtag->qid = r->qid;
2426 
2427 		/* add hints for ecn */
2428 		pf_mtag->hdr = mtod(m, struct ip *);
2429 	}
2430 #endif /* ALTQ */
2431 	m->m_data += max_linkhdr;
2432 	m->m_pkthdr.len = m->m_len = len;
2433 	m->m_pkthdr.rcvif = NULL;
2434 	bzero(m->m_data, len);
2435 	switch (af) {
2436 #ifdef INET
2437 	case AF_INET:
2438 		h = mtod(m, struct ip *);
2439 
2440 		/* IP header fields included in the TCP checksum */
2441 		h->ip_p = IPPROTO_TCP;
2442 		h->ip_len = htons(tlen);
2443 		h->ip_src.s_addr = saddr->v4.s_addr;
2444 		h->ip_dst.s_addr = daddr->v4.s_addr;
2445 
2446 		th = (struct tcphdr *)((caddr_t)h + sizeof(struct ip));
2447 		break;
2448 #endif /* INET */
2449 #ifdef INET6
2450 	case AF_INET6:
2451 		h6 = mtod(m, struct ip6_hdr *);
2452 
2453 		/* IP header fields included in the TCP checksum */
2454 		h6->ip6_nxt = IPPROTO_TCP;
2455 		h6->ip6_plen = htons(tlen);
2456 		memcpy(&h6->ip6_src, &saddr->v6, sizeof(struct in6_addr));
2457 		memcpy(&h6->ip6_dst, &daddr->v6, sizeof(struct in6_addr));
2458 
2459 		th = (struct tcphdr *)((caddr_t)h6 + sizeof(struct ip6_hdr));
2460 		break;
2461 #endif /* INET6 */
2462 	}
2463 
2464 	/* TCP header */
2465 	th->th_sport = sport;
2466 	th->th_dport = dport;
2467 	th->th_seq = htonl(seq);
2468 	th->th_ack = htonl(ack);
2469 	th->th_off = tlen >> 2;
2470 	th->th_flags = flags;
2471 	th->th_win = htons(win);
2472 
2473 	if (mss) {
2474 		opt = (char *)(th + 1);
2475 		opt[0] = TCPOPT_MAXSEG;
2476 		opt[1] = 4;
2477 		HTONS(mss);
2478 		bcopy((caddr_t)&mss, (caddr_t)(opt + 2), 2);
2479 	}
2480 
2481 	switch (af) {
2482 #ifdef INET
2483 	case AF_INET:
2484 		/* TCP checksum */
2485 		th->th_sum = in_cksum(m, len);
2486 
2487 		/* Finish the IP header */
2488 		h->ip_v = 4;
2489 		h->ip_hl = sizeof(*h) >> 2;
2490 		h->ip_tos = IPTOS_LOWDELAY;
2491 		h->ip_off = htons(V_path_mtu_discovery ? IP_DF : 0);
2492 		h->ip_len = htons(len);
2493 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
2494 		h->ip_sum = 0;
2495 
2496 		pfse->pfse_type = PFSE_IP;
2497 		break;
2498 #endif /* INET */
2499 #ifdef INET6
2500 	case AF_INET6:
2501 		/* TCP checksum */
2502 		th->th_sum = in6_cksum(m, IPPROTO_TCP,
2503 		    sizeof(struct ip6_hdr), tlen);
2504 
2505 		h6->ip6_vfc |= IPV6_VERSION;
2506 		h6->ip6_hlim = IPV6_DEFHLIM;
2507 
2508 		pfse->pfse_type = PFSE_IP6;
2509 		break;
2510 #endif /* INET6 */
2511 	}
2512 	pfse->pfse_m = m;
2513 	pf_send(pfse);
2514 }
2515 
2516 static void
2517 pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af,
2518     struct pf_rule *r)
2519 {
2520 	struct pf_send_entry *pfse;
2521 	struct mbuf *m0;
2522 	struct pf_mtag *pf_mtag;
2523 
2524 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
2525 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
2526 	if (pfse == NULL)
2527 		return;
2528 
2529 	if ((m0 = m_copypacket(m, M_NOWAIT)) == NULL) {
2530 		free(pfse, M_PFTEMP);
2531 		return;
2532 	}
2533 
2534 	if ((pf_mtag = pf_get_mtag(m0)) == NULL) {
2535 		free(pfse, M_PFTEMP);
2536 		return;
2537 	}
2538 	/* XXX: revisit */
2539 	m0->m_flags |= M_SKIP_FIREWALL;
2540 
2541 	if (r->rtableid >= 0)
2542 		M_SETFIB(m0, r->rtableid);
2543 
2544 #ifdef ALTQ
2545 	if (r->qid) {
2546 		pf_mtag->qid = r->qid;
2547 		/* add hints for ecn */
2548 		pf_mtag->hdr = mtod(m0, struct ip *);
2549 	}
2550 #endif /* ALTQ */
2551 
2552 	switch (af) {
2553 #ifdef INET
2554 	case AF_INET:
2555 		pfse->pfse_type = PFSE_ICMP;
2556 		break;
2557 #endif /* INET */
2558 #ifdef INET6
2559 	case AF_INET6:
2560 		pfse->pfse_type = PFSE_ICMP6;
2561 		break;
2562 #endif /* INET6 */
2563 	}
2564 	pfse->pfse_m = m0;
2565 	pfse->pfse_icmp_type = type;
2566 	pfse->pfse_icmp_code = code;
2567 	pf_send(pfse);
2568 }
2569 
2570 /*
2571  * Return 1 if the addresses a and b match (with mask m), otherwise return 0.
2572  * If n is 0, they match if they are equal. If n is != 0, they match if they
2573  * are different.
2574  */
2575 int
2576 pf_match_addr(u_int8_t n, struct pf_addr *a, struct pf_addr *m,
2577     struct pf_addr *b, sa_family_t af)
2578 {
2579 	int	match = 0;
2580 
2581 	switch (af) {
2582 #ifdef INET
2583 	case AF_INET:
2584 		if ((a->addr32[0] & m->addr32[0]) ==
2585 		    (b->addr32[0] & m->addr32[0]))
2586 			match++;
2587 		break;
2588 #endif /* INET */
2589 #ifdef INET6
2590 	case AF_INET6:
2591 		if (((a->addr32[0] & m->addr32[0]) ==
2592 		     (b->addr32[0] & m->addr32[0])) &&
2593 		    ((a->addr32[1] & m->addr32[1]) ==
2594 		     (b->addr32[1] & m->addr32[1])) &&
2595 		    ((a->addr32[2] & m->addr32[2]) ==
2596 		     (b->addr32[2] & m->addr32[2])) &&
2597 		    ((a->addr32[3] & m->addr32[3]) ==
2598 		     (b->addr32[3] & m->addr32[3])))
2599 			match++;
2600 		break;
2601 #endif /* INET6 */
2602 	}
2603 	if (match) {
2604 		if (n)
2605 			return (0);
2606 		else
2607 			return (1);
2608 	} else {
2609 		if (n)
2610 			return (1);
2611 		else
2612 			return (0);
2613 	}
2614 }
2615 
2616 /*
2617  * Return 1 if b <= a <= e, otherwise return 0.
2618  */
2619 int
2620 pf_match_addr_range(struct pf_addr *b, struct pf_addr *e,
2621     struct pf_addr *a, sa_family_t af)
2622 {
2623 	switch (af) {
2624 #ifdef INET
2625 	case AF_INET:
2626 		if ((a->addr32[0] < b->addr32[0]) ||
2627 		    (a->addr32[0] > e->addr32[0]))
2628 			return (0);
2629 		break;
2630 #endif /* INET */
2631 #ifdef INET6
2632 	case AF_INET6: {
2633 		int	i;
2634 
2635 		/* check a >= b */
2636 		for (i = 0; i < 4; ++i)
2637 			if (a->addr32[i] > b->addr32[i])
2638 				break;
2639 			else if (a->addr32[i] < b->addr32[i])
2640 				return (0);
2641 		/* check a <= e */
2642 		for (i = 0; i < 4; ++i)
2643 			if (a->addr32[i] < e->addr32[i])
2644 				break;
2645 			else if (a->addr32[i] > e->addr32[i])
2646 				return (0);
2647 		break;
2648 	}
2649 #endif /* INET6 */
2650 	}
2651 	return (1);
2652 }
2653 
2654 static int
2655 pf_match(u_int8_t op, u_int32_t a1, u_int32_t a2, u_int32_t p)
2656 {
2657 	switch (op) {
2658 	case PF_OP_IRG:
2659 		return ((p > a1) && (p < a2));
2660 	case PF_OP_XRG:
2661 		return ((p < a1) || (p > a2));
2662 	case PF_OP_RRG:
2663 		return ((p >= a1) && (p <= a2));
2664 	case PF_OP_EQ:
2665 		return (p == a1);
2666 	case PF_OP_NE:
2667 		return (p != a1);
2668 	case PF_OP_LT:
2669 		return (p < a1);
2670 	case PF_OP_LE:
2671 		return (p <= a1);
2672 	case PF_OP_GT:
2673 		return (p > a1);
2674 	case PF_OP_GE:
2675 		return (p >= a1);
2676 	}
2677 	return (0); /* never reached */
2678 }
2679 
2680 int
2681 pf_match_port(u_int8_t op, u_int16_t a1, u_int16_t a2, u_int16_t p)
2682 {
2683 	NTOHS(a1);
2684 	NTOHS(a2);
2685 	NTOHS(p);
2686 	return (pf_match(op, a1, a2, p));
2687 }
2688 
2689 static int
2690 pf_match_uid(u_int8_t op, uid_t a1, uid_t a2, uid_t u)
2691 {
2692 	if (u == UID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
2693 		return (0);
2694 	return (pf_match(op, a1, a2, u));
2695 }
2696 
2697 static int
2698 pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g)
2699 {
2700 	if (g == GID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
2701 		return (0);
2702 	return (pf_match(op, a1, a2, g));
2703 }
2704 
2705 int
2706 pf_match_tag(struct mbuf *m, struct pf_rule *r, int *tag, int mtag)
2707 {
2708 	if (*tag == -1)
2709 		*tag = mtag;
2710 
2711 	return ((!r->match_tag_not && r->match_tag == *tag) ||
2712 	    (r->match_tag_not && r->match_tag != *tag));
2713 }
2714 
2715 int
2716 pf_tag_packet(struct mbuf *m, struct pf_pdesc *pd, int tag)
2717 {
2718 
2719 	KASSERT(tag > 0, ("%s: tag %d", __func__, tag));
2720 
2721 	if (pd->pf_mtag == NULL && ((pd->pf_mtag = pf_get_mtag(m)) == NULL))
2722 		return (ENOMEM);
2723 
2724 	pd->pf_mtag->tag = tag;
2725 
2726 	return (0);
2727 }
2728 
2729 #define	PF_ANCHOR_STACKSIZE	32
2730 struct pf_anchor_stackframe {
2731 	struct pf_ruleset	*rs;
2732 	struct pf_rule		*r;	/* XXX: + match bit */
2733 	struct pf_anchor	*child;
2734 };
2735 
2736 /*
2737  * XXX: We rely on malloc(9) returning pointer aligned addresses.
2738  */
2739 #define	PF_ANCHORSTACK_MATCH	0x00000001
2740 #define	PF_ANCHORSTACK_MASK	(PF_ANCHORSTACK_MATCH)
2741 
2742 #define	PF_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
2743 #define	PF_ANCHOR_RULE(f)	(struct pf_rule *)			\
2744 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
2745 #define	PF_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 			\
2746 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
2747 } while (0)
2748 
2749 void
2750 pf_step_into_anchor(struct pf_anchor_stackframe *stack, int *depth,
2751     struct pf_ruleset **rs, int n, struct pf_rule **r, struct pf_rule **a,
2752     int *match)
2753 {
2754 	struct pf_anchor_stackframe	*f;
2755 
2756 	PF_RULES_RASSERT();
2757 
2758 	if (match)
2759 		*match = 0;
2760 	if (*depth >= PF_ANCHOR_STACKSIZE) {
2761 		printf("%s: anchor stack overflow on %s\n",
2762 		    __func__, (*r)->anchor->name);
2763 		*r = TAILQ_NEXT(*r, entries);
2764 		return;
2765 	} else if (*depth == 0 && a != NULL)
2766 		*a = *r;
2767 	f = stack + (*depth)++;
2768 	f->rs = *rs;
2769 	f->r = *r;
2770 	if ((*r)->anchor_wildcard) {
2771 		struct pf_anchor_node *parent = &(*r)->anchor->children;
2772 
2773 		if ((f->child = RB_MIN(pf_anchor_node, parent)) == NULL) {
2774 			*r = NULL;
2775 			return;
2776 		}
2777 		*rs = &f->child->ruleset;
2778 	} else {
2779 		f->child = NULL;
2780 		*rs = &(*r)->anchor->ruleset;
2781 	}
2782 	*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
2783 }
2784 
2785 int
2786 pf_step_out_of_anchor(struct pf_anchor_stackframe *stack, int *depth,
2787     struct pf_ruleset **rs, int n, struct pf_rule **r, struct pf_rule **a,
2788     int *match)
2789 {
2790 	struct pf_anchor_stackframe	*f;
2791 	struct pf_rule *fr;
2792 	int quick = 0;
2793 
2794 	PF_RULES_RASSERT();
2795 
2796 	do {
2797 		if (*depth <= 0)
2798 			break;
2799 		f = stack + *depth - 1;
2800 		fr = PF_ANCHOR_RULE(f);
2801 		if (f->child != NULL) {
2802 			struct pf_anchor_node *parent;
2803 
2804 			/*
2805 			 * This block traverses through
2806 			 * a wildcard anchor.
2807 			 */
2808 			parent = &fr->anchor->children;
2809 			if (match != NULL && *match) {
2810 				/*
2811 				 * If any of "*" matched, then
2812 				 * "foo/ *" matched, mark frame
2813 				 * appropriately.
2814 				 */
2815 				PF_ANCHOR_SET_MATCH(f);
2816 				*match = 0;
2817 			}
2818 			f->child = RB_NEXT(pf_anchor_node, parent, f->child);
2819 			if (f->child != NULL) {
2820 				*rs = &f->child->ruleset;
2821 				*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
2822 				if (*r == NULL)
2823 					continue;
2824 				else
2825 					break;
2826 			}
2827 		}
2828 		(*depth)--;
2829 		if (*depth == 0 && a != NULL)
2830 			*a = NULL;
2831 		*rs = f->rs;
2832 		if (PF_ANCHOR_MATCH(f) || (match != NULL && *match))
2833 			quick = fr->quick;
2834 		*r = TAILQ_NEXT(fr, entries);
2835 	} while (*r == NULL);
2836 
2837 	return (quick);
2838 }
2839 
2840 #ifdef INET6
2841 void
2842 pf_poolmask(struct pf_addr *naddr, struct pf_addr *raddr,
2843     struct pf_addr *rmask, struct pf_addr *saddr, sa_family_t af)
2844 {
2845 	switch (af) {
2846 #ifdef INET
2847 	case AF_INET:
2848 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
2849 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
2850 		break;
2851 #endif /* INET */
2852 	case AF_INET6:
2853 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
2854 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
2855 		naddr->addr32[1] = (raddr->addr32[1] & rmask->addr32[1]) |
2856 		((rmask->addr32[1] ^ 0xffffffff ) & saddr->addr32[1]);
2857 		naddr->addr32[2] = (raddr->addr32[2] & rmask->addr32[2]) |
2858 		((rmask->addr32[2] ^ 0xffffffff ) & saddr->addr32[2]);
2859 		naddr->addr32[3] = (raddr->addr32[3] & rmask->addr32[3]) |
2860 		((rmask->addr32[3] ^ 0xffffffff ) & saddr->addr32[3]);
2861 		break;
2862 	}
2863 }
2864 
2865 void
2866 pf_addr_inc(struct pf_addr *addr, sa_family_t af)
2867 {
2868 	switch (af) {
2869 #ifdef INET
2870 	case AF_INET:
2871 		addr->addr32[0] = htonl(ntohl(addr->addr32[0]) + 1);
2872 		break;
2873 #endif /* INET */
2874 	case AF_INET6:
2875 		if (addr->addr32[3] == 0xffffffff) {
2876 			addr->addr32[3] = 0;
2877 			if (addr->addr32[2] == 0xffffffff) {
2878 				addr->addr32[2] = 0;
2879 				if (addr->addr32[1] == 0xffffffff) {
2880 					addr->addr32[1] = 0;
2881 					addr->addr32[0] =
2882 					    htonl(ntohl(addr->addr32[0]) + 1);
2883 				} else
2884 					addr->addr32[1] =
2885 					    htonl(ntohl(addr->addr32[1]) + 1);
2886 			} else
2887 				addr->addr32[2] =
2888 				    htonl(ntohl(addr->addr32[2]) + 1);
2889 		} else
2890 			addr->addr32[3] =
2891 			    htonl(ntohl(addr->addr32[3]) + 1);
2892 		break;
2893 	}
2894 }
2895 #endif /* INET6 */
2896 
2897 int
2898 pf_socket_lookup(int direction, struct pf_pdesc *pd, struct mbuf *m)
2899 {
2900 	struct pf_addr		*saddr, *daddr;
2901 	u_int16_t		 sport, dport;
2902 	struct inpcbinfo	*pi;
2903 	struct inpcb		*inp;
2904 
2905 	pd->lookup.uid = UID_MAX;
2906 	pd->lookup.gid = GID_MAX;
2907 
2908 	switch (pd->proto) {
2909 	case IPPROTO_TCP:
2910 		if (pd->hdr.tcp == NULL)
2911 			return (-1);
2912 		sport = pd->hdr.tcp->th_sport;
2913 		dport = pd->hdr.tcp->th_dport;
2914 		pi = &V_tcbinfo;
2915 		break;
2916 	case IPPROTO_UDP:
2917 		if (pd->hdr.udp == NULL)
2918 			return (-1);
2919 		sport = pd->hdr.udp->uh_sport;
2920 		dport = pd->hdr.udp->uh_dport;
2921 		pi = &V_udbinfo;
2922 		break;
2923 	default:
2924 		return (-1);
2925 	}
2926 	if (direction == PF_IN) {
2927 		saddr = pd->src;
2928 		daddr = pd->dst;
2929 	} else {
2930 		u_int16_t	p;
2931 
2932 		p = sport;
2933 		sport = dport;
2934 		dport = p;
2935 		saddr = pd->dst;
2936 		daddr = pd->src;
2937 	}
2938 	switch (pd->af) {
2939 #ifdef INET
2940 	case AF_INET:
2941 		inp = in_pcblookup_mbuf(pi, saddr->v4, sport, daddr->v4,
2942 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
2943 		if (inp == NULL) {
2944 			inp = in_pcblookup_mbuf(pi, saddr->v4, sport,
2945 			   daddr->v4, dport, INPLOOKUP_WILDCARD |
2946 			   INPLOOKUP_RLOCKPCB, NULL, m);
2947 			if (inp == NULL)
2948 				return (-1);
2949 		}
2950 		break;
2951 #endif /* INET */
2952 #ifdef INET6
2953 	case AF_INET6:
2954 		inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport, &daddr->v6,
2955 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
2956 		if (inp == NULL) {
2957 			inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport,
2958 			    &daddr->v6, dport, INPLOOKUP_WILDCARD |
2959 			    INPLOOKUP_RLOCKPCB, NULL, m);
2960 			if (inp == NULL)
2961 				return (-1);
2962 		}
2963 		break;
2964 #endif /* INET6 */
2965 
2966 	default:
2967 		return (-1);
2968 	}
2969 	INP_RLOCK_ASSERT(inp);
2970 	pd->lookup.uid = inp->inp_cred->cr_uid;
2971 	pd->lookup.gid = inp->inp_cred->cr_groups[0];
2972 	INP_RUNLOCK(inp);
2973 
2974 	return (1);
2975 }
2976 
2977 static u_int8_t
2978 pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
2979 {
2980 	int		 hlen;
2981 	u_int8_t	 hdr[60];
2982 	u_int8_t	*opt, optlen;
2983 	u_int8_t	 wscale = 0;
2984 
2985 	hlen = th_off << 2;		/* hlen <= sizeof(hdr) */
2986 	if (hlen <= sizeof(struct tcphdr))
2987 		return (0);
2988 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
2989 		return (0);
2990 	opt = hdr + sizeof(struct tcphdr);
2991 	hlen -= sizeof(struct tcphdr);
2992 	while (hlen >= 3) {
2993 		switch (*opt) {
2994 		case TCPOPT_EOL:
2995 		case TCPOPT_NOP:
2996 			++opt;
2997 			--hlen;
2998 			break;
2999 		case TCPOPT_WINDOW:
3000 			wscale = opt[2];
3001 			if (wscale > TCP_MAX_WINSHIFT)
3002 				wscale = TCP_MAX_WINSHIFT;
3003 			wscale |= PF_WSCALE_FLAG;
3004 			/* FALLTHROUGH */
3005 		default:
3006 			optlen = opt[1];
3007 			if (optlen < 2)
3008 				optlen = 2;
3009 			hlen -= optlen;
3010 			opt += optlen;
3011 			break;
3012 		}
3013 	}
3014 	return (wscale);
3015 }
3016 
3017 static u_int16_t
3018 pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
3019 {
3020 	int		 hlen;
3021 	u_int8_t	 hdr[60];
3022 	u_int8_t	*opt, optlen;
3023 	u_int16_t	 mss = V_tcp_mssdflt;
3024 
3025 	hlen = th_off << 2;	/* hlen <= sizeof(hdr) */
3026 	if (hlen <= sizeof(struct tcphdr))
3027 		return (0);
3028 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
3029 		return (0);
3030 	opt = hdr + sizeof(struct tcphdr);
3031 	hlen -= sizeof(struct tcphdr);
3032 	while (hlen >= TCPOLEN_MAXSEG) {
3033 		switch (*opt) {
3034 		case TCPOPT_EOL:
3035 		case TCPOPT_NOP:
3036 			++opt;
3037 			--hlen;
3038 			break;
3039 		case TCPOPT_MAXSEG:
3040 			bcopy((caddr_t)(opt + 2), (caddr_t)&mss, 2);
3041 			NTOHS(mss);
3042 			/* FALLTHROUGH */
3043 		default:
3044 			optlen = opt[1];
3045 			if (optlen < 2)
3046 				optlen = 2;
3047 			hlen -= optlen;
3048 			opt += optlen;
3049 			break;
3050 		}
3051 	}
3052 	return (mss);
3053 }
3054 
3055 static u_int16_t
3056 pf_calc_mss(struct pf_addr *addr, sa_family_t af, int rtableid, u_int16_t offer)
3057 {
3058 #ifdef INET
3059 	struct sockaddr_in	*dst;
3060 	struct route		 ro;
3061 #endif /* INET */
3062 #ifdef INET6
3063 	struct sockaddr_in6	*dst6;
3064 	struct route_in6	 ro6;
3065 #endif /* INET6 */
3066 	struct rtentry		*rt = NULL;
3067 	int			 hlen = 0;
3068 	u_int16_t		 mss = V_tcp_mssdflt;
3069 
3070 	switch (af) {
3071 #ifdef INET
3072 	case AF_INET:
3073 		hlen = sizeof(struct ip);
3074 		bzero(&ro, sizeof(ro));
3075 		dst = (struct sockaddr_in *)&ro.ro_dst;
3076 		dst->sin_family = AF_INET;
3077 		dst->sin_len = sizeof(*dst);
3078 		dst->sin_addr = addr->v4;
3079 		in_rtalloc_ign(&ro, 0, rtableid);
3080 		rt = ro.ro_rt;
3081 		break;
3082 #endif /* INET */
3083 #ifdef INET6
3084 	case AF_INET6:
3085 		hlen = sizeof(struct ip6_hdr);
3086 		bzero(&ro6, sizeof(ro6));
3087 		dst6 = (struct sockaddr_in6 *)&ro6.ro_dst;
3088 		dst6->sin6_family = AF_INET6;
3089 		dst6->sin6_len = sizeof(*dst6);
3090 		dst6->sin6_addr = addr->v6;
3091 		in6_rtalloc_ign(&ro6, 0, rtableid);
3092 		rt = ro6.ro_rt;
3093 		break;
3094 #endif /* INET6 */
3095 	}
3096 
3097 	if (rt && rt->rt_ifp) {
3098 		mss = rt->rt_ifp->if_mtu - hlen - sizeof(struct tcphdr);
3099 		mss = max(V_tcp_mssdflt, mss);
3100 		RTFREE(rt);
3101 	}
3102 	mss = min(mss, offer);
3103 	mss = max(mss, 64);		/* sanity - at least max opt space */
3104 	return (mss);
3105 }
3106 
3107 static void
3108 pf_set_rt_ifp(struct pf_state *s, struct pf_addr *saddr)
3109 {
3110 	struct pf_rule *r = s->rule.ptr;
3111 	struct pf_src_node *sn = NULL;
3112 
3113 	s->rt_kif = NULL;
3114 	if (!r->rt || r->rt == PF_FASTROUTE)
3115 		return;
3116 	switch (s->key[PF_SK_WIRE]->af) {
3117 #ifdef INET
3118 	case AF_INET:
3119 		pf_map_addr(AF_INET, r, saddr, &s->rt_addr, NULL, &sn);
3120 		s->rt_kif = r->rpool.cur->kif;
3121 		break;
3122 #endif /* INET */
3123 #ifdef INET6
3124 	case AF_INET6:
3125 		pf_map_addr(AF_INET6, r, saddr, &s->rt_addr, NULL, &sn);
3126 		s->rt_kif = r->rpool.cur->kif;
3127 		break;
3128 #endif /* INET6 */
3129 	}
3130 }
3131 
3132 static u_int32_t
3133 pf_tcp_iss(struct pf_pdesc *pd)
3134 {
3135 	MD5_CTX ctx;
3136 	u_int32_t digest[4];
3137 
3138 	if (V_pf_tcp_secret_init == 0) {
3139 		read_random(&V_pf_tcp_secret, sizeof(V_pf_tcp_secret));
3140 		MD5Init(&V_pf_tcp_secret_ctx);
3141 		MD5Update(&V_pf_tcp_secret_ctx, V_pf_tcp_secret,
3142 		    sizeof(V_pf_tcp_secret));
3143 		V_pf_tcp_secret_init = 1;
3144 	}
3145 
3146 	ctx = V_pf_tcp_secret_ctx;
3147 
3148 	MD5Update(&ctx, (char *)&pd->hdr.tcp->th_sport, sizeof(u_short));
3149 	MD5Update(&ctx, (char *)&pd->hdr.tcp->th_dport, sizeof(u_short));
3150 	if (pd->af == AF_INET6) {
3151 		MD5Update(&ctx, (char *)&pd->src->v6, sizeof(struct in6_addr));
3152 		MD5Update(&ctx, (char *)&pd->dst->v6, sizeof(struct in6_addr));
3153 	} else {
3154 		MD5Update(&ctx, (char *)&pd->src->v4, sizeof(struct in_addr));
3155 		MD5Update(&ctx, (char *)&pd->dst->v4, sizeof(struct in_addr));
3156 	}
3157 	MD5Final((u_char *)digest, &ctx);
3158 	V_pf_tcp_iss_off += 4096;
3159 #define	ISN_RANDOM_INCREMENT (4096 - 1)
3160 	return (digest[0] + (arc4random() & ISN_RANDOM_INCREMENT) +
3161 	    V_pf_tcp_iss_off);
3162 #undef	ISN_RANDOM_INCREMENT
3163 }
3164 
3165 static int
3166 pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction,
3167     struct pfi_kif *kif, struct mbuf *m, int off, struct pf_pdesc *pd,
3168     struct pf_rule **am, struct pf_ruleset **rsm, struct inpcb *inp)
3169 {
3170 	struct pf_rule		*nr = NULL;
3171 	struct pf_addr		* const saddr = pd->src;
3172 	struct pf_addr		* const daddr = pd->dst;
3173 	sa_family_t		 af = pd->af;
3174 	struct pf_rule		*r, *a = NULL;
3175 	struct pf_ruleset	*ruleset = NULL;
3176 	struct pf_src_node	*nsn = NULL;
3177 	struct tcphdr		*th = pd->hdr.tcp;
3178 	struct pf_state_key	*sk = NULL, *nk = NULL;
3179 	u_short			 reason;
3180 	int			 rewrite = 0, hdrlen = 0;
3181 	int			 tag = -1, rtableid = -1;
3182 	int			 asd = 0;
3183 	int			 match = 0;
3184 	int			 state_icmp = 0, icmp_dir, multi;
3185 	uint16_t		 sport = 0 , dport = 0, virtual_type = 0, virtual_id = 0;
3186 	u_int16_t		 bproto_sum = 0, bip_sum = 0;
3187 	u_int8_t		 icmptype = 0, icmpcode = 0;
3188 	struct pf_anchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
3189 
3190 	PF_RULES_RASSERT();
3191 
3192 	if (inp != NULL) {
3193 		INP_LOCK_ASSERT(inp);
3194 		pd->lookup.uid = inp->inp_cred->cr_uid;
3195 		pd->lookup.gid = inp->inp_cred->cr_groups[0];
3196 		pd->lookup.done = 1;
3197 	}
3198 
3199 	switch (pd->proto) {
3200 	case IPPROTO_TCP:
3201 		sport = th->th_sport;
3202 		dport = th->th_dport;
3203 		hdrlen = sizeof(*th);
3204 		break;
3205 	case IPPROTO_UDP:
3206 		sport = pd->hdr.udp->uh_sport;
3207 		dport = pd->hdr.udp->uh_dport;
3208 		hdrlen = sizeof(*pd->hdr.udp);
3209 		break;
3210 #ifdef INET
3211 	case IPPROTO_ICMP:
3212 		if (pd->af != AF_INET)
3213 			break;
3214 		hdrlen = sizeof(*pd->hdr.icmp);
3215 		icmptype = pd->hdr.icmp->icmp_type;
3216 		icmpcode = pd->hdr.icmp->icmp_code;
3217 
3218 		state_icmp = pf_icmp_mapping(pd, icmptype,
3219 		    &icmp_dir, &multi, &virtual_id, &virtual_type);
3220 		if (icmp_dir == PF_IN) {
3221 			sport = virtual_id;
3222 			dport = virtual_type;
3223 		} else {
3224 			sport = virtual_type;
3225 			dport = virtual_id;
3226 		}
3227 		break;
3228 #endif /* INET */
3229 #ifdef INET6
3230 	case IPPROTO_ICMPV6:
3231 		if (af != AF_INET6)
3232 			break;
3233 		hdrlen = sizeof(*pd->hdr.icmp6);
3234 		icmptype = pd->hdr.icmp6->icmp6_type;
3235 		icmpcode = pd->hdr.icmp6->icmp6_code;
3236 
3237 		state_icmp = pf_icmp_mapping(pd, icmptype,
3238 		    &icmp_dir, &multi, &virtual_id, &virtual_type);
3239 		if (icmp_dir == PF_IN) {
3240 			sport = virtual_id;
3241 			dport = virtual_type;
3242 		} else {
3243 			sport = virtual_type;
3244 			dport = virtual_id;
3245 		}
3246 		break;
3247 #endif /* INET6 */
3248 	default:
3249 		sport = dport = hdrlen = 0;
3250 		break;
3251 	}
3252 
3253 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
3254 
3255 	/* check packet for BINAT/NAT/RDR */
3256 	if ((nr = pf_get_translation(pd, m, off, direction, kif, &nsn, &sk,
3257 	    &nk, saddr, daddr, sport, dport, anchor_stack)) != NULL) {
3258 		KASSERT(sk != NULL, ("%s: null sk", __func__));
3259 		KASSERT(nk != NULL, ("%s: null nk", __func__));
3260 
3261 		if (pd->ip_sum)
3262 			bip_sum = *pd->ip_sum;
3263 
3264 		switch (pd->proto) {
3265 		case IPPROTO_TCP:
3266 			bproto_sum = th->th_sum;
3267 			pd->proto_sum = &th->th_sum;
3268 
3269 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
3270 			    nk->port[pd->sidx] != sport) {
3271 				pf_change_ap(saddr, &th->th_sport, pd->ip_sum,
3272 				    &th->th_sum, &nk->addr[pd->sidx],
3273 				    nk->port[pd->sidx], 0, af);
3274 				pd->sport = &th->th_sport;
3275 				sport = th->th_sport;
3276 			}
3277 
3278 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
3279 			    nk->port[pd->didx] != dport) {
3280 				pf_change_ap(daddr, &th->th_dport, pd->ip_sum,
3281 				    &th->th_sum, &nk->addr[pd->didx],
3282 				    nk->port[pd->didx], 0, af);
3283 				dport = th->th_dport;
3284 				pd->dport = &th->th_dport;
3285 			}
3286 			rewrite++;
3287 			break;
3288 		case IPPROTO_UDP:
3289 			bproto_sum = pd->hdr.udp->uh_sum;
3290 			pd->proto_sum = &pd->hdr.udp->uh_sum;
3291 
3292 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
3293 			    nk->port[pd->sidx] != sport) {
3294 				pf_change_ap(saddr, &pd->hdr.udp->uh_sport,
3295 				    pd->ip_sum, &pd->hdr.udp->uh_sum,
3296 				    &nk->addr[pd->sidx],
3297 				    nk->port[pd->sidx], 1, af);
3298 				sport = pd->hdr.udp->uh_sport;
3299 				pd->sport = &pd->hdr.udp->uh_sport;
3300 			}
3301 
3302 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
3303 			    nk->port[pd->didx] != dport) {
3304 				pf_change_ap(daddr, &pd->hdr.udp->uh_dport,
3305 				    pd->ip_sum, &pd->hdr.udp->uh_sum,
3306 				    &nk->addr[pd->didx],
3307 				    nk->port[pd->didx], 1, af);
3308 				dport = pd->hdr.udp->uh_dport;
3309 				pd->dport = &pd->hdr.udp->uh_dport;
3310 			}
3311 			rewrite++;
3312 			break;
3313 #ifdef INET
3314 		case IPPROTO_ICMP:
3315 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET))
3316 				pf_change_a(&saddr->v4.s_addr, pd->ip_sum,
3317 				    nk->addr[pd->sidx].v4.s_addr, 0);
3318 
3319 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET))
3320 				pf_change_a(&daddr->v4.s_addr, pd->ip_sum,
3321 				    nk->addr[pd->didx].v4.s_addr, 0);
3322 
3323 			if (virtual_type == ICMP_ECHO &&
3324 			     nk->port[pd->sidx] != pd->hdr.icmp->icmp_id) {
3325 				pd->hdr.icmp->icmp_cksum = pf_cksum_fixup(
3326 				    pd->hdr.icmp->icmp_cksum, sport,
3327 				    nk->port[pd->sidx], 0);
3328 				pd->hdr.icmp->icmp_id = nk->port[pd->sidx];
3329 				pd->sport = &pd->hdr.icmp->icmp_id;
3330 			}
3331 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)pd->hdr.icmp);
3332 			break;
3333 #endif /* INET */
3334 #ifdef INET6
3335 		case IPPROTO_ICMPV6:
3336 			nk->port[0] = nk->port[1];
3337 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET6))
3338 				pf_change_a6(saddr, &pd->hdr.icmp6->icmp6_cksum,
3339 				    &nk->addr[pd->sidx], 0);
3340 
3341 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET6))
3342 				pf_change_a6(daddr, &pd->hdr.icmp6->icmp6_cksum,
3343 				    &nk->addr[pd->didx], 0);
3344 			rewrite++;
3345 			break;
3346 #endif /* INET */
3347 		default:
3348 			switch (af) {
3349 #ifdef INET
3350 			case AF_INET:
3351 				if (PF_ANEQ(saddr,
3352 				    &nk->addr[pd->sidx], AF_INET))
3353 					pf_change_a(&saddr->v4.s_addr,
3354 					    pd->ip_sum,
3355 					    nk->addr[pd->sidx].v4.s_addr, 0);
3356 
3357 				if (PF_ANEQ(daddr,
3358 				    &nk->addr[pd->didx], AF_INET))
3359 					pf_change_a(&daddr->v4.s_addr,
3360 					    pd->ip_sum,
3361 					    nk->addr[pd->didx].v4.s_addr, 0);
3362 				break;
3363 #endif /* INET */
3364 #ifdef INET6
3365 			case AF_INET6:
3366 				if (PF_ANEQ(saddr,
3367 				    &nk->addr[pd->sidx], AF_INET6))
3368 					PF_ACPY(saddr, &nk->addr[pd->sidx], af);
3369 
3370 				if (PF_ANEQ(daddr,
3371 				    &nk->addr[pd->didx], AF_INET6))
3372 					PF_ACPY(saddr, &nk->addr[pd->didx], af);
3373 				break;
3374 #endif /* INET */
3375 			}
3376 			break;
3377 		}
3378 		if (nr->natpass)
3379 			r = NULL;
3380 		pd->nat_rule = nr;
3381 	}
3382 
3383 	while (r != NULL) {
3384 		r->evaluations++;
3385 		if (pfi_kif_match(r->kif, kif) == r->ifnot)
3386 			r = r->skip[PF_SKIP_IFP].ptr;
3387 		else if (r->direction && r->direction != direction)
3388 			r = r->skip[PF_SKIP_DIR].ptr;
3389 		else if (r->af && r->af != af)
3390 			r = r->skip[PF_SKIP_AF].ptr;
3391 		else if (r->proto && r->proto != pd->proto)
3392 			r = r->skip[PF_SKIP_PROTO].ptr;
3393 		else if (PF_MISMATCHAW(&r->src.addr, saddr, af,
3394 		    r->src.neg, kif, M_GETFIB(m)))
3395 			r = r->skip[PF_SKIP_SRC_ADDR].ptr;
3396 		/* tcp/udp only. port_op always 0 in other cases */
3397 		else if (r->src.port_op && !pf_match_port(r->src.port_op,
3398 		    r->src.port[0], r->src.port[1], sport))
3399 			r = r->skip[PF_SKIP_SRC_PORT].ptr;
3400 		else if (PF_MISMATCHAW(&r->dst.addr, daddr, af,
3401 		    r->dst.neg, NULL, M_GETFIB(m)))
3402 			r = r->skip[PF_SKIP_DST_ADDR].ptr;
3403 		/* tcp/udp only. port_op always 0 in other cases */
3404 		else if (r->dst.port_op && !pf_match_port(r->dst.port_op,
3405 		    r->dst.port[0], r->dst.port[1], dport))
3406 			r = r->skip[PF_SKIP_DST_PORT].ptr;
3407 		/* icmp only. type always 0 in other cases */
3408 		else if (r->type && r->type != icmptype + 1)
3409 			r = TAILQ_NEXT(r, entries);
3410 		/* icmp only. type always 0 in other cases */
3411 		else if (r->code && r->code != icmpcode + 1)
3412 			r = TAILQ_NEXT(r, entries);
3413 		else if (r->tos && !(r->tos == pd->tos))
3414 			r = TAILQ_NEXT(r, entries);
3415 		else if (r->rule_flag & PFRULE_FRAGMENT)
3416 			r = TAILQ_NEXT(r, entries);
3417 		else if (pd->proto == IPPROTO_TCP &&
3418 		    (r->flagset & th->th_flags) != r->flags)
3419 			r = TAILQ_NEXT(r, entries);
3420 		/* tcp/udp only. uid.op always 0 in other cases */
3421 		else if (r->uid.op && (pd->lookup.done || (pd->lookup.done =
3422 		    pf_socket_lookup(direction, pd, m), 1)) &&
3423 		    !pf_match_uid(r->uid.op, r->uid.uid[0], r->uid.uid[1],
3424 		    pd->lookup.uid))
3425 			r = TAILQ_NEXT(r, entries);
3426 		/* tcp/udp only. gid.op always 0 in other cases */
3427 		else if (r->gid.op && (pd->lookup.done || (pd->lookup.done =
3428 		    pf_socket_lookup(direction, pd, m), 1)) &&
3429 		    !pf_match_gid(r->gid.op, r->gid.gid[0], r->gid.gid[1],
3430 		    pd->lookup.gid))
3431 			r = TAILQ_NEXT(r, entries);
3432 		else if (r->prob &&
3433 		    r->prob <= arc4random())
3434 			r = TAILQ_NEXT(r, entries);
3435 		else if (r->match_tag && !pf_match_tag(m, r, &tag,
3436 		    pd->pf_mtag ? pd->pf_mtag->tag : 0))
3437 			r = TAILQ_NEXT(r, entries);
3438 		else if (r->os_fingerprint != PF_OSFP_ANY &&
3439 		    (pd->proto != IPPROTO_TCP || !pf_osfp_match(
3440 		    pf_osfp_fingerprint(pd, m, off, th),
3441 		    r->os_fingerprint)))
3442 			r = TAILQ_NEXT(r, entries);
3443 		else {
3444 			if (r->tag)
3445 				tag = r->tag;
3446 			if (r->rtableid >= 0)
3447 				rtableid = r->rtableid;
3448 			if (r->anchor == NULL) {
3449 				match = 1;
3450 				*rm = r;
3451 				*am = a;
3452 				*rsm = ruleset;
3453 				if ((*rm)->quick)
3454 					break;
3455 				r = TAILQ_NEXT(r, entries);
3456 			} else
3457 				pf_step_into_anchor(anchor_stack, &asd,
3458 				    &ruleset, PF_RULESET_FILTER, &r, &a,
3459 				    &match);
3460 		}
3461 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
3462 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
3463 			break;
3464 	}
3465 	r = *rm;
3466 	a = *am;
3467 	ruleset = *rsm;
3468 
3469 	REASON_SET(&reason, PFRES_MATCH);
3470 
3471 	if (r->log || (nr != NULL && nr->log)) {
3472 		if (rewrite)
3473 			m_copyback(m, off, hdrlen, pd->hdr.any);
3474 		PFLOG_PACKET(kif, m, af, direction, reason, r->log ? r : nr, a,
3475 		    ruleset, pd, 1);
3476 	}
3477 
3478 	if ((r->action == PF_DROP) &&
3479 	    ((r->rule_flag & PFRULE_RETURNRST) ||
3480 	    (r->rule_flag & PFRULE_RETURNICMP) ||
3481 	    (r->rule_flag & PFRULE_RETURN))) {
3482 		/* undo NAT changes, if they have taken place */
3483 		if (nr != NULL) {
3484 			PF_ACPY(saddr, &sk->addr[pd->sidx], af);
3485 			PF_ACPY(daddr, &sk->addr[pd->didx], af);
3486 			if (pd->sport)
3487 				*pd->sport = sk->port[pd->sidx];
3488 			if (pd->dport)
3489 				*pd->dport = sk->port[pd->didx];
3490 			if (pd->proto_sum)
3491 				*pd->proto_sum = bproto_sum;
3492 			if (pd->ip_sum)
3493 				*pd->ip_sum = bip_sum;
3494 			m_copyback(m, off, hdrlen, pd->hdr.any);
3495 		}
3496 		if (pd->proto == IPPROTO_TCP &&
3497 		    ((r->rule_flag & PFRULE_RETURNRST) ||
3498 		    (r->rule_flag & PFRULE_RETURN)) &&
3499 		    !(th->th_flags & TH_RST)) {
3500 			u_int32_t	 ack = ntohl(th->th_seq) + pd->p_len;
3501 			int		 len = 0;
3502 #ifdef INET
3503 			struct ip	*h4;
3504 #endif
3505 #ifdef INET6
3506 			struct ip6_hdr	*h6;
3507 #endif
3508 
3509 			switch (af) {
3510 #ifdef INET
3511 			case AF_INET:
3512 				h4 = mtod(m, struct ip *);
3513 				len = ntohs(h4->ip_len) - off;
3514 				break;
3515 #endif
3516 #ifdef INET6
3517 			case AF_INET6:
3518 				h6 = mtod(m, struct ip6_hdr *);
3519 				len = ntohs(h6->ip6_plen) - (off - sizeof(*h6));
3520 				break;
3521 #endif
3522 			}
3523 
3524 			if (pf_check_proto_cksum(m, off, len, IPPROTO_TCP, af))
3525 				REASON_SET(&reason, PFRES_PROTCKSUM);
3526 			else {
3527 				if (th->th_flags & TH_SYN)
3528 					ack++;
3529 				if (th->th_flags & TH_FIN)
3530 					ack++;
3531 				pf_send_tcp(m, r, af, pd->dst,
3532 				    pd->src, th->th_dport, th->th_sport,
3533 				    ntohl(th->th_ack), ack, TH_RST|TH_ACK, 0, 0,
3534 				    r->return_ttl, 1, 0, kif->pfik_ifp);
3535 			}
3536 		} else if (pd->proto != IPPROTO_ICMP && af == AF_INET &&
3537 		    r->return_icmp)
3538 			pf_send_icmp(m, r->return_icmp >> 8,
3539 			    r->return_icmp & 255, af, r);
3540 		else if (pd->proto != IPPROTO_ICMPV6 && af == AF_INET6 &&
3541 		    r->return_icmp6)
3542 			pf_send_icmp(m, r->return_icmp6 >> 8,
3543 			    r->return_icmp6 & 255, af, r);
3544 	}
3545 
3546 	if (r->action == PF_DROP)
3547 		goto cleanup;
3548 
3549 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
3550 		REASON_SET(&reason, PFRES_MEMORY);
3551 		goto cleanup;
3552 	}
3553 	if (rtableid >= 0)
3554 		M_SETFIB(m, rtableid);
3555 
3556 	if (!state_icmp && (r->keep_state || nr != NULL ||
3557 	    (pd->flags & PFDESC_TCP_NORM))) {
3558 		int action;
3559 		action = pf_create_state(r, nr, a, pd, nsn, nk, sk, m, off,
3560 		    sport, dport, &rewrite, kif, sm, tag, bproto_sum, bip_sum,
3561 		    hdrlen);
3562 		if (action != PF_PASS)
3563 			return (action);
3564 	} else {
3565 		if (sk != NULL)
3566 			uma_zfree(V_pf_state_key_z, sk);
3567 		if (nk != NULL)
3568 			uma_zfree(V_pf_state_key_z, nk);
3569 	}
3570 
3571 	/* copy back packet headers if we performed NAT operations */
3572 	if (rewrite)
3573 		m_copyback(m, off, hdrlen, pd->hdr.any);
3574 
3575 	if (*sm != NULL && !((*sm)->state_flags & PFSTATE_NOSYNC) &&
3576 	    direction == PF_OUT &&
3577 	    pfsync_defer_ptr != NULL && pfsync_defer_ptr(*sm, m))
3578 		/*
3579 		 * We want the state created, but we dont
3580 		 * want to send this in case a partner
3581 		 * firewall has to know about it to allow
3582 		 * replies through it.
3583 		 */
3584 		return (PF_DEFER);
3585 
3586 	return (PF_PASS);
3587 
3588 cleanup:
3589 	if (sk != NULL)
3590 		uma_zfree(V_pf_state_key_z, sk);
3591 	if (nk != NULL)
3592 		uma_zfree(V_pf_state_key_z, nk);
3593 	return (PF_DROP);
3594 }
3595 
3596 static int
3597 pf_create_state(struct pf_rule *r, struct pf_rule *nr, struct pf_rule *a,
3598     struct pf_pdesc *pd, struct pf_src_node *nsn, struct pf_state_key *nk,
3599     struct pf_state_key *sk, struct mbuf *m, int off, u_int16_t sport,
3600     u_int16_t dport, int *rewrite, struct pfi_kif *kif, struct pf_state **sm,
3601     int tag, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen)
3602 {
3603 	struct pf_state		*s = NULL;
3604 	struct pf_src_node	*sn = NULL;
3605 	struct tcphdr		*th = pd->hdr.tcp;
3606 	u_int16_t		 mss = V_tcp_mssdflt;
3607 	u_short			 reason;
3608 
3609 	/* check maximums */
3610 	if (r->max_states && (r->states_cur >= r->max_states)) {
3611 		V_pf_status.lcounters[LCNT_STATES]++;
3612 		REASON_SET(&reason, PFRES_MAXSTATES);
3613 		return (PF_DROP);
3614 	}
3615 	/* src node for filter rule */
3616 	if ((r->rule_flag & PFRULE_SRCTRACK ||
3617 	    r->rpool.opts & PF_POOL_STICKYADDR) &&
3618 	    pf_insert_src_node(&sn, r, pd->src, pd->af) != 0) {
3619 		REASON_SET(&reason, PFRES_SRCLIMIT);
3620 		goto csfailed;
3621 	}
3622 	/* src node for translation rule */
3623 	if (nr != NULL && (nr->rpool.opts & PF_POOL_STICKYADDR) &&
3624 	    pf_insert_src_node(&nsn, nr, &sk->addr[pd->sidx], pd->af)) {
3625 		REASON_SET(&reason, PFRES_SRCLIMIT);
3626 		goto csfailed;
3627 	}
3628 	s = uma_zalloc(V_pf_state_z, M_NOWAIT | M_ZERO);
3629 	if (s == NULL) {
3630 		REASON_SET(&reason, PFRES_MEMORY);
3631 		goto csfailed;
3632 	}
3633 	s->rule.ptr = r;
3634 	s->nat_rule.ptr = nr;
3635 	s->anchor.ptr = a;
3636 	STATE_INC_COUNTERS(s);
3637 	if (r->allow_opts)
3638 		s->state_flags |= PFSTATE_ALLOWOPTS;
3639 	if (r->rule_flag & PFRULE_STATESLOPPY)
3640 		s->state_flags |= PFSTATE_SLOPPY;
3641 	s->log = r->log & PF_LOG_ALL;
3642 	s->sync_state = PFSYNC_S_NONE;
3643 	if (nr != NULL)
3644 		s->log |= nr->log & PF_LOG_ALL;
3645 	switch (pd->proto) {
3646 	case IPPROTO_TCP:
3647 		s->src.seqlo = ntohl(th->th_seq);
3648 		s->src.seqhi = s->src.seqlo + pd->p_len + 1;
3649 		if ((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN &&
3650 		    r->keep_state == PF_STATE_MODULATE) {
3651 			/* Generate sequence number modulator */
3652 			if ((s->src.seqdiff = pf_tcp_iss(pd) - s->src.seqlo) ==
3653 			    0)
3654 				s->src.seqdiff = 1;
3655 			pf_change_a(&th->th_seq, &th->th_sum,
3656 			    htonl(s->src.seqlo + s->src.seqdiff), 0);
3657 			*rewrite = 1;
3658 		} else
3659 			s->src.seqdiff = 0;
3660 		if (th->th_flags & TH_SYN) {
3661 			s->src.seqhi++;
3662 			s->src.wscale = pf_get_wscale(m, off,
3663 			    th->th_off, pd->af);
3664 		}
3665 		s->src.max_win = MAX(ntohs(th->th_win), 1);
3666 		if (s->src.wscale & PF_WSCALE_MASK) {
3667 			/* Remove scale factor from initial window */
3668 			int win = s->src.max_win;
3669 			win += 1 << (s->src.wscale & PF_WSCALE_MASK);
3670 			s->src.max_win = (win - 1) >>
3671 			    (s->src.wscale & PF_WSCALE_MASK);
3672 		}
3673 		if (th->th_flags & TH_FIN)
3674 			s->src.seqhi++;
3675 		s->dst.seqhi = 1;
3676 		s->dst.max_win = 1;
3677 		s->src.state = TCPS_SYN_SENT;
3678 		s->dst.state = TCPS_CLOSED;
3679 		s->timeout = PFTM_TCP_FIRST_PACKET;
3680 		break;
3681 	case IPPROTO_UDP:
3682 		s->src.state = PFUDPS_SINGLE;
3683 		s->dst.state = PFUDPS_NO_TRAFFIC;
3684 		s->timeout = PFTM_UDP_FIRST_PACKET;
3685 		break;
3686 	case IPPROTO_ICMP:
3687 #ifdef INET6
3688 	case IPPROTO_ICMPV6:
3689 #endif
3690 		s->timeout = PFTM_ICMP_FIRST_PACKET;
3691 		break;
3692 	default:
3693 		s->src.state = PFOTHERS_SINGLE;
3694 		s->dst.state = PFOTHERS_NO_TRAFFIC;
3695 		s->timeout = PFTM_OTHER_FIRST_PACKET;
3696 	}
3697 
3698 	s->creation = time_uptime;
3699 	s->expire = time_uptime;
3700 
3701 	if (sn != NULL) {
3702 		s->src_node = sn;
3703 		s->src_node->states++;
3704 	}
3705 	if (nsn != NULL) {
3706 		/* XXX We only modify one side for now. */
3707 		PF_ACPY(&nsn->raddr, &nk->addr[1], pd->af);
3708 		s->nat_src_node = nsn;
3709 		s->nat_src_node->states++;
3710 	}
3711 	if (pd->proto == IPPROTO_TCP) {
3712 		if ((pd->flags & PFDESC_TCP_NORM) && pf_normalize_tcp_init(m,
3713 		    off, pd, th, &s->src, &s->dst)) {
3714 			REASON_SET(&reason, PFRES_MEMORY);
3715 			pf_src_tree_remove_state(s);
3716 			STATE_DEC_COUNTERS(s);
3717 			uma_zfree(V_pf_state_z, s);
3718 			return (PF_DROP);
3719 		}
3720 		if ((pd->flags & PFDESC_TCP_NORM) && s->src.scrub &&
3721 		    pf_normalize_tcp_stateful(m, off, pd, &reason, th, s,
3722 		    &s->src, &s->dst, rewrite)) {
3723 			/* This really shouldn't happen!!! */
3724 			DPFPRINTF(PF_DEBUG_URGENT,
3725 			    ("pf_normalize_tcp_stateful failed on first pkt"));
3726 			pf_normalize_tcp_cleanup(s);
3727 			pf_src_tree_remove_state(s);
3728 			STATE_DEC_COUNTERS(s);
3729 			uma_zfree(V_pf_state_z, s);
3730 			return (PF_DROP);
3731 		}
3732 	}
3733 	s->direction = pd->dir;
3734 
3735 	/*
3736 	 * sk/nk could already been setup by pf_get_translation().
3737 	 */
3738 	if (nr == NULL) {
3739 		KASSERT((sk == NULL && nk == NULL), ("%s: nr %p sk %p, nk %p",
3740 		    __func__, nr, sk, nk));
3741 		sk = pf_state_key_setup(pd, pd->src, pd->dst, sport, dport);
3742 		if (sk == NULL)
3743 			goto csfailed;
3744 		nk = sk;
3745 	} else
3746 		KASSERT((sk != NULL && nk != NULL), ("%s: nr %p sk %p, nk %p",
3747 		    __func__, nr, sk, nk));
3748 
3749 	/* Swap sk/nk for PF_OUT. */
3750 	if (pf_state_insert(BOUND_IFACE(r, kif),
3751 	    (pd->dir == PF_IN) ? sk : nk,
3752 	    (pd->dir == PF_IN) ? nk : sk, s)) {
3753 		if (pd->proto == IPPROTO_TCP)
3754 			pf_normalize_tcp_cleanup(s);
3755 		REASON_SET(&reason, PFRES_STATEINS);
3756 		pf_src_tree_remove_state(s);
3757 		STATE_DEC_COUNTERS(s);
3758 		uma_zfree(V_pf_state_z, s);
3759 		return (PF_DROP);
3760 	} else
3761 		*sm = s;
3762 
3763 	pf_set_rt_ifp(s, pd->src);	/* needs s->state_key set */
3764 	if (tag > 0)
3765 		s->tag = tag;
3766 	if (pd->proto == IPPROTO_TCP && (th->th_flags & (TH_SYN|TH_ACK)) ==
3767 	    TH_SYN && r->keep_state == PF_STATE_SYNPROXY) {
3768 		s->src.state = PF_TCPS_PROXY_SRC;
3769 		/* undo NAT changes, if they have taken place */
3770 		if (nr != NULL) {
3771 			struct pf_state_key *skt = s->key[PF_SK_WIRE];
3772 			if (pd->dir == PF_OUT)
3773 				skt = s->key[PF_SK_STACK];
3774 			PF_ACPY(pd->src, &skt->addr[pd->sidx], pd->af);
3775 			PF_ACPY(pd->dst, &skt->addr[pd->didx], pd->af);
3776 			if (pd->sport)
3777 				*pd->sport = skt->port[pd->sidx];
3778 			if (pd->dport)
3779 				*pd->dport = skt->port[pd->didx];
3780 			if (pd->proto_sum)
3781 				*pd->proto_sum = bproto_sum;
3782 			if (pd->ip_sum)
3783 				*pd->ip_sum = bip_sum;
3784 			m_copyback(m, off, hdrlen, pd->hdr.any);
3785 		}
3786 		s->src.seqhi = htonl(arc4random());
3787 		/* Find mss option */
3788 		int rtid = M_GETFIB(m);
3789 		mss = pf_get_mss(m, off, th->th_off, pd->af);
3790 		mss = pf_calc_mss(pd->src, pd->af, rtid, mss);
3791 		mss = pf_calc_mss(pd->dst, pd->af, rtid, mss);
3792 		s->src.mss = mss;
3793 		pf_send_tcp(NULL, r, pd->af, pd->dst, pd->src, th->th_dport,
3794 		    th->th_sport, s->src.seqhi, ntohl(th->th_seq) + 1,
3795 		    TH_SYN|TH_ACK, 0, s->src.mss, 0, 1, 0, NULL);
3796 		REASON_SET(&reason, PFRES_SYNPROXY);
3797 		return (PF_SYNPROXY_DROP);
3798 	}
3799 
3800 	return (PF_PASS);
3801 
3802 csfailed:
3803 	if (sk != NULL)
3804 		uma_zfree(V_pf_state_key_z, sk);
3805 	if (nk != NULL)
3806 		uma_zfree(V_pf_state_key_z, nk);
3807 
3808 	if (sn != NULL && sn->states == 0 && sn->expire == 0) {
3809 		pf_unlink_src_node(sn);
3810 		pf_free_src_node(sn);
3811 	}
3812 
3813 	if (nsn != sn && nsn != NULL && nsn->states == 0 && nsn->expire == 0) {
3814 		pf_unlink_src_node(nsn);
3815 		pf_free_src_node(nsn);
3816 	}
3817 
3818 	return (PF_DROP);
3819 }
3820 
3821 static int
3822 pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif,
3823     struct mbuf *m, void *h, struct pf_pdesc *pd, struct pf_rule **am,
3824     struct pf_ruleset **rsm)
3825 {
3826 	struct pf_rule		*r, *a = NULL;
3827 	struct pf_ruleset	*ruleset = NULL;
3828 	sa_family_t		 af = pd->af;
3829 	u_short			 reason;
3830 	int			 tag = -1;
3831 	int			 asd = 0;
3832 	int			 match = 0;
3833 	struct pf_anchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
3834 
3835 	PF_RULES_RASSERT();
3836 
3837 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
3838 	while (r != NULL) {
3839 		r->evaluations++;
3840 		if (pfi_kif_match(r->kif, kif) == r->ifnot)
3841 			r = r->skip[PF_SKIP_IFP].ptr;
3842 		else if (r->direction && r->direction != direction)
3843 			r = r->skip[PF_SKIP_DIR].ptr;
3844 		else if (r->af && r->af != af)
3845 			r = r->skip[PF_SKIP_AF].ptr;
3846 		else if (r->proto && r->proto != pd->proto)
3847 			r = r->skip[PF_SKIP_PROTO].ptr;
3848 		else if (PF_MISMATCHAW(&r->src.addr, pd->src, af,
3849 		    r->src.neg, kif, M_GETFIB(m)))
3850 			r = r->skip[PF_SKIP_SRC_ADDR].ptr;
3851 		else if (PF_MISMATCHAW(&r->dst.addr, pd->dst, af,
3852 		    r->dst.neg, NULL, M_GETFIB(m)))
3853 			r = r->skip[PF_SKIP_DST_ADDR].ptr;
3854 		else if (r->tos && !(r->tos == pd->tos))
3855 			r = TAILQ_NEXT(r, entries);
3856 		else if (r->os_fingerprint != PF_OSFP_ANY)
3857 			r = TAILQ_NEXT(r, entries);
3858 		else if (pd->proto == IPPROTO_UDP &&
3859 		    (r->src.port_op || r->dst.port_op))
3860 			r = TAILQ_NEXT(r, entries);
3861 		else if (pd->proto == IPPROTO_TCP &&
3862 		    (r->src.port_op || r->dst.port_op || r->flagset))
3863 			r = TAILQ_NEXT(r, entries);
3864 		else if ((pd->proto == IPPROTO_ICMP ||
3865 		    pd->proto == IPPROTO_ICMPV6) &&
3866 		    (r->type || r->code))
3867 			r = TAILQ_NEXT(r, entries);
3868 		else if (r->prob && r->prob <=
3869 		    (arc4random() % (UINT_MAX - 1) + 1))
3870 			r = TAILQ_NEXT(r, entries);
3871 		else if (r->match_tag && !pf_match_tag(m, r, &tag,
3872 		    pd->pf_mtag ? pd->pf_mtag->tag : 0))
3873 			r = TAILQ_NEXT(r, entries);
3874 		else {
3875 			if (r->anchor == NULL) {
3876 				match = 1;
3877 				*rm = r;
3878 				*am = a;
3879 				*rsm = ruleset;
3880 				if ((*rm)->quick)
3881 					break;
3882 				r = TAILQ_NEXT(r, entries);
3883 			} else
3884 				pf_step_into_anchor(anchor_stack, &asd,
3885 				    &ruleset, PF_RULESET_FILTER, &r, &a,
3886 				    &match);
3887 		}
3888 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
3889 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
3890 			break;
3891 	}
3892 	r = *rm;
3893 	a = *am;
3894 	ruleset = *rsm;
3895 
3896 	REASON_SET(&reason, PFRES_MATCH);
3897 
3898 	if (r->log)
3899 		PFLOG_PACKET(kif, m, af, direction, reason, r, a, ruleset, pd,
3900 		    1);
3901 
3902 	if (r->action != PF_PASS)
3903 		return (PF_DROP);
3904 
3905 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
3906 		REASON_SET(&reason, PFRES_MEMORY);
3907 		return (PF_DROP);
3908 	}
3909 
3910 	return (PF_PASS);
3911 }
3912 
3913 static int
3914 pf_tcp_track_full(struct pf_state_peer *src, struct pf_state_peer *dst,
3915 	struct pf_state **state, struct pfi_kif *kif, struct mbuf *m, int off,
3916 	struct pf_pdesc *pd, u_short *reason, int *copyback)
3917 {
3918 	struct tcphdr		*th = pd->hdr.tcp;
3919 	u_int16_t		 win = ntohs(th->th_win);
3920 	u_int32_t		 ack, end, seq, orig_seq;
3921 	u_int8_t		 sws, dws;
3922 	int			 ackskew;
3923 
3924 	if (src->wscale && dst->wscale && !(th->th_flags & TH_SYN)) {
3925 		sws = src->wscale & PF_WSCALE_MASK;
3926 		dws = dst->wscale & PF_WSCALE_MASK;
3927 	} else
3928 		sws = dws = 0;
3929 
3930 	/*
3931 	 * Sequence tracking algorithm from Guido van Rooij's paper:
3932 	 *   http://www.madison-gurkha.com/publications/tcp_filtering/
3933 	 *	tcp_filtering.ps
3934 	 */
3935 
3936 	orig_seq = seq = ntohl(th->th_seq);
3937 	if (src->seqlo == 0) {
3938 		/* First packet from this end. Set its state */
3939 
3940 		if ((pd->flags & PFDESC_TCP_NORM || dst->scrub) &&
3941 		    src->scrub == NULL) {
3942 			if (pf_normalize_tcp_init(m, off, pd, th, src, dst)) {
3943 				REASON_SET(reason, PFRES_MEMORY);
3944 				return (PF_DROP);
3945 			}
3946 		}
3947 
3948 		/* Deferred generation of sequence number modulator */
3949 		if (dst->seqdiff && !src->seqdiff) {
3950 			/* use random iss for the TCP server */
3951 			while ((src->seqdiff = arc4random() - seq) == 0)
3952 				;
3953 			ack = ntohl(th->th_ack) - dst->seqdiff;
3954 			pf_change_a(&th->th_seq, &th->th_sum, htonl(seq +
3955 			    src->seqdiff), 0);
3956 			pf_change_a(&th->th_ack, &th->th_sum, htonl(ack), 0);
3957 			*copyback = 1;
3958 		} else {
3959 			ack = ntohl(th->th_ack);
3960 		}
3961 
3962 		end = seq + pd->p_len;
3963 		if (th->th_flags & TH_SYN) {
3964 			end++;
3965 			if (dst->wscale & PF_WSCALE_FLAG) {
3966 				src->wscale = pf_get_wscale(m, off, th->th_off,
3967 				    pd->af);
3968 				if (src->wscale & PF_WSCALE_FLAG) {
3969 					/* Remove scale factor from initial
3970 					 * window */
3971 					sws = src->wscale & PF_WSCALE_MASK;
3972 					win = ((u_int32_t)win + (1 << sws) - 1)
3973 					    >> sws;
3974 					dws = dst->wscale & PF_WSCALE_MASK;
3975 				} else {
3976 					/* fixup other window */
3977 					dst->max_win <<= dst->wscale &
3978 					    PF_WSCALE_MASK;
3979 					/* in case of a retrans SYN|ACK */
3980 					dst->wscale = 0;
3981 				}
3982 			}
3983 		}
3984 		if (th->th_flags & TH_FIN)
3985 			end++;
3986 
3987 		src->seqlo = seq;
3988 		if (src->state < TCPS_SYN_SENT)
3989 			src->state = TCPS_SYN_SENT;
3990 
3991 		/*
3992 		 * May need to slide the window (seqhi may have been set by
3993 		 * the crappy stack check or if we picked up the connection
3994 		 * after establishment)
3995 		 */
3996 		if (src->seqhi == 1 ||
3997 		    SEQ_GEQ(end + MAX(1, dst->max_win << dws), src->seqhi))
3998 			src->seqhi = end + MAX(1, dst->max_win << dws);
3999 		if (win > src->max_win)
4000 			src->max_win = win;
4001 
4002 	} else {
4003 		ack = ntohl(th->th_ack) - dst->seqdiff;
4004 		if (src->seqdiff) {
4005 			/* Modulate sequence numbers */
4006 			pf_change_a(&th->th_seq, &th->th_sum, htonl(seq +
4007 			    src->seqdiff), 0);
4008 			pf_change_a(&th->th_ack, &th->th_sum, htonl(ack), 0);
4009 			*copyback = 1;
4010 		}
4011 		end = seq + pd->p_len;
4012 		if (th->th_flags & TH_SYN)
4013 			end++;
4014 		if (th->th_flags & TH_FIN)
4015 			end++;
4016 	}
4017 
4018 	if ((th->th_flags & TH_ACK) == 0) {
4019 		/* Let it pass through the ack skew check */
4020 		ack = dst->seqlo;
4021 	} else if ((ack == 0 &&
4022 	    (th->th_flags & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) ||
4023 	    /* broken tcp stacks do not set ack */
4024 	    (dst->state < TCPS_SYN_SENT)) {
4025 		/*
4026 		 * Many stacks (ours included) will set the ACK number in an
4027 		 * FIN|ACK if the SYN times out -- no sequence to ACK.
4028 		 */
4029 		ack = dst->seqlo;
4030 	}
4031 
4032 	if (seq == end) {
4033 		/* Ease sequencing restrictions on no data packets */
4034 		seq = src->seqlo;
4035 		end = seq;
4036 	}
4037 
4038 	ackskew = dst->seqlo - ack;
4039 
4040 
4041 	/*
4042 	 * Need to demodulate the sequence numbers in any TCP SACK options
4043 	 * (Selective ACK). We could optionally validate the SACK values
4044 	 * against the current ACK window, either forwards or backwards, but
4045 	 * I'm not confident that SACK has been implemented properly
4046 	 * everywhere. It wouldn't surprise me if several stacks accidently
4047 	 * SACK too far backwards of previously ACKed data. There really aren't
4048 	 * any security implications of bad SACKing unless the target stack
4049 	 * doesn't validate the option length correctly. Someone trying to
4050 	 * spoof into a TCP connection won't bother blindly sending SACK
4051 	 * options anyway.
4052 	 */
4053 	if (dst->seqdiff && (th->th_off << 2) > sizeof(struct tcphdr)) {
4054 		if (pf_modulate_sack(m, off, pd, th, dst))
4055 			*copyback = 1;
4056 	}
4057 
4058 
4059 #define	MAXACKWINDOW (0xffff + 1500)	/* 1500 is an arbitrary fudge factor */
4060 	if (SEQ_GEQ(src->seqhi, end) &&
4061 	    /* Last octet inside other's window space */
4062 	    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) &&
4063 	    /* Retrans: not more than one window back */
4064 	    (ackskew >= -MAXACKWINDOW) &&
4065 	    /* Acking not more than one reassembled fragment backwards */
4066 	    (ackskew <= (MAXACKWINDOW << sws)) &&
4067 	    /* Acking not more than one window forward */
4068 	    ((th->th_flags & TH_RST) == 0 || orig_seq == src->seqlo ||
4069 	    (orig_seq == src->seqlo + 1) || (orig_seq + 1 == src->seqlo) ||
4070 	    (pd->flags & PFDESC_IP_REAS) == 0)) {
4071 	    /* Require an exact/+1 sequence match on resets when possible */
4072 
4073 		if (dst->scrub || src->scrub) {
4074 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
4075 			    *state, src, dst, copyback))
4076 				return (PF_DROP);
4077 		}
4078 
4079 		/* update max window */
4080 		if (src->max_win < win)
4081 			src->max_win = win;
4082 		/* synchronize sequencing */
4083 		if (SEQ_GT(end, src->seqlo))
4084 			src->seqlo = end;
4085 		/* slide the window of what the other end can send */
4086 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
4087 			dst->seqhi = ack + MAX((win << sws), 1);
4088 
4089 
4090 		/* update states */
4091 		if (th->th_flags & TH_SYN)
4092 			if (src->state < TCPS_SYN_SENT)
4093 				src->state = TCPS_SYN_SENT;
4094 		if (th->th_flags & TH_FIN)
4095 			if (src->state < TCPS_CLOSING)
4096 				src->state = TCPS_CLOSING;
4097 		if (th->th_flags & TH_ACK) {
4098 			if (dst->state == TCPS_SYN_SENT) {
4099 				dst->state = TCPS_ESTABLISHED;
4100 				if (src->state == TCPS_ESTABLISHED &&
4101 				    (*state)->src_node != NULL &&
4102 				    pf_src_connlimit(state)) {
4103 					REASON_SET(reason, PFRES_SRCLIMIT);
4104 					return (PF_DROP);
4105 				}
4106 			} else if (dst->state == TCPS_CLOSING)
4107 				dst->state = TCPS_FIN_WAIT_2;
4108 		}
4109 		if (th->th_flags & TH_RST)
4110 			src->state = dst->state = TCPS_TIME_WAIT;
4111 
4112 		/* update expire time */
4113 		(*state)->expire = time_uptime;
4114 		if (src->state >= TCPS_FIN_WAIT_2 &&
4115 		    dst->state >= TCPS_FIN_WAIT_2)
4116 			(*state)->timeout = PFTM_TCP_CLOSED;
4117 		else if (src->state >= TCPS_CLOSING &&
4118 		    dst->state >= TCPS_CLOSING)
4119 			(*state)->timeout = PFTM_TCP_FIN_WAIT;
4120 		else if (src->state < TCPS_ESTABLISHED ||
4121 		    dst->state < TCPS_ESTABLISHED)
4122 			(*state)->timeout = PFTM_TCP_OPENING;
4123 		else if (src->state >= TCPS_CLOSING ||
4124 		    dst->state >= TCPS_CLOSING)
4125 			(*state)->timeout = PFTM_TCP_CLOSING;
4126 		else
4127 			(*state)->timeout = PFTM_TCP_ESTABLISHED;
4128 
4129 		/* Fall through to PASS packet */
4130 
4131 	} else if ((dst->state < TCPS_SYN_SENT ||
4132 		dst->state >= TCPS_FIN_WAIT_2 ||
4133 		src->state >= TCPS_FIN_WAIT_2) &&
4134 	    SEQ_GEQ(src->seqhi + MAXACKWINDOW, end) &&
4135 	    /* Within a window forward of the originating packet */
4136 	    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW)) {
4137 	    /* Within a window backward of the originating packet */
4138 
4139 		/*
4140 		 * This currently handles three situations:
4141 		 *  1) Stupid stacks will shotgun SYNs before their peer
4142 		 *     replies.
4143 		 *  2) When PF catches an already established stream (the
4144 		 *     firewall rebooted, the state table was flushed, routes
4145 		 *     changed...)
4146 		 *  3) Packets get funky immediately after the connection
4147 		 *     closes (this should catch Solaris spurious ACK|FINs
4148 		 *     that web servers like to spew after a close)
4149 		 *
4150 		 * This must be a little more careful than the above code
4151 		 * since packet floods will also be caught here. We don't
4152 		 * update the TTL here to mitigate the damage of a packet
4153 		 * flood and so the same code can handle awkward establishment
4154 		 * and a loosened connection close.
4155 		 * In the establishment case, a correct peer response will
4156 		 * validate the connection, go through the normal state code
4157 		 * and keep updating the state TTL.
4158 		 */
4159 
4160 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
4161 			printf("pf: loose state match: ");
4162 			pf_print_state(*state);
4163 			pf_print_flags(th->th_flags);
4164 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
4165 			    "pkts=%llu:%llu dir=%s,%s\n", seq, orig_seq, ack,
4166 			    pd->p_len, ackskew, (unsigned long long)(*state)->packets[0],
4167 			    (unsigned long long)(*state)->packets[1],
4168 			    pd->dir == PF_IN ? "in" : "out",
4169 			    pd->dir == (*state)->direction ? "fwd" : "rev");
4170 		}
4171 
4172 		if (dst->scrub || src->scrub) {
4173 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
4174 			    *state, src, dst, copyback))
4175 				return (PF_DROP);
4176 		}
4177 
4178 		/* update max window */
4179 		if (src->max_win < win)
4180 			src->max_win = win;
4181 		/* synchronize sequencing */
4182 		if (SEQ_GT(end, src->seqlo))
4183 			src->seqlo = end;
4184 		/* slide the window of what the other end can send */
4185 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
4186 			dst->seqhi = ack + MAX((win << sws), 1);
4187 
4188 		/*
4189 		 * Cannot set dst->seqhi here since this could be a shotgunned
4190 		 * SYN and not an already established connection.
4191 		 */
4192 
4193 		if (th->th_flags & TH_FIN)
4194 			if (src->state < TCPS_CLOSING)
4195 				src->state = TCPS_CLOSING;
4196 		if (th->th_flags & TH_RST)
4197 			src->state = dst->state = TCPS_TIME_WAIT;
4198 
4199 		/* Fall through to PASS packet */
4200 
4201 	} else {
4202 		if ((*state)->dst.state == TCPS_SYN_SENT &&
4203 		    (*state)->src.state == TCPS_SYN_SENT) {
4204 			/* Send RST for state mismatches during handshake */
4205 			if (!(th->th_flags & TH_RST))
4206 				pf_send_tcp(NULL, (*state)->rule.ptr, pd->af,
4207 				    pd->dst, pd->src, th->th_dport,
4208 				    th->th_sport, ntohl(th->th_ack), 0,
4209 				    TH_RST, 0, 0,
4210 				    (*state)->rule.ptr->return_ttl, 1, 0,
4211 				    kif->pfik_ifp);
4212 			src->seqlo = 0;
4213 			src->seqhi = 1;
4214 			src->max_win = 1;
4215 		} else if (V_pf_status.debug >= PF_DEBUG_MISC) {
4216 			printf("pf: BAD state: ");
4217 			pf_print_state(*state);
4218 			pf_print_flags(th->th_flags);
4219 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
4220 			    "pkts=%llu:%llu dir=%s,%s\n",
4221 			    seq, orig_seq, ack, pd->p_len, ackskew,
4222 			    (unsigned long long)(*state)->packets[0],
4223 			    (unsigned long long)(*state)->packets[1],
4224 			    pd->dir == PF_IN ? "in" : "out",
4225 			    pd->dir == (*state)->direction ? "fwd" : "rev");
4226 			printf("pf: State failure on: %c %c %c %c | %c %c\n",
4227 			    SEQ_GEQ(src->seqhi, end) ? ' ' : '1',
4228 			    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) ?
4229 			    ' ': '2',
4230 			    (ackskew >= -MAXACKWINDOW) ? ' ' : '3',
4231 			    (ackskew <= (MAXACKWINDOW << sws)) ? ' ' : '4',
4232 			    SEQ_GEQ(src->seqhi + MAXACKWINDOW, end) ?' ' :'5',
4233 			    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW) ?' ' :'6');
4234 		}
4235 		REASON_SET(reason, PFRES_BADSTATE);
4236 		return (PF_DROP);
4237 	}
4238 
4239 	return (PF_PASS);
4240 }
4241 
4242 static int
4243 pf_tcp_track_sloppy(struct pf_state_peer *src, struct pf_state_peer *dst,
4244 	struct pf_state **state, struct pf_pdesc *pd, u_short *reason)
4245 {
4246 	struct tcphdr		*th = pd->hdr.tcp;
4247 
4248 	if (th->th_flags & TH_SYN)
4249 		if (src->state < TCPS_SYN_SENT)
4250 			src->state = TCPS_SYN_SENT;
4251 	if (th->th_flags & TH_FIN)
4252 		if (src->state < TCPS_CLOSING)
4253 			src->state = TCPS_CLOSING;
4254 	if (th->th_flags & TH_ACK) {
4255 		if (dst->state == TCPS_SYN_SENT) {
4256 			dst->state = TCPS_ESTABLISHED;
4257 			if (src->state == TCPS_ESTABLISHED &&
4258 			    (*state)->src_node != NULL &&
4259 			    pf_src_connlimit(state)) {
4260 				REASON_SET(reason, PFRES_SRCLIMIT);
4261 				return (PF_DROP);
4262 			}
4263 		} else if (dst->state == TCPS_CLOSING) {
4264 			dst->state = TCPS_FIN_WAIT_2;
4265 		} else if (src->state == TCPS_SYN_SENT &&
4266 		    dst->state < TCPS_SYN_SENT) {
4267 			/*
4268 			 * Handle a special sloppy case where we only see one
4269 			 * half of the connection. If there is a ACK after
4270 			 * the initial SYN without ever seeing a packet from
4271 			 * the destination, set the connection to established.
4272 			 */
4273 			dst->state = src->state = TCPS_ESTABLISHED;
4274 			if ((*state)->src_node != NULL &&
4275 			    pf_src_connlimit(state)) {
4276 				REASON_SET(reason, PFRES_SRCLIMIT);
4277 				return (PF_DROP);
4278 			}
4279 		} else if (src->state == TCPS_CLOSING &&
4280 		    dst->state == TCPS_ESTABLISHED &&
4281 		    dst->seqlo == 0) {
4282 			/*
4283 			 * Handle the closing of half connections where we
4284 			 * don't see the full bidirectional FIN/ACK+ACK
4285 			 * handshake.
4286 			 */
4287 			dst->state = TCPS_CLOSING;
4288 		}
4289 	}
4290 	if (th->th_flags & TH_RST)
4291 		src->state = dst->state = TCPS_TIME_WAIT;
4292 
4293 	/* update expire time */
4294 	(*state)->expire = time_uptime;
4295 	if (src->state >= TCPS_FIN_WAIT_2 &&
4296 	    dst->state >= TCPS_FIN_WAIT_2)
4297 		(*state)->timeout = PFTM_TCP_CLOSED;
4298 	else if (src->state >= TCPS_CLOSING &&
4299 	    dst->state >= TCPS_CLOSING)
4300 		(*state)->timeout = PFTM_TCP_FIN_WAIT;
4301 	else if (src->state < TCPS_ESTABLISHED ||
4302 	    dst->state < TCPS_ESTABLISHED)
4303 		(*state)->timeout = PFTM_TCP_OPENING;
4304 	else if (src->state >= TCPS_CLOSING ||
4305 	    dst->state >= TCPS_CLOSING)
4306 		(*state)->timeout = PFTM_TCP_CLOSING;
4307 	else
4308 		(*state)->timeout = PFTM_TCP_ESTABLISHED;
4309 
4310 	return (PF_PASS);
4311 }
4312 
4313 static int
4314 pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif,
4315     struct mbuf *m, int off, void *h, struct pf_pdesc *pd,
4316     u_short *reason)
4317 {
4318 	struct pf_state_key_cmp	 key;
4319 	struct tcphdr		*th = pd->hdr.tcp;
4320 	int			 copyback = 0;
4321 	struct pf_state_peer	*src, *dst;
4322 	struct pf_state_key	*sk;
4323 
4324 	bzero(&key, sizeof(key));
4325 	key.af = pd->af;
4326 	key.proto = IPPROTO_TCP;
4327 	if (direction == PF_IN)	{	/* wire side, straight */
4328 		PF_ACPY(&key.addr[0], pd->src, key.af);
4329 		PF_ACPY(&key.addr[1], pd->dst, key.af);
4330 		key.port[0] = th->th_sport;
4331 		key.port[1] = th->th_dport;
4332 	} else {			/* stack side, reverse */
4333 		PF_ACPY(&key.addr[1], pd->src, key.af);
4334 		PF_ACPY(&key.addr[0], pd->dst, key.af);
4335 		key.port[1] = th->th_sport;
4336 		key.port[0] = th->th_dport;
4337 	}
4338 
4339 	STATE_LOOKUP(kif, &key, direction, *state, pd);
4340 
4341 	if (direction == (*state)->direction) {
4342 		src = &(*state)->src;
4343 		dst = &(*state)->dst;
4344 	} else {
4345 		src = &(*state)->dst;
4346 		dst = &(*state)->src;
4347 	}
4348 
4349 	sk = (*state)->key[pd->didx];
4350 
4351 	if ((*state)->src.state == PF_TCPS_PROXY_SRC) {
4352 		if (direction != (*state)->direction) {
4353 			REASON_SET(reason, PFRES_SYNPROXY);
4354 			return (PF_SYNPROXY_DROP);
4355 		}
4356 		if (th->th_flags & TH_SYN) {
4357 			if (ntohl(th->th_seq) != (*state)->src.seqlo) {
4358 				REASON_SET(reason, PFRES_SYNPROXY);
4359 				return (PF_DROP);
4360 			}
4361 			pf_send_tcp(NULL, (*state)->rule.ptr, pd->af, pd->dst,
4362 			    pd->src, th->th_dport, th->th_sport,
4363 			    (*state)->src.seqhi, ntohl(th->th_seq) + 1,
4364 			    TH_SYN|TH_ACK, 0, (*state)->src.mss, 0, 1, 0, NULL);
4365 			REASON_SET(reason, PFRES_SYNPROXY);
4366 			return (PF_SYNPROXY_DROP);
4367 		} else if (!(th->th_flags & TH_ACK) ||
4368 		    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
4369 		    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
4370 			REASON_SET(reason, PFRES_SYNPROXY);
4371 			return (PF_DROP);
4372 		} else if ((*state)->src_node != NULL &&
4373 		    pf_src_connlimit(state)) {
4374 			REASON_SET(reason, PFRES_SRCLIMIT);
4375 			return (PF_DROP);
4376 		} else
4377 			(*state)->src.state = PF_TCPS_PROXY_DST;
4378 	}
4379 	if ((*state)->src.state == PF_TCPS_PROXY_DST) {
4380 		if (direction == (*state)->direction) {
4381 			if (((th->th_flags & (TH_SYN|TH_ACK)) != TH_ACK) ||
4382 			    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
4383 			    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
4384 				REASON_SET(reason, PFRES_SYNPROXY);
4385 				return (PF_DROP);
4386 			}
4387 			(*state)->src.max_win = MAX(ntohs(th->th_win), 1);
4388 			if ((*state)->dst.seqhi == 1)
4389 				(*state)->dst.seqhi = htonl(arc4random());
4390 			pf_send_tcp(NULL, (*state)->rule.ptr, pd->af,
4391 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
4392 			    sk->port[pd->sidx], sk->port[pd->didx],
4393 			    (*state)->dst.seqhi, 0, TH_SYN, 0,
4394 			    (*state)->src.mss, 0, 0, (*state)->tag, NULL);
4395 			REASON_SET(reason, PFRES_SYNPROXY);
4396 			return (PF_SYNPROXY_DROP);
4397 		} else if (((th->th_flags & (TH_SYN|TH_ACK)) !=
4398 		    (TH_SYN|TH_ACK)) ||
4399 		    (ntohl(th->th_ack) != (*state)->dst.seqhi + 1)) {
4400 			REASON_SET(reason, PFRES_SYNPROXY);
4401 			return (PF_DROP);
4402 		} else {
4403 			(*state)->dst.max_win = MAX(ntohs(th->th_win), 1);
4404 			(*state)->dst.seqlo = ntohl(th->th_seq);
4405 			pf_send_tcp(NULL, (*state)->rule.ptr, pd->af, pd->dst,
4406 			    pd->src, th->th_dport, th->th_sport,
4407 			    ntohl(th->th_ack), ntohl(th->th_seq) + 1,
4408 			    TH_ACK, (*state)->src.max_win, 0, 0, 0,
4409 			    (*state)->tag, NULL);
4410 			pf_send_tcp(NULL, (*state)->rule.ptr, pd->af,
4411 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
4412 			    sk->port[pd->sidx], sk->port[pd->didx],
4413 			    (*state)->src.seqhi + 1, (*state)->src.seqlo + 1,
4414 			    TH_ACK, (*state)->dst.max_win, 0, 0, 1, 0, NULL);
4415 			(*state)->src.seqdiff = (*state)->dst.seqhi -
4416 			    (*state)->src.seqlo;
4417 			(*state)->dst.seqdiff = (*state)->src.seqhi -
4418 			    (*state)->dst.seqlo;
4419 			(*state)->src.seqhi = (*state)->src.seqlo +
4420 			    (*state)->dst.max_win;
4421 			(*state)->dst.seqhi = (*state)->dst.seqlo +
4422 			    (*state)->src.max_win;
4423 			(*state)->src.wscale = (*state)->dst.wscale = 0;
4424 			(*state)->src.state = (*state)->dst.state =
4425 			    TCPS_ESTABLISHED;
4426 			REASON_SET(reason, PFRES_SYNPROXY);
4427 			return (PF_SYNPROXY_DROP);
4428 		}
4429 	}
4430 
4431 	if (((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN) &&
4432 	    dst->state >= TCPS_FIN_WAIT_2 &&
4433 	    src->state >= TCPS_FIN_WAIT_2) {
4434 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
4435 			printf("pf: state reuse ");
4436 			pf_print_state(*state);
4437 			pf_print_flags(th->th_flags);
4438 			printf("\n");
4439 		}
4440 		/* XXX make sure it's the same direction ?? */
4441 		(*state)->src.state = (*state)->dst.state = TCPS_CLOSED;
4442 		pf_unlink_state(*state, PF_ENTER_LOCKED);
4443 		*state = NULL;
4444 		return (PF_DROP);
4445 	}
4446 
4447 	if ((*state)->state_flags & PFSTATE_SLOPPY) {
4448 		if (pf_tcp_track_sloppy(src, dst, state, pd, reason) == PF_DROP)
4449 			return (PF_DROP);
4450 	} else {
4451 		if (pf_tcp_track_full(src, dst, state, kif, m, off, pd, reason,
4452 		    &copyback) == PF_DROP)
4453 			return (PF_DROP);
4454 	}
4455 
4456 	/* translate source/destination address, if necessary */
4457 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
4458 		struct pf_state_key *nk = (*state)->key[pd->didx];
4459 
4460 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
4461 		    nk->port[pd->sidx] != th->th_sport)
4462 			pf_change_ap(pd->src, &th->th_sport, pd->ip_sum,
4463 			    &th->th_sum, &nk->addr[pd->sidx],
4464 			    nk->port[pd->sidx], 0, pd->af);
4465 
4466 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
4467 		    nk->port[pd->didx] != th->th_dport)
4468 			pf_change_ap(pd->dst, &th->th_dport, pd->ip_sum,
4469 			    &th->th_sum, &nk->addr[pd->didx],
4470 			    nk->port[pd->didx], 0, pd->af);
4471 		copyback = 1;
4472 	}
4473 
4474 	/* Copyback sequence modulation or stateful scrub changes if needed */
4475 	if (copyback)
4476 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
4477 
4478 	return (PF_PASS);
4479 }
4480 
4481 static int
4482 pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif,
4483     struct mbuf *m, int off, void *h, struct pf_pdesc *pd)
4484 {
4485 	struct pf_state_peer	*src, *dst;
4486 	struct pf_state_key_cmp	 key;
4487 	struct udphdr		*uh = pd->hdr.udp;
4488 
4489 	bzero(&key, sizeof(key));
4490 	key.af = pd->af;
4491 	key.proto = IPPROTO_UDP;
4492 	if (direction == PF_IN)	{	/* wire side, straight */
4493 		PF_ACPY(&key.addr[0], pd->src, key.af);
4494 		PF_ACPY(&key.addr[1], pd->dst, key.af);
4495 		key.port[0] = uh->uh_sport;
4496 		key.port[1] = uh->uh_dport;
4497 	} else {			/* stack side, reverse */
4498 		PF_ACPY(&key.addr[1], pd->src, key.af);
4499 		PF_ACPY(&key.addr[0], pd->dst, key.af);
4500 		key.port[1] = uh->uh_sport;
4501 		key.port[0] = uh->uh_dport;
4502 	}
4503 
4504 	STATE_LOOKUP(kif, &key, direction, *state, pd);
4505 
4506 	if (direction == (*state)->direction) {
4507 		src = &(*state)->src;
4508 		dst = &(*state)->dst;
4509 	} else {
4510 		src = &(*state)->dst;
4511 		dst = &(*state)->src;
4512 	}
4513 
4514 	/* update states */
4515 	if (src->state < PFUDPS_SINGLE)
4516 		src->state = PFUDPS_SINGLE;
4517 	if (dst->state == PFUDPS_SINGLE)
4518 		dst->state = PFUDPS_MULTIPLE;
4519 
4520 	/* update expire time */
4521 	(*state)->expire = time_uptime;
4522 	if (src->state == PFUDPS_MULTIPLE && dst->state == PFUDPS_MULTIPLE)
4523 		(*state)->timeout = PFTM_UDP_MULTIPLE;
4524 	else
4525 		(*state)->timeout = PFTM_UDP_SINGLE;
4526 
4527 	/* translate source/destination address, if necessary */
4528 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
4529 		struct pf_state_key *nk = (*state)->key[pd->didx];
4530 
4531 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
4532 		    nk->port[pd->sidx] != uh->uh_sport)
4533 			pf_change_ap(pd->src, &uh->uh_sport, pd->ip_sum,
4534 			    &uh->uh_sum, &nk->addr[pd->sidx],
4535 			    nk->port[pd->sidx], 1, pd->af);
4536 
4537 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
4538 		    nk->port[pd->didx] != uh->uh_dport)
4539 			pf_change_ap(pd->dst, &uh->uh_dport, pd->ip_sum,
4540 			    &uh->uh_sum, &nk->addr[pd->didx],
4541 			    nk->port[pd->didx], 1, pd->af);
4542 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
4543 	}
4544 
4545 	return (PF_PASS);
4546 }
4547 
4548 static int
4549 pf_icmp_state_lookup(struct pf_state_key_cmp *key, struct pf_pdesc *pd,
4550     struct pf_state **state, struct mbuf *m, int direction, struct pfi_kif *kif,
4551     uint16_t icmpid, uint16_t type, int icmp_dir, int *iidx, int multi)
4552 {
4553 
4554 	key->af = pd->af;
4555 	key->proto = pd->proto;
4556 	if (icmp_dir == PF_IN) {
4557 		*iidx = pd->sidx;
4558 		key->port[pd->sidx] = icmpid;
4559 		key->port[pd->didx] = type;
4560 	} else {
4561 		*iidx = pd->didx;
4562 		key->port[pd->sidx] = type;
4563 		key->port[pd->didx] = icmpid;
4564 	}
4565 #ifdef INET6
4566 	if (pd->af == AF_INET6 && multi != PF_ICMP_MULTI_NONE) {
4567 		switch (multi) {
4568 		case PF_ICMP_MULTI_SOLICITED:
4569 			key->addr[pd->sidx].addr32[0] = IPV6_ADDR_INT32_MLL;
4570 			key->addr[pd->sidx].addr32[1] = 0;
4571 			key->addr[pd->sidx].addr32[2] = IPV6_ADDR_INT32_ONE;
4572 			key->addr[pd->sidx].addr32[3] = pd->src->addr32[3];
4573 			key->addr[pd->sidx].addr8[12] = 0xff;
4574 			break;
4575 		case PF_ICMP_MULTI_LINK:
4576 			key->addr[pd->sidx].addr32[0] = IPV6_ADDR_INT32_MLL;
4577 			key->addr[pd->sidx].addr32[1] = 0;
4578 			key->addr[pd->sidx].addr32[2] = 0;
4579 			key->addr[pd->sidx].addr32[3] = IPV6_ADDR_INT32_ONE;
4580 			break;
4581 		}
4582 	} else
4583 #endif
4584 		PF_ACPY(&key->addr[pd->sidx], pd->src, key->af);
4585 	PF_ACPY(&key->addr[pd->didx], pd->dst, key->af);
4586 
4587 	STATE_LOOKUP(kif, key, direction, *state, pd);
4588 
4589 	/* Is this ICMP message flowing in right direction? */
4590 	if ((*state)->rule.ptr->type &&
4591 	    (((*state)->direction == direction) ?
4592 	    PF_IN : PF_OUT) != icmp_dir) {
4593 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
4594 			printf("pf: icmp type %d in wrong direction (%d): ",
4595 			    icmp_dir, direction);
4596 			pf_print_state(*state);
4597 			printf("\n");
4598 		}
4599 		return (PF_DROP);
4600 	}
4601 	return (-1);
4602 }
4603 
4604 static int
4605 pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif,
4606     struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason)
4607 {
4608 	struct pf_addr  *saddr = pd->src, *daddr = pd->dst;
4609 	u_int16_t	 icmpid = 0, *icmpsum, virtual_id, virtual_type;
4610 	u_int8_t	 icmptype;
4611 	int		 icmp_dir, iidx, ret, multi;
4612 	struct pf_state_key_cmp key;
4613 
4614 	bzero(&key, sizeof(key));
4615 	switch (pd->proto) {
4616 #ifdef INET
4617 	case IPPROTO_ICMP:
4618 		icmptype = pd->hdr.icmp->icmp_type;
4619 		icmpid = pd->hdr.icmp->icmp_id;
4620 		icmpsum = &pd->hdr.icmp->icmp_cksum;
4621 
4622 		break;
4623 #endif /* INET */
4624 #ifdef INET6
4625 	case IPPROTO_ICMPV6:
4626 		icmptype = pd->hdr.icmp6->icmp6_type;
4627 		icmpid = pd->hdr.icmp6->icmp6_id;
4628 		icmpsum = &pd->hdr.icmp6->icmp6_cksum;
4629 
4630 		break;
4631 #endif /* INET6 */
4632 	default:
4633 		panic("%s: proto %d\n", __func__, pd->proto);
4634 	}
4635 
4636 	if (pf_icmp_mapping(pd, icmptype, &icmp_dir, &multi,
4637 	    &virtual_id, &virtual_type) == 0) {
4638 		/*
4639 		 * ICMP query/reply message not related to a TCP/UDP packet.
4640 		 * Search for an ICMP state.
4641 		 */
4642 		ret = pf_icmp_state_lookup(&key, pd, state, m, direction,
4643 		    kif, virtual_id, virtual_type, icmp_dir, &iidx,
4644 		    PF_ICMP_MULTI_NONE);
4645 		if (ret >= 0) {
4646 			if (ret == PF_DROP && pd->af == AF_INET6 &&
4647 			    icmp_dir == PF_OUT) {
4648 				if (*state)
4649 					PF_STATE_UNLOCK(*state);
4650 				ret = pf_icmp_state_lookup(&key, pd, state, m,
4651 				    direction, kif, virtual_id, virtual_type,
4652 				    icmp_dir, &iidx, multi);
4653 				if (ret >= 0)
4654 					return (ret);
4655 			} else
4656 				return (ret);
4657 		}
4658 
4659 		(*state)->expire = time_uptime;
4660 		(*state)->timeout = PFTM_ICMP_ERROR_REPLY;
4661 
4662 		/* translate source/destination address, if necessary */
4663 		if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
4664 			struct pf_state_key *nk = (*state)->key[pd->didx];
4665 
4666 			switch (pd->af) {
4667 #ifdef INET
4668 			case AF_INET:
4669 				if (PF_ANEQ(pd->src,
4670 				    &nk->addr[pd->sidx], AF_INET))
4671 					pf_change_a(&saddr->v4.s_addr,
4672 					    pd->ip_sum,
4673 					    nk->addr[pd->sidx].v4.s_addr, 0);
4674 
4675 				if (PF_ANEQ(pd->dst, &nk->addr[pd->didx],
4676 				    AF_INET))
4677 					pf_change_a(&daddr->v4.s_addr,
4678 					    pd->ip_sum,
4679 					    nk->addr[pd->didx].v4.s_addr, 0);
4680 
4681 				if (nk->port[iidx] !=
4682 				    pd->hdr.icmp->icmp_id) {
4683 					pd->hdr.icmp->icmp_cksum =
4684 					    pf_cksum_fixup(
4685 					    pd->hdr.icmp->icmp_cksum, icmpid,
4686 					    nk->port[iidx], 0);
4687 					pd->hdr.icmp->icmp_id = nk->port[iidx];
4688 				}
4689 
4690 				m_copyback(m, off, ICMP_MINLEN,
4691 				    (caddr_t )pd->hdr.icmp);
4692 				break;
4693 #endif /* INET */
4694 #ifdef INET6
4695 			case AF_INET6:
4696 				if (PF_ANEQ(pd->src,
4697 				    &nk->addr[pd->sidx], AF_INET6))
4698 					pf_change_a6(saddr,
4699 					    &pd->hdr.icmp6->icmp6_cksum,
4700 					    &nk->addr[pd->sidx], 0);
4701 
4702 				if (PF_ANEQ(pd->dst,
4703 				    &nk->addr[pd->didx], AF_INET6))
4704 					pf_change_a6(daddr,
4705 					    &pd->hdr.icmp6->icmp6_cksum,
4706 					    &nk->addr[pd->didx], 0);
4707 
4708 				m_copyback(m, off, sizeof(struct icmp6_hdr),
4709 				    (caddr_t )pd->hdr.icmp6);
4710 				break;
4711 #endif /* INET6 */
4712 			}
4713 		}
4714 		return (PF_PASS);
4715 
4716 	} else {
4717 		/*
4718 		 * ICMP error message in response to a TCP/UDP packet.
4719 		 * Extract the inner TCP/UDP header and search for that state.
4720 		 */
4721 
4722 		struct pf_pdesc	pd2;
4723 		bzero(&pd2, sizeof pd2);
4724 #ifdef INET
4725 		struct ip	h2;
4726 #endif /* INET */
4727 #ifdef INET6
4728 		struct ip6_hdr	h2_6;
4729 		int		terminal = 0;
4730 #endif /* INET6 */
4731 		int		ipoff2 = 0;
4732 		int		off2 = 0;
4733 
4734 		pd2.af = pd->af;
4735 		/* Payload packet is from the opposite direction. */
4736 		pd2.sidx = (direction == PF_IN) ? 1 : 0;
4737 		pd2.didx = (direction == PF_IN) ? 0 : 1;
4738 		switch (pd->af) {
4739 #ifdef INET
4740 		case AF_INET:
4741 			/* offset of h2 in mbuf chain */
4742 			ipoff2 = off + ICMP_MINLEN;
4743 
4744 			if (!pf_pull_hdr(m, ipoff2, &h2, sizeof(h2),
4745 			    NULL, reason, pd2.af)) {
4746 				DPFPRINTF(PF_DEBUG_MISC,
4747 				    ("pf: ICMP error message too short "
4748 				    "(ip)\n"));
4749 				return (PF_DROP);
4750 			}
4751 			/*
4752 			 * ICMP error messages don't refer to non-first
4753 			 * fragments
4754 			 */
4755 			if (h2.ip_off & htons(IP_OFFMASK)) {
4756 				REASON_SET(reason, PFRES_FRAG);
4757 				return (PF_DROP);
4758 			}
4759 
4760 			/* offset of protocol header that follows h2 */
4761 			off2 = ipoff2 + (h2.ip_hl << 2);
4762 
4763 			pd2.proto = h2.ip_p;
4764 			pd2.src = (struct pf_addr *)&h2.ip_src;
4765 			pd2.dst = (struct pf_addr *)&h2.ip_dst;
4766 			pd2.ip_sum = &h2.ip_sum;
4767 			break;
4768 #endif /* INET */
4769 #ifdef INET6
4770 		case AF_INET6:
4771 			ipoff2 = off + sizeof(struct icmp6_hdr);
4772 
4773 			if (!pf_pull_hdr(m, ipoff2, &h2_6, sizeof(h2_6),
4774 			    NULL, reason, pd2.af)) {
4775 				DPFPRINTF(PF_DEBUG_MISC,
4776 				    ("pf: ICMP error message too short "
4777 				    "(ip6)\n"));
4778 				return (PF_DROP);
4779 			}
4780 			pd2.proto = h2_6.ip6_nxt;
4781 			pd2.src = (struct pf_addr *)&h2_6.ip6_src;
4782 			pd2.dst = (struct pf_addr *)&h2_6.ip6_dst;
4783 			pd2.ip_sum = NULL;
4784 			off2 = ipoff2 + sizeof(h2_6);
4785 			do {
4786 				switch (pd2.proto) {
4787 				case IPPROTO_FRAGMENT:
4788 					/*
4789 					 * ICMPv6 error messages for
4790 					 * non-first fragments
4791 					 */
4792 					REASON_SET(reason, PFRES_FRAG);
4793 					return (PF_DROP);
4794 				case IPPROTO_AH:
4795 				case IPPROTO_HOPOPTS:
4796 				case IPPROTO_ROUTING:
4797 				case IPPROTO_DSTOPTS: {
4798 					/* get next header and header length */
4799 					struct ip6_ext opt6;
4800 
4801 					if (!pf_pull_hdr(m, off2, &opt6,
4802 					    sizeof(opt6), NULL, reason,
4803 					    pd2.af)) {
4804 						DPFPRINTF(PF_DEBUG_MISC,
4805 						    ("pf: ICMPv6 short opt\n"));
4806 						return (PF_DROP);
4807 					}
4808 					if (pd2.proto == IPPROTO_AH)
4809 						off2 += (opt6.ip6e_len + 2) * 4;
4810 					else
4811 						off2 += (opt6.ip6e_len + 1) * 8;
4812 					pd2.proto = opt6.ip6e_nxt;
4813 					/* goto the next header */
4814 					break;
4815 				}
4816 				default:
4817 					terminal++;
4818 					break;
4819 				}
4820 			} while (!terminal);
4821 			break;
4822 #endif /* INET6 */
4823 		}
4824 
4825 		switch (pd2.proto) {
4826 		case IPPROTO_TCP: {
4827 			struct tcphdr		 th;
4828 			u_int32_t		 seq;
4829 			struct pf_state_peer	*src, *dst;
4830 			u_int8_t		 dws;
4831 			int			 copyback = 0;
4832 
4833 			/*
4834 			 * Only the first 8 bytes of the TCP header can be
4835 			 * expected. Don't access any TCP header fields after
4836 			 * th_seq, an ackskew test is not possible.
4837 			 */
4838 			if (!pf_pull_hdr(m, off2, &th, 8, NULL, reason,
4839 			    pd2.af)) {
4840 				DPFPRINTF(PF_DEBUG_MISC,
4841 				    ("pf: ICMP error message too short "
4842 				    "(tcp)\n"));
4843 				return (PF_DROP);
4844 			}
4845 
4846 			key.af = pd2.af;
4847 			key.proto = IPPROTO_TCP;
4848 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
4849 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
4850 			key.port[pd2.sidx] = th.th_sport;
4851 			key.port[pd2.didx] = th.th_dport;
4852 
4853 			STATE_LOOKUP(kif, &key, direction, *state, pd);
4854 
4855 			if (direction == (*state)->direction) {
4856 				src = &(*state)->dst;
4857 				dst = &(*state)->src;
4858 			} else {
4859 				src = &(*state)->src;
4860 				dst = &(*state)->dst;
4861 			}
4862 
4863 			if (src->wscale && dst->wscale)
4864 				dws = dst->wscale & PF_WSCALE_MASK;
4865 			else
4866 				dws = 0;
4867 
4868 			/* Demodulate sequence number */
4869 			seq = ntohl(th.th_seq) - src->seqdiff;
4870 			if (src->seqdiff) {
4871 				pf_change_a(&th.th_seq, icmpsum,
4872 				    htonl(seq), 0);
4873 				copyback = 1;
4874 			}
4875 
4876 			if (!((*state)->state_flags & PFSTATE_SLOPPY) &&
4877 			    (!SEQ_GEQ(src->seqhi, seq) ||
4878 			    !SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)))) {
4879 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
4880 					printf("pf: BAD ICMP %d:%d ",
4881 					    icmptype, pd->hdr.icmp->icmp_code);
4882 					pf_print_host(pd->src, 0, pd->af);
4883 					printf(" -> ");
4884 					pf_print_host(pd->dst, 0, pd->af);
4885 					printf(" state: ");
4886 					pf_print_state(*state);
4887 					printf(" seq=%u\n", seq);
4888 				}
4889 				REASON_SET(reason, PFRES_BADSTATE);
4890 				return (PF_DROP);
4891 			} else {
4892 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
4893 					printf("pf: OK ICMP %d:%d ",
4894 					    icmptype, pd->hdr.icmp->icmp_code);
4895 					pf_print_host(pd->src, 0, pd->af);
4896 					printf(" -> ");
4897 					pf_print_host(pd->dst, 0, pd->af);
4898 					printf(" state: ");
4899 					pf_print_state(*state);
4900 					printf(" seq=%u\n", seq);
4901 				}
4902 			}
4903 
4904 			/* translate source/destination address, if necessary */
4905 			if ((*state)->key[PF_SK_WIRE] !=
4906 			    (*state)->key[PF_SK_STACK]) {
4907 				struct pf_state_key *nk =
4908 				    (*state)->key[pd->didx];
4909 
4910 				if (PF_ANEQ(pd2.src,
4911 				    &nk->addr[pd2.sidx], pd2.af) ||
4912 				    nk->port[pd2.sidx] != th.th_sport)
4913 					pf_change_icmp(pd2.src, &th.th_sport,
4914 					    daddr, &nk->addr[pd2.sidx],
4915 					    nk->port[pd2.sidx], NULL,
4916 					    pd2.ip_sum, icmpsum,
4917 					    pd->ip_sum, 0, pd2.af);
4918 
4919 				if (PF_ANEQ(pd2.dst,
4920 				    &nk->addr[pd2.didx], pd2.af) ||
4921 				    nk->port[pd2.didx] != th.th_dport)
4922 					pf_change_icmp(pd2.dst, &th.th_dport,
4923 					    NULL, /* XXX Inbound NAT? */
4924 					    &nk->addr[pd2.didx],
4925 					    nk->port[pd2.didx], NULL,
4926 					    pd2.ip_sum, icmpsum,
4927 					    pd->ip_sum, 0, pd2.af);
4928 				copyback = 1;
4929 			}
4930 
4931 			if (copyback) {
4932 				switch (pd2.af) {
4933 #ifdef INET
4934 				case AF_INET:
4935 					m_copyback(m, off, ICMP_MINLEN,
4936 					    (caddr_t )pd->hdr.icmp);
4937 					m_copyback(m, ipoff2, sizeof(h2),
4938 					    (caddr_t )&h2);
4939 					break;
4940 #endif /* INET */
4941 #ifdef INET6
4942 				case AF_INET6:
4943 					m_copyback(m, off,
4944 					    sizeof(struct icmp6_hdr),
4945 					    (caddr_t )pd->hdr.icmp6);
4946 					m_copyback(m, ipoff2, sizeof(h2_6),
4947 					    (caddr_t )&h2_6);
4948 					break;
4949 #endif /* INET6 */
4950 				}
4951 				m_copyback(m, off2, 8, (caddr_t)&th);
4952 			}
4953 
4954 			return (PF_PASS);
4955 			break;
4956 		}
4957 		case IPPROTO_UDP: {
4958 			struct udphdr		uh;
4959 
4960 			if (!pf_pull_hdr(m, off2, &uh, sizeof(uh),
4961 			    NULL, reason, pd2.af)) {
4962 				DPFPRINTF(PF_DEBUG_MISC,
4963 				    ("pf: ICMP error message too short "
4964 				    "(udp)\n"));
4965 				return (PF_DROP);
4966 			}
4967 
4968 			key.af = pd2.af;
4969 			key.proto = IPPROTO_UDP;
4970 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
4971 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
4972 			key.port[pd2.sidx] = uh.uh_sport;
4973 			key.port[pd2.didx] = uh.uh_dport;
4974 
4975 			STATE_LOOKUP(kif, &key, direction, *state, pd);
4976 
4977 			/* translate source/destination address, if necessary */
4978 			if ((*state)->key[PF_SK_WIRE] !=
4979 			    (*state)->key[PF_SK_STACK]) {
4980 				struct pf_state_key *nk =
4981 				    (*state)->key[pd->didx];
4982 
4983 				if (PF_ANEQ(pd2.src,
4984 				    &nk->addr[pd2.sidx], pd2.af) ||
4985 				    nk->port[pd2.sidx] != uh.uh_sport)
4986 					pf_change_icmp(pd2.src, &uh.uh_sport,
4987 					    daddr, &nk->addr[pd2.sidx],
4988 					    nk->port[pd2.sidx], &uh.uh_sum,
4989 					    pd2.ip_sum, icmpsum,
4990 					    pd->ip_sum, 1, pd2.af);
4991 
4992 				if (PF_ANEQ(pd2.dst,
4993 				    &nk->addr[pd2.didx], pd2.af) ||
4994 				    nk->port[pd2.didx] != uh.uh_dport)
4995 					pf_change_icmp(pd2.dst, &uh.uh_dport,
4996 					    NULL, /* XXX Inbound NAT? */
4997 					    &nk->addr[pd2.didx],
4998 					    nk->port[pd2.didx], &uh.uh_sum,
4999 					    pd2.ip_sum, icmpsum,
5000 					    pd->ip_sum, 1, pd2.af);
5001 
5002 				switch (pd2.af) {
5003 #ifdef INET
5004 				case AF_INET:
5005 					m_copyback(m, off, ICMP_MINLEN,
5006 					    (caddr_t )pd->hdr.icmp);
5007 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
5008 					break;
5009 #endif /* INET */
5010 #ifdef INET6
5011 				case AF_INET6:
5012 					m_copyback(m, off,
5013 					    sizeof(struct icmp6_hdr),
5014 					    (caddr_t )pd->hdr.icmp6);
5015 					m_copyback(m, ipoff2, sizeof(h2_6),
5016 					    (caddr_t )&h2_6);
5017 					break;
5018 #endif /* INET6 */
5019 				}
5020 				m_copyback(m, off2, sizeof(uh), (caddr_t)&uh);
5021 			}
5022 			return (PF_PASS);
5023 			break;
5024 		}
5025 #ifdef INET
5026 		case IPPROTO_ICMP: {
5027 			struct icmp		iih;
5028 
5029 			if (!pf_pull_hdr(m, off2, &iih, ICMP_MINLEN,
5030 			    NULL, reason, pd2.af)) {
5031 				DPFPRINTF(PF_DEBUG_MISC,
5032 				    ("pf: ICMP error message too short i"
5033 				    "(icmp)\n"));
5034 				return (PF_DROP);
5035 			}
5036 
5037 			pd2.hdr.icmp = &iih;
5038 			pf_icmp_mapping(&pd2, iih.icmp_type,
5039 			    &icmp_dir, &multi, &virtual_id, &virtual_type);
5040 
5041 			ret = pf_icmp_state_lookup(&key, &pd2, state, m,
5042 			    direction, kif, virtual_id, virtual_type,
5043 			    icmp_dir, &iidx, PF_ICMP_MULTI_NONE);
5044 			if (ret >= 0)
5045 				return (ret);
5046 
5047 			/* translate source/destination address, if necessary */
5048 			if ((*state)->key[PF_SK_WIRE] !=
5049 			    (*state)->key[PF_SK_STACK]) {
5050 				struct pf_state_key *nk =
5051 				    (*state)->key[pd->didx];
5052 
5053 				if (PF_ANEQ(pd2.src,
5054 				    &nk->addr[pd2.sidx], pd2.af) ||
5055 				    (virtual_type == ICMP_ECHO &&
5056 				    nk->port[iidx] != iih.icmp_id))
5057 					pf_change_icmp(pd2.src,
5058 					    (virtual_type == ICMP_ECHO) ?
5059 					    &iih.icmp_id : NULL,
5060 					    daddr, &nk->addr[pd2.sidx],
5061 					    nk->port[pd2.sidx], NULL,
5062 					    pd2.ip_sum, icmpsum,
5063 					    pd->ip_sum, 0, AF_INET);
5064 
5065 				if (PF_ANEQ(pd2.dst,
5066 				    &nk->addr[pd2.didx], pd2.af))
5067                                        pf_change_icmp(pd2.dst, NULL, NULL,
5068 					    &nk->addr[pd2.didx], 0, NULL,
5069 					    pd2.ip_sum, icmpsum,
5070 					    pd->ip_sum, 0, AF_INET);
5071 
5072 				m_copyback(m, off, ICMP_MINLEN, (caddr_t)pd->hdr.icmp);
5073 				m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
5074 				m_copyback(m, off2, ICMP_MINLEN, (caddr_t)&iih);
5075 			}
5076 			return (PF_PASS);
5077 			break;
5078 		}
5079 #endif /* INET */
5080 #ifdef INET6
5081 		case IPPROTO_ICMPV6: {
5082 			struct icmp6_hdr	iih;
5083 
5084 			if (!pf_pull_hdr(m, off2, &iih,
5085 			    sizeof(struct icmp6_hdr), NULL, reason, pd2.af)) {
5086 				DPFPRINTF(PF_DEBUG_MISC,
5087 				    ("pf: ICMP error message too short "
5088 				    "(icmp6)\n"));
5089 				return (PF_DROP);
5090 			}
5091 
5092 			pd2.hdr.icmp6 = &iih;
5093 			pf_icmp_mapping(&pd2, iih.icmp6_type,
5094 			    &icmp_dir, &multi, &virtual_id, &virtual_type);
5095 			ret = pf_icmp_state_lookup(&key, &pd2, state, m,
5096 			    direction, kif, virtual_id, virtual_type,
5097 			    icmp_dir, &iidx, PF_ICMP_MULTI_NONE);
5098 			if (ret >= 0) {
5099 				if (ret == PF_DROP && pd->af == AF_INET6 &&
5100 				    icmp_dir == PF_OUT) {
5101 					if (*state)
5102 						PF_STATE_UNLOCK(*state);
5103 					ret = pf_icmp_state_lookup(&key, pd,
5104 					    state, m, direction, kif,
5105 					    virtual_id, virtual_type,
5106 					    icmp_dir, &iidx, multi);
5107 					if (ret >= 0)
5108 						return (ret);
5109 				} else
5110 					return (ret);
5111 			}
5112 
5113 			/* translate source/destination address, if necessary */
5114 			if ((*state)->key[PF_SK_WIRE] !=
5115 			    (*state)->key[PF_SK_STACK]) {
5116 				struct pf_state_key *nk =
5117 				    (*state)->key[pd->didx];
5118 
5119 				if (PF_ANEQ(pd2.src,
5120 				    &nk->addr[pd2.sidx], pd2.af) ||
5121 			 	    ((virtual_type == ICMP6_ECHO_REQUEST) &&
5122 				    nk->port[pd2.sidx] != iih.icmp6_id))
5123 					pf_change_icmp(pd2.src,
5124 					    (virtual_type == ICMP6_ECHO_REQUEST)
5125 					    ? &iih.icmp6_id : NULL,
5126 					    daddr, &nk->addr[pd2.sidx],
5127 					    (virtual_type == ICMP6_ECHO_REQUEST)
5128 					    ? nk->port[iidx] : 0, NULL,
5129 					    pd2.ip_sum, icmpsum,
5130 					    pd->ip_sum, 0, AF_INET6);
5131 
5132 				if (PF_ANEQ(pd2.dst,
5133 				    &nk->addr[pd2.didx], pd2.af))
5134 					pf_change_icmp(pd2.dst, NULL, NULL,
5135 					    &nk->addr[pd2.didx], 0, NULL,
5136 					    pd2.ip_sum, icmpsum,
5137 					    pd->ip_sum, 0, AF_INET6);
5138 
5139 				m_copyback(m, off, sizeof(struct icmp6_hdr),
5140 				    (caddr_t)pd->hdr.icmp6);
5141 				m_copyback(m, ipoff2, sizeof(h2_6), (caddr_t)&h2_6);
5142 				m_copyback(m, off2, sizeof(struct icmp6_hdr),
5143 				    (caddr_t)&iih);
5144 			}
5145 			return (PF_PASS);
5146 			break;
5147 		}
5148 #endif /* INET6 */
5149 		default: {
5150 			key.af = pd2.af;
5151 			key.proto = pd2.proto;
5152 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
5153 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
5154 			key.port[0] = key.port[1] = 0;
5155 
5156 			STATE_LOOKUP(kif, &key, direction, *state, pd);
5157 
5158 			/* translate source/destination address, if necessary */
5159 			if ((*state)->key[PF_SK_WIRE] !=
5160 			    (*state)->key[PF_SK_STACK]) {
5161 				struct pf_state_key *nk =
5162 				    (*state)->key[pd->didx];
5163 
5164 				if (PF_ANEQ(pd2.src,
5165 				    &nk->addr[pd2.sidx], pd2.af))
5166 					pf_change_icmp(pd2.src, NULL, daddr,
5167 					    &nk->addr[pd2.sidx], 0, NULL,
5168 					    pd2.ip_sum, icmpsum,
5169 					    pd->ip_sum, 0, pd2.af);
5170 
5171 				if (PF_ANEQ(pd2.dst,
5172 				    &nk->addr[pd2.didx], pd2.af))
5173 					pf_change_icmp(pd2.src, NULL,
5174 					    NULL, /* XXX Inbound NAT? */
5175 					    &nk->addr[pd2.didx], 0, NULL,
5176 					    pd2.ip_sum, icmpsum,
5177 					    pd->ip_sum, 0, pd2.af);
5178 
5179 				switch (pd2.af) {
5180 #ifdef INET
5181 				case AF_INET:
5182 					m_copyback(m, off, ICMP_MINLEN,
5183 					    (caddr_t)pd->hdr.icmp);
5184 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
5185 					break;
5186 #endif /* INET */
5187 #ifdef INET6
5188 				case AF_INET6:
5189 					m_copyback(m, off,
5190 					    sizeof(struct icmp6_hdr),
5191 					    (caddr_t )pd->hdr.icmp6);
5192 					m_copyback(m, ipoff2, sizeof(h2_6),
5193 					    (caddr_t )&h2_6);
5194 					break;
5195 #endif /* INET6 */
5196 				}
5197 			}
5198 			return (PF_PASS);
5199 			break;
5200 		}
5201 		}
5202 	}
5203 }
5204 
5205 static int
5206 pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif,
5207     struct mbuf *m, struct pf_pdesc *pd)
5208 {
5209 	struct pf_state_peer	*src, *dst;
5210 	struct pf_state_key_cmp	 key;
5211 
5212 	bzero(&key, sizeof(key));
5213 	key.af = pd->af;
5214 	key.proto = pd->proto;
5215 	if (direction == PF_IN)	{
5216 		PF_ACPY(&key.addr[0], pd->src, key.af);
5217 		PF_ACPY(&key.addr[1], pd->dst, key.af);
5218 		key.port[0] = key.port[1] = 0;
5219 	} else {
5220 		PF_ACPY(&key.addr[1], pd->src, key.af);
5221 		PF_ACPY(&key.addr[0], pd->dst, key.af);
5222 		key.port[1] = key.port[0] = 0;
5223 	}
5224 
5225 	STATE_LOOKUP(kif, &key, direction, *state, pd);
5226 
5227 	if (direction == (*state)->direction) {
5228 		src = &(*state)->src;
5229 		dst = &(*state)->dst;
5230 	} else {
5231 		src = &(*state)->dst;
5232 		dst = &(*state)->src;
5233 	}
5234 
5235 	/* update states */
5236 	if (src->state < PFOTHERS_SINGLE)
5237 		src->state = PFOTHERS_SINGLE;
5238 	if (dst->state == PFOTHERS_SINGLE)
5239 		dst->state = PFOTHERS_MULTIPLE;
5240 
5241 	/* update expire time */
5242 	(*state)->expire = time_uptime;
5243 	if (src->state == PFOTHERS_MULTIPLE && dst->state == PFOTHERS_MULTIPLE)
5244 		(*state)->timeout = PFTM_OTHER_MULTIPLE;
5245 	else
5246 		(*state)->timeout = PFTM_OTHER_SINGLE;
5247 
5248 	/* translate source/destination address, if necessary */
5249 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
5250 		struct pf_state_key *nk = (*state)->key[pd->didx];
5251 
5252 		KASSERT(nk, ("%s: nk is null", __func__));
5253 		KASSERT(pd, ("%s: pd is null", __func__));
5254 		KASSERT(pd->src, ("%s: pd->src is null", __func__));
5255 		KASSERT(pd->dst, ("%s: pd->dst is null", __func__));
5256 		switch (pd->af) {
5257 #ifdef INET
5258 		case AF_INET:
5259 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET))
5260 				pf_change_a(&pd->src->v4.s_addr,
5261 				    pd->ip_sum,
5262 				    nk->addr[pd->sidx].v4.s_addr,
5263 				    0);
5264 
5265 
5266 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET))
5267 				pf_change_a(&pd->dst->v4.s_addr,
5268 				    pd->ip_sum,
5269 				    nk->addr[pd->didx].v4.s_addr,
5270 				    0);
5271 
5272 				break;
5273 #endif /* INET */
5274 #ifdef INET6
5275 		case AF_INET6:
5276 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET))
5277 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
5278 
5279 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET))
5280 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
5281 #endif /* INET6 */
5282 		}
5283 	}
5284 	return (PF_PASS);
5285 }
5286 
5287 /*
5288  * ipoff and off are measured from the start of the mbuf chain.
5289  * h must be at "ipoff" on the mbuf chain.
5290  */
5291 void *
5292 pf_pull_hdr(struct mbuf *m, int off, void *p, int len,
5293     u_short *actionp, u_short *reasonp, sa_family_t af)
5294 {
5295 	switch (af) {
5296 #ifdef INET
5297 	case AF_INET: {
5298 		struct ip	*h = mtod(m, struct ip *);
5299 		u_int16_t	 fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3;
5300 
5301 		if (fragoff) {
5302 			if (fragoff >= len)
5303 				ACTION_SET(actionp, PF_PASS);
5304 			else {
5305 				ACTION_SET(actionp, PF_DROP);
5306 				REASON_SET(reasonp, PFRES_FRAG);
5307 			}
5308 			return (NULL);
5309 		}
5310 		if (m->m_pkthdr.len < off + len ||
5311 		    ntohs(h->ip_len) < off + len) {
5312 			ACTION_SET(actionp, PF_DROP);
5313 			REASON_SET(reasonp, PFRES_SHORT);
5314 			return (NULL);
5315 		}
5316 		break;
5317 	}
5318 #endif /* INET */
5319 #ifdef INET6
5320 	case AF_INET6: {
5321 		struct ip6_hdr	*h = mtod(m, struct ip6_hdr *);
5322 
5323 		if (m->m_pkthdr.len < off + len ||
5324 		    (ntohs(h->ip6_plen) + sizeof(struct ip6_hdr)) <
5325 		    (unsigned)(off + len)) {
5326 			ACTION_SET(actionp, PF_DROP);
5327 			REASON_SET(reasonp, PFRES_SHORT);
5328 			return (NULL);
5329 		}
5330 		break;
5331 	}
5332 #endif /* INET6 */
5333 	}
5334 	m_copydata(m, off, len, p);
5335 	return (p);
5336 }
5337 
5338 int
5339 pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kif *kif,
5340     int rtableid)
5341 {
5342 #ifdef RADIX_MPATH
5343 	struct radix_node_head	*rnh;
5344 #endif
5345 	struct sockaddr_in	*dst;
5346 	int			 ret = 1;
5347 	int			 check_mpath;
5348 #ifdef INET6
5349 	struct sockaddr_in6	*dst6;
5350 	struct route_in6	 ro;
5351 #else
5352 	struct route		 ro;
5353 #endif
5354 	struct radix_node	*rn;
5355 	struct rtentry		*rt;
5356 	struct ifnet		*ifp;
5357 
5358 	check_mpath = 0;
5359 #ifdef RADIX_MPATH
5360 	/* XXX: stick to table 0 for now */
5361 	rnh = rt_tables_get_rnh(0, af);
5362 	if (rnh != NULL && rn_mpath_capable(rnh))
5363 		check_mpath = 1;
5364 #endif
5365 	bzero(&ro, sizeof(ro));
5366 	switch (af) {
5367 	case AF_INET:
5368 		dst = satosin(&ro.ro_dst);
5369 		dst->sin_family = AF_INET;
5370 		dst->sin_len = sizeof(*dst);
5371 		dst->sin_addr = addr->v4;
5372 		break;
5373 #ifdef INET6
5374 	case AF_INET6:
5375 		/*
5376 		 * Skip check for addresses with embedded interface scope,
5377 		 * as they would always match anyway.
5378 		 */
5379 		if (IN6_IS_SCOPE_EMBED(&addr->v6))
5380 			goto out;
5381 		dst6 = (struct sockaddr_in6 *)&ro.ro_dst;
5382 		dst6->sin6_family = AF_INET6;
5383 		dst6->sin6_len = sizeof(*dst6);
5384 		dst6->sin6_addr = addr->v6;
5385 		break;
5386 #endif /* INET6 */
5387 	default:
5388 		return (0);
5389 	}
5390 
5391 	/* Skip checks for ipsec interfaces */
5392 	if (kif != NULL && kif->pfik_ifp->if_type == IFT_ENC)
5393 		goto out;
5394 
5395 	switch (af) {
5396 #ifdef INET6
5397 	case AF_INET6:
5398 		in6_rtalloc_ign(&ro, 0, rtableid);
5399 		break;
5400 #endif
5401 #ifdef INET
5402 	case AF_INET:
5403 		in_rtalloc_ign((struct route *)&ro, 0, rtableid);
5404 		break;
5405 #endif
5406 	default:
5407 		rtalloc_ign((struct route *)&ro, 0);	/* No/default FIB. */
5408 		break;
5409 	}
5410 
5411 	if (ro.ro_rt != NULL) {
5412 		/* No interface given, this is a no-route check */
5413 		if (kif == NULL)
5414 			goto out;
5415 
5416 		if (kif->pfik_ifp == NULL) {
5417 			ret = 0;
5418 			goto out;
5419 		}
5420 
5421 		/* Perform uRPF check if passed input interface */
5422 		ret = 0;
5423 		rn = (struct radix_node *)ro.ro_rt;
5424 		do {
5425 			rt = (struct rtentry *)rn;
5426 			ifp = rt->rt_ifp;
5427 
5428 			if (kif->pfik_ifp == ifp)
5429 				ret = 1;
5430 #ifdef RADIX_MPATH
5431 			rn = rn_mpath_next(rn);
5432 #endif
5433 		} while (check_mpath == 1 && rn != NULL && ret == 0);
5434 	} else
5435 		ret = 0;
5436 out:
5437 	if (ro.ro_rt != NULL)
5438 		RTFREE(ro.ro_rt);
5439 	return (ret);
5440 }
5441 
5442 #ifdef INET
5443 static void
5444 pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
5445     struct pf_state *s, struct pf_pdesc *pd)
5446 {
5447 	struct mbuf		*m0, *m1;
5448 	struct sockaddr_in	dst;
5449 	struct ip		*ip;
5450 	struct ifnet		*ifp = NULL;
5451 	struct pf_addr		 naddr;
5452 	struct pf_src_node	*sn = NULL;
5453 	int			 error = 0;
5454 	uint16_t		 ip_len, ip_off;
5455 
5456 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
5457 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: invalid direction",
5458 	    __func__));
5459 
5460 	if ((pd->pf_mtag == NULL &&
5461 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
5462 	    pd->pf_mtag->routed++ > 3) {
5463 		m0 = *m;
5464 		*m = NULL;
5465 		goto bad_locked;
5466 	}
5467 
5468 	if (r->rt == PF_DUPTO) {
5469 		if ((m0 = m_dup(*m, M_NOWAIT)) == NULL) {
5470 			if (s)
5471 				PF_STATE_UNLOCK(s);
5472 			return;
5473 		}
5474 	} else {
5475 		if ((r->rt == PF_REPLYTO) == (r->direction == dir)) {
5476 			if (s)
5477 				PF_STATE_UNLOCK(s);
5478 			return;
5479 		}
5480 		m0 = *m;
5481 	}
5482 
5483 	ip = mtod(m0, struct ip *);
5484 
5485 	bzero(&dst, sizeof(dst));
5486 	dst.sin_family = AF_INET;
5487 	dst.sin_len = sizeof(dst);
5488 	dst.sin_addr = ip->ip_dst;
5489 
5490 	if (r->rt == PF_FASTROUTE) {
5491 		struct rtentry *rt;
5492 
5493 		if (s)
5494 			PF_STATE_UNLOCK(s);
5495 		rt = rtalloc1_fib(sintosa(&dst), 0, 0, M_GETFIB(m0));
5496 		if (rt == NULL) {
5497 			RTFREE_LOCKED(rt);
5498 			KMOD_IPSTAT_INC(ips_noroute);
5499 			error = EHOSTUNREACH;
5500 			goto bad;
5501 		}
5502 
5503 		ifp = rt->rt_ifp;
5504 		rt->rt_rmx.rmx_pksent++;
5505 
5506 		if (rt->rt_flags & RTF_GATEWAY)
5507 			bcopy(satosin(rt->rt_gateway), &dst, sizeof(dst));
5508 		RTFREE_LOCKED(rt);
5509 	} else {
5510 		if (TAILQ_EMPTY(&r->rpool.list)) {
5511 			DPFPRINTF(PF_DEBUG_URGENT,
5512 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
5513 			goto bad_locked;
5514 		}
5515 		if (s == NULL) {
5516 			pf_map_addr(AF_INET, r, (struct pf_addr *)&ip->ip_src,
5517 			    &naddr, NULL, &sn);
5518 			if (!PF_AZERO(&naddr, AF_INET))
5519 				dst.sin_addr.s_addr = naddr.v4.s_addr;
5520 			ifp = r->rpool.cur->kif ?
5521 			    r->rpool.cur->kif->pfik_ifp : NULL;
5522 		} else {
5523 			if (!PF_AZERO(&s->rt_addr, AF_INET))
5524 				dst.sin_addr.s_addr =
5525 				    s->rt_addr.v4.s_addr;
5526 			ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
5527 			PF_STATE_UNLOCK(s);
5528 		}
5529 	}
5530 	if (ifp == NULL)
5531 		goto bad;
5532 
5533 	if (oifp != ifp) {
5534 		if (pf_test(PF_OUT, ifp, &m0, NULL) != PF_PASS)
5535 			goto bad;
5536 		else if (m0 == NULL)
5537 			goto done;
5538 		if (m0->m_len < sizeof(struct ip)) {
5539 			DPFPRINTF(PF_DEBUG_URGENT,
5540 			    ("%s: m0->m_len < sizeof(struct ip)\n", __func__));
5541 			goto bad;
5542 		}
5543 		ip = mtod(m0, struct ip *);
5544 	}
5545 
5546 	if (ifp->if_flags & IFF_LOOPBACK)
5547 		m0->m_flags |= M_SKIP_FIREWALL;
5548 
5549 	ip_len = ntohs(ip->ip_len);
5550 	ip_off = ntohs(ip->ip_off);
5551 
5552 	/* Copied from FreeBSD 10.0-CURRENT ip_output. */
5553 	m0->m_pkthdr.csum_flags |= CSUM_IP;
5554 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~ifp->if_hwassist) {
5555 		in_delayed_cksum(m0);
5556 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
5557 	}
5558 #ifdef SCTP
5559 	if (m0->m_pkthdr.csum_flags & CSUM_SCTP & ~ifp->if_hwassist) {
5560 		sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
5561 		m0->m_pkthdr.csum_flags &= ~CSUM_SCTP;
5562 	}
5563 #endif
5564 
5565 	/*
5566 	 * If small enough for interface, or the interface will take
5567 	 * care of the fragmentation for us, we can just send directly.
5568 	 */
5569 	if (ip_len <= ifp->if_mtu ||
5570 	    (m0->m_pkthdr.csum_flags & ifp->if_hwassist & CSUM_TSO) != 0 ||
5571 	    ((ip_off & IP_DF) == 0 && (ifp->if_hwassist & CSUM_FRAGMENT))) {
5572 		ip->ip_sum = 0;
5573 		if (m0->m_pkthdr.csum_flags & CSUM_IP & ~ifp->if_hwassist) {
5574 			ip->ip_sum = in_cksum(m0, ip->ip_hl << 2);
5575 			m0->m_pkthdr.csum_flags &= ~CSUM_IP;
5576 		}
5577 		m_clrprotoflags(m0);	/* Avoid confusing lower layers. */
5578 		error = (*ifp->if_output)(ifp, m0, sintosa(&dst), NULL);
5579 		goto done;
5580 	}
5581 
5582 	/* Balk when DF bit is set or the interface didn't support TSO. */
5583 	if ((ip_off & IP_DF) || (m0->m_pkthdr.csum_flags & CSUM_TSO)) {
5584 		error = EMSGSIZE;
5585 		KMOD_IPSTAT_INC(ips_cantfrag);
5586 		if (r->rt != PF_DUPTO) {
5587 			icmp_error(m0, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG, 0,
5588 			    ifp->if_mtu);
5589 			goto done;
5590 		} else
5591 			goto bad;
5592 	}
5593 
5594 	error = ip_fragment(ip, &m0, ifp->if_mtu, ifp->if_hwassist);
5595 	if (error)
5596 		goto bad;
5597 
5598 	for (; m0; m0 = m1) {
5599 		m1 = m0->m_nextpkt;
5600 		m0->m_nextpkt = NULL;
5601 		if (error == 0) {
5602 			m_clrprotoflags(m0);
5603 			error = (*ifp->if_output)(ifp, m0, sintosa(&dst), NULL);
5604 		} else
5605 			m_freem(m0);
5606 	}
5607 
5608 	if (error == 0)
5609 		KMOD_IPSTAT_INC(ips_fragmented);
5610 
5611 done:
5612 	if (r->rt != PF_DUPTO)
5613 		*m = NULL;
5614 	return;
5615 
5616 bad_locked:
5617 	if (s)
5618 		PF_STATE_UNLOCK(s);
5619 bad:
5620 	m_freem(m0);
5621 	goto done;
5622 }
5623 #endif /* INET */
5624 
5625 #ifdef INET6
5626 static void
5627 pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
5628     struct pf_state *s, struct pf_pdesc *pd)
5629 {
5630 	struct mbuf		*m0;
5631 	struct sockaddr_in6	dst;
5632 	struct ip6_hdr		*ip6;
5633 	struct ifnet		*ifp = NULL;
5634 	struct pf_addr		 naddr;
5635 	struct pf_src_node	*sn = NULL;
5636 
5637 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
5638 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: invalid direction",
5639 	    __func__));
5640 
5641 	if ((pd->pf_mtag == NULL &&
5642 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
5643 	    pd->pf_mtag->routed++ > 3) {
5644 		m0 = *m;
5645 		*m = NULL;
5646 		goto bad_locked;
5647 	}
5648 
5649 	if (r->rt == PF_DUPTO) {
5650 		if ((m0 = m_dup(*m, M_NOWAIT)) == NULL) {
5651 			if (s)
5652 				PF_STATE_UNLOCK(s);
5653 			return;
5654 		}
5655 	} else {
5656 		if ((r->rt == PF_REPLYTO) == (r->direction == dir)) {
5657 			if (s)
5658 				PF_STATE_UNLOCK(s);
5659 			return;
5660 		}
5661 		m0 = *m;
5662 	}
5663 
5664 	ip6 = mtod(m0, struct ip6_hdr *);
5665 
5666 	bzero(&dst, sizeof(dst));
5667 	dst.sin6_family = AF_INET6;
5668 	dst.sin6_len = sizeof(dst);
5669 	dst.sin6_addr = ip6->ip6_dst;
5670 
5671 	/* Cheat. XXX why only in the v6 case??? */
5672 	if (r->rt == PF_FASTROUTE) {
5673 		if (s)
5674 			PF_STATE_UNLOCK(s);
5675 		m0->m_flags |= M_SKIP_FIREWALL;
5676 		ip6_output(m0, NULL, NULL, 0, NULL, NULL, NULL);
5677 		return;
5678 	}
5679 
5680 	if (TAILQ_EMPTY(&r->rpool.list)) {
5681 		DPFPRINTF(PF_DEBUG_URGENT,
5682 		    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
5683 		goto bad_locked;
5684 	}
5685 	if (s == NULL) {
5686 		pf_map_addr(AF_INET6, r, (struct pf_addr *)&ip6->ip6_src,
5687 		    &naddr, NULL, &sn);
5688 		if (!PF_AZERO(&naddr, AF_INET6))
5689 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
5690 			    &naddr, AF_INET6);
5691 		ifp = r->rpool.cur->kif ? r->rpool.cur->kif->pfik_ifp : NULL;
5692 	} else {
5693 		if (!PF_AZERO(&s->rt_addr, AF_INET6))
5694 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
5695 			    &s->rt_addr, AF_INET6);
5696 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
5697 	}
5698 
5699 	if (s)
5700 		PF_STATE_UNLOCK(s);
5701 
5702 	if (ifp == NULL)
5703 		goto bad;
5704 
5705 	if (oifp != ifp) {
5706 		if (pf_test6(PF_OUT, ifp, &m0, NULL) != PF_PASS)
5707 			goto bad;
5708 		else if (m0 == NULL)
5709 			goto done;
5710 		if (m0->m_len < sizeof(struct ip6_hdr)) {
5711 			DPFPRINTF(PF_DEBUG_URGENT,
5712 			    ("%s: m0->m_len < sizeof(struct ip6_hdr)\n",
5713 			    __func__));
5714 			goto bad;
5715 		}
5716 		ip6 = mtod(m0, struct ip6_hdr *);
5717 	}
5718 
5719 	if (ifp->if_flags & IFF_LOOPBACK)
5720 		m0->m_flags |= M_SKIP_FIREWALL;
5721 
5722 	/*
5723 	 * If the packet is too large for the outgoing interface,
5724 	 * send back an icmp6 error.
5725 	 */
5726 	if (IN6_IS_SCOPE_EMBED(&dst.sin6_addr))
5727 		dst.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
5728 	if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu)
5729 		nd6_output(ifp, ifp, m0, &dst, NULL);
5730 	else {
5731 		in6_ifstat_inc(ifp, ifs6_in_toobig);
5732 		if (r->rt != PF_DUPTO)
5733 			icmp6_error(m0, ICMP6_PACKET_TOO_BIG, 0, ifp->if_mtu);
5734 		else
5735 			goto bad;
5736 	}
5737 
5738 done:
5739 	if (r->rt != PF_DUPTO)
5740 		*m = NULL;
5741 	return;
5742 
5743 bad_locked:
5744 	if (s)
5745 		PF_STATE_UNLOCK(s);
5746 bad:
5747 	m_freem(m0);
5748 	goto done;
5749 }
5750 #endif /* INET6 */
5751 
5752 /*
5753  * FreeBSD supports cksum offloads for the following drivers.
5754  *  em(4), fxp(4), ixgb(4), lge(4), ndis(4), nge(4), re(4),
5755  *   ti(4), txp(4), xl(4)
5756  *
5757  * CSUM_DATA_VALID | CSUM_PSEUDO_HDR :
5758  *  network driver performed cksum including pseudo header, need to verify
5759  *   csum_data
5760  * CSUM_DATA_VALID :
5761  *  network driver performed cksum, needs to additional pseudo header
5762  *  cksum computation with partial csum_data(i.e. lack of H/W support for
5763  *  pseudo header, for instance hme(4), sk(4) and possibly gem(4))
5764  *
5765  * After validating the cksum of packet, set both flag CSUM_DATA_VALID and
5766  * CSUM_PSEUDO_HDR in order to avoid recomputation of the cksum in upper
5767  * TCP/UDP layer.
5768  * Also, set csum_data to 0xffff to force cksum validation.
5769  */
5770 static int
5771 pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t af)
5772 {
5773 	u_int16_t sum = 0;
5774 	int hw_assist = 0;
5775 	struct ip *ip;
5776 
5777 	if (off < sizeof(struct ip) || len < sizeof(struct udphdr))
5778 		return (1);
5779 	if (m->m_pkthdr.len < off + len)
5780 		return (1);
5781 
5782 	switch (p) {
5783 	case IPPROTO_TCP:
5784 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
5785 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
5786 				sum = m->m_pkthdr.csum_data;
5787 			} else {
5788 				ip = mtod(m, struct ip *);
5789 				sum = in_pseudo(ip->ip_src.s_addr,
5790 				ip->ip_dst.s_addr, htonl((u_short)len +
5791 				m->m_pkthdr.csum_data + IPPROTO_TCP));
5792 			}
5793 			sum ^= 0xffff;
5794 			++hw_assist;
5795 		}
5796 		break;
5797 	case IPPROTO_UDP:
5798 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
5799 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
5800 				sum = m->m_pkthdr.csum_data;
5801 			} else {
5802 				ip = mtod(m, struct ip *);
5803 				sum = in_pseudo(ip->ip_src.s_addr,
5804 				ip->ip_dst.s_addr, htonl((u_short)len +
5805 				m->m_pkthdr.csum_data + IPPROTO_UDP));
5806 			}
5807 			sum ^= 0xffff;
5808 			++hw_assist;
5809 		}
5810 		break;
5811 	case IPPROTO_ICMP:
5812 #ifdef INET6
5813 	case IPPROTO_ICMPV6:
5814 #endif /* INET6 */
5815 		break;
5816 	default:
5817 		return (1);
5818 	}
5819 
5820 	if (!hw_assist) {
5821 		switch (af) {
5822 		case AF_INET:
5823 			if (p == IPPROTO_ICMP) {
5824 				if (m->m_len < off)
5825 					return (1);
5826 				m->m_data += off;
5827 				m->m_len -= off;
5828 				sum = in_cksum(m, len);
5829 				m->m_data -= off;
5830 				m->m_len += off;
5831 			} else {
5832 				if (m->m_len < sizeof(struct ip))
5833 					return (1);
5834 				sum = in4_cksum(m, p, off, len);
5835 			}
5836 			break;
5837 #ifdef INET6
5838 		case AF_INET6:
5839 			if (m->m_len < sizeof(struct ip6_hdr))
5840 				return (1);
5841 			sum = in6_cksum(m, p, off, len);
5842 			break;
5843 #endif /* INET6 */
5844 		default:
5845 			return (1);
5846 		}
5847 	}
5848 	if (sum) {
5849 		switch (p) {
5850 		case IPPROTO_TCP:
5851 		    {
5852 			KMOD_TCPSTAT_INC(tcps_rcvbadsum);
5853 			break;
5854 		    }
5855 		case IPPROTO_UDP:
5856 		    {
5857 			KMOD_UDPSTAT_INC(udps_badsum);
5858 			break;
5859 		    }
5860 #ifdef INET
5861 		case IPPROTO_ICMP:
5862 		    {
5863 			KMOD_ICMPSTAT_INC(icps_checksum);
5864 			break;
5865 		    }
5866 #endif
5867 #ifdef INET6
5868 		case IPPROTO_ICMPV6:
5869 		    {
5870 			KMOD_ICMP6STAT_INC(icp6s_checksum);
5871 			break;
5872 		    }
5873 #endif /* INET6 */
5874 		}
5875 		return (1);
5876 	} else {
5877 		if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
5878 			m->m_pkthdr.csum_flags |=
5879 			    (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5880 			m->m_pkthdr.csum_data = 0xffff;
5881 		}
5882 	}
5883 	return (0);
5884 }
5885 
5886 
5887 #ifdef INET
5888 int
5889 pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp)
5890 {
5891 	struct pfi_kif		*kif;
5892 	u_short			 action, reason = 0, log = 0;
5893 	struct mbuf		*m = *m0;
5894 	struct ip		*h = NULL;
5895 	struct m_tag		*ipfwtag;
5896 	struct pf_rule		*a = NULL, *r = &V_pf_default_rule, *tr, *nr;
5897 	struct pf_state		*s = NULL;
5898 	struct pf_ruleset	*ruleset = NULL;
5899 	struct pf_pdesc		 pd;
5900 	int			 off, dirndx, pqid = 0;
5901 
5902 	M_ASSERTPKTHDR(m);
5903 
5904 	if (!V_pf_status.running)
5905 		return (PF_PASS);
5906 
5907 	memset(&pd, 0, sizeof(pd));
5908 
5909 	kif = (struct pfi_kif *)ifp->if_pf_kif;
5910 
5911 	if (kif == NULL) {
5912 		DPFPRINTF(PF_DEBUG_URGENT,
5913 		    ("pf_test: kif == NULL, if_xname %s\n", ifp->if_xname));
5914 		return (PF_DROP);
5915 	}
5916 	if (kif->pfik_flags & PFI_IFLAG_SKIP)
5917 		return (PF_PASS);
5918 
5919 	if (m->m_flags & M_SKIP_FIREWALL)
5920 		return (PF_PASS);
5921 
5922 	pd.pf_mtag = pf_find_mtag(m);
5923 
5924 	PF_RULES_RLOCK();
5925 
5926 	if (ip_divert_ptr != NULL &&
5927 	    ((ipfwtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL)) != NULL)) {
5928 		struct ipfw_rule_ref *rr = (struct ipfw_rule_ref *)(ipfwtag+1);
5929 		if (rr->info & IPFW_IS_DIVERT && rr->rulenum == 0) {
5930 			if (pd.pf_mtag == NULL &&
5931 			    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
5932 				action = PF_DROP;
5933 				goto done;
5934 			}
5935 			pd.pf_mtag->flags |= PF_PACKET_LOOPED;
5936 			m_tag_delete(m, ipfwtag);
5937 		}
5938 		if (pd.pf_mtag && pd.pf_mtag->flags & PF_FASTFWD_OURS_PRESENT) {
5939 			m->m_flags |= M_FASTFWD_OURS;
5940 			pd.pf_mtag->flags &= ~PF_FASTFWD_OURS_PRESENT;
5941 		}
5942 	} else if (pf_normalize_ip(m0, dir, kif, &reason, &pd) != PF_PASS) {
5943 		/* We do IP header normalization and packet reassembly here */
5944 		action = PF_DROP;
5945 		goto done;
5946 	}
5947 	m = *m0;	/* pf_normalize messes with m0 */
5948 	h = mtod(m, struct ip *);
5949 
5950 	off = h->ip_hl << 2;
5951 	if (off < (int)sizeof(struct ip)) {
5952 		action = PF_DROP;
5953 		REASON_SET(&reason, PFRES_SHORT);
5954 		log = 1;
5955 		goto done;
5956 	}
5957 
5958 	pd.src = (struct pf_addr *)&h->ip_src;
5959 	pd.dst = (struct pf_addr *)&h->ip_dst;
5960 	pd.sport = pd.dport = NULL;
5961 	pd.ip_sum = &h->ip_sum;
5962 	pd.proto_sum = NULL;
5963 	pd.proto = h->ip_p;
5964 	pd.dir = dir;
5965 	pd.sidx = (dir == PF_IN) ? 0 : 1;
5966 	pd.didx = (dir == PF_IN) ? 1 : 0;
5967 	pd.af = AF_INET;
5968 	pd.tos = h->ip_tos;
5969 	pd.tot_len = ntohs(h->ip_len);
5970 
5971 	/* handle fragments that didn't get reassembled by normalization */
5972 	if (h->ip_off & htons(IP_MF | IP_OFFMASK)) {
5973 		action = pf_test_fragment(&r, dir, kif, m, h,
5974 		    &pd, &a, &ruleset);
5975 		goto done;
5976 	}
5977 
5978 	switch (h->ip_p) {
5979 
5980 	case IPPROTO_TCP: {
5981 		struct tcphdr	th;
5982 
5983 		pd.hdr.tcp = &th;
5984 		if (!pf_pull_hdr(m, off, &th, sizeof(th),
5985 		    &action, &reason, AF_INET)) {
5986 			log = action != PF_PASS;
5987 			goto done;
5988 		}
5989 		pd.p_len = pd.tot_len - off - (th.th_off << 2);
5990 		if ((th.th_flags & TH_ACK) && pd.p_len == 0)
5991 			pqid = 1;
5992 		action = pf_normalize_tcp(dir, kif, m, 0, off, h, &pd);
5993 		if (action == PF_DROP)
5994 			goto done;
5995 		action = pf_test_state_tcp(&s, dir, kif, m, off, h, &pd,
5996 		    &reason);
5997 		if (action == PF_PASS) {
5998 			if (pfsync_update_state_ptr != NULL)
5999 				pfsync_update_state_ptr(s);
6000 			r = s->rule.ptr;
6001 			a = s->anchor.ptr;
6002 			log = s->log;
6003 		} else if (s == NULL)
6004 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6005 			    &a, &ruleset, inp);
6006 		break;
6007 	}
6008 
6009 	case IPPROTO_UDP: {
6010 		struct udphdr	uh;
6011 
6012 		pd.hdr.udp = &uh;
6013 		if (!pf_pull_hdr(m, off, &uh, sizeof(uh),
6014 		    &action, &reason, AF_INET)) {
6015 			log = action != PF_PASS;
6016 			goto done;
6017 		}
6018 		if (uh.uh_dport == 0 ||
6019 		    ntohs(uh.uh_ulen) > m->m_pkthdr.len - off ||
6020 		    ntohs(uh.uh_ulen) < sizeof(struct udphdr)) {
6021 			action = PF_DROP;
6022 			REASON_SET(&reason, PFRES_SHORT);
6023 			goto done;
6024 		}
6025 		action = pf_test_state_udp(&s, dir, kif, m, off, h, &pd);
6026 		if (action == PF_PASS) {
6027 			if (pfsync_update_state_ptr != NULL)
6028 				pfsync_update_state_ptr(s);
6029 			r = s->rule.ptr;
6030 			a = s->anchor.ptr;
6031 			log = s->log;
6032 		} else if (s == NULL)
6033 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6034 			    &a, &ruleset, inp);
6035 		break;
6036 	}
6037 
6038 	case IPPROTO_ICMP: {
6039 		struct icmp	ih;
6040 
6041 		pd.hdr.icmp = &ih;
6042 		if (!pf_pull_hdr(m, off, &ih, ICMP_MINLEN,
6043 		    &action, &reason, AF_INET)) {
6044 			log = action != PF_PASS;
6045 			goto done;
6046 		}
6047 		action = pf_test_state_icmp(&s, dir, kif, m, off, h, &pd,
6048 		    &reason);
6049 		if (action == PF_PASS) {
6050 			if (pfsync_update_state_ptr != NULL)
6051 				pfsync_update_state_ptr(s);
6052 			r = s->rule.ptr;
6053 			a = s->anchor.ptr;
6054 			log = s->log;
6055 		} else if (s == NULL)
6056 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6057 			    &a, &ruleset, inp);
6058 		break;
6059 	}
6060 
6061 #ifdef INET6
6062 	case IPPROTO_ICMPV6: {
6063 		action = PF_DROP;
6064 		DPFPRINTF(PF_DEBUG_MISC,
6065 		    ("pf: dropping IPv4 packet with ICMPv6 payload\n"));
6066 		goto done;
6067 	}
6068 #endif
6069 
6070 	default:
6071 		action = pf_test_state_other(&s, dir, kif, m, &pd);
6072 		if (action == PF_PASS) {
6073 			if (pfsync_update_state_ptr != NULL)
6074 				pfsync_update_state_ptr(s);
6075 			r = s->rule.ptr;
6076 			a = s->anchor.ptr;
6077 			log = s->log;
6078 		} else if (s == NULL)
6079 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6080 			    &a, &ruleset, inp);
6081 		break;
6082 	}
6083 
6084 done:
6085 	PF_RULES_RUNLOCK();
6086 	if (action == PF_PASS && h->ip_hl > 5 &&
6087 	    !((s && s->state_flags & PFSTATE_ALLOWOPTS) || r->allow_opts)) {
6088 		action = PF_DROP;
6089 		REASON_SET(&reason, PFRES_IPOPTIONS);
6090 		log = 1;
6091 		DPFPRINTF(PF_DEBUG_MISC,
6092 		    ("pf: dropping packet with ip options\n"));
6093 	}
6094 
6095 	if (s && s->tag > 0 && pf_tag_packet(m, &pd, s->tag)) {
6096 		action = PF_DROP;
6097 		REASON_SET(&reason, PFRES_MEMORY);
6098 	}
6099 	if (r->rtableid >= 0)
6100 		M_SETFIB(m, r->rtableid);
6101 
6102 #ifdef ALTQ
6103 	if (action == PF_PASS && r->qid) {
6104 		if (pd.pf_mtag == NULL &&
6105 		    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
6106 			action = PF_DROP;
6107 			REASON_SET(&reason, PFRES_MEMORY);
6108 		}
6109 		if (pqid || (pd.tos & IPTOS_LOWDELAY))
6110 			pd.pf_mtag->qid = r->pqid;
6111 		else
6112 			pd.pf_mtag->qid = r->qid;
6113 		/* add hints for ecn */
6114 		pd.pf_mtag->hdr = h;
6115 
6116 	}
6117 #endif /* ALTQ */
6118 
6119 	/*
6120 	 * connections redirected to loopback should not match sockets
6121 	 * bound specifically to loopback due to security implications,
6122 	 * see tcp_input() and in_pcblookup_listen().
6123 	 */
6124 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
6125 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL &&
6126 	    (s->nat_rule.ptr->action == PF_RDR ||
6127 	    s->nat_rule.ptr->action == PF_BINAT) &&
6128 	    (ntohl(pd.dst->v4.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET)
6129 		m->m_flags |= M_SKIP_FIREWALL;
6130 
6131 	if (action == PF_PASS && r->divert.port && ip_divert_ptr != NULL &&
6132 	    !PACKET_LOOPED(&pd)) {
6133 
6134 		ipfwtag = m_tag_alloc(MTAG_IPFW_RULE, 0,
6135 		    sizeof(struct ipfw_rule_ref), M_NOWAIT | M_ZERO);
6136 		if (ipfwtag != NULL) {
6137 			((struct ipfw_rule_ref *)(ipfwtag+1))->info =
6138 			    ntohs(r->divert.port);
6139 			((struct ipfw_rule_ref *)(ipfwtag+1))->rulenum = dir;
6140 
6141 			if (s)
6142 				PF_STATE_UNLOCK(s);
6143 
6144 			m_tag_prepend(m, ipfwtag);
6145 			if (m->m_flags & M_FASTFWD_OURS) {
6146 				if (pd.pf_mtag == NULL &&
6147 				    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
6148 					action = PF_DROP;
6149 					REASON_SET(&reason, PFRES_MEMORY);
6150 					log = 1;
6151 					DPFPRINTF(PF_DEBUG_MISC,
6152 					    ("pf: failed to allocate tag\n"));
6153 				}
6154 				pd.pf_mtag->flags |= PF_FASTFWD_OURS_PRESENT;
6155 				m->m_flags &= ~M_FASTFWD_OURS;
6156 			}
6157 			ip_divert_ptr(*m0, dir ==  PF_IN ? DIR_IN : DIR_OUT);
6158 			*m0 = NULL;
6159 
6160 			return (action);
6161 		} else {
6162 			/* XXX: ipfw has the same behaviour! */
6163 			action = PF_DROP;
6164 			REASON_SET(&reason, PFRES_MEMORY);
6165 			log = 1;
6166 			DPFPRINTF(PF_DEBUG_MISC,
6167 			    ("pf: failed to allocate divert tag\n"));
6168 		}
6169 	}
6170 
6171 	if (log) {
6172 		struct pf_rule *lr;
6173 
6174 		if (s != NULL && s->nat_rule.ptr != NULL &&
6175 		    s->nat_rule.ptr->log & PF_LOG_ALL)
6176 			lr = s->nat_rule.ptr;
6177 		else
6178 			lr = r;
6179 		PFLOG_PACKET(kif, m, AF_INET, dir, reason, lr, a, ruleset, &pd,
6180 		    (s == NULL));
6181 	}
6182 
6183 	kif->pfik_bytes[0][dir == PF_OUT][action != PF_PASS] += pd.tot_len;
6184 	kif->pfik_packets[0][dir == PF_OUT][action != PF_PASS]++;
6185 
6186 	if (action == PF_PASS || r->action == PF_DROP) {
6187 		dirndx = (dir == PF_OUT);
6188 		r->packets[dirndx]++;
6189 		r->bytes[dirndx] += pd.tot_len;
6190 		if (a != NULL) {
6191 			a->packets[dirndx]++;
6192 			a->bytes[dirndx] += pd.tot_len;
6193 		}
6194 		if (s != NULL) {
6195 			if (s->nat_rule.ptr != NULL) {
6196 				s->nat_rule.ptr->packets[dirndx]++;
6197 				s->nat_rule.ptr->bytes[dirndx] += pd.tot_len;
6198 			}
6199 			if (s->src_node != NULL) {
6200 				s->src_node->packets[dirndx]++;
6201 				s->src_node->bytes[dirndx] += pd.tot_len;
6202 			}
6203 			if (s->nat_src_node != NULL) {
6204 				s->nat_src_node->packets[dirndx]++;
6205 				s->nat_src_node->bytes[dirndx] += pd.tot_len;
6206 			}
6207 			dirndx = (dir == s->direction) ? 0 : 1;
6208 			s->packets[dirndx]++;
6209 			s->bytes[dirndx] += pd.tot_len;
6210 		}
6211 		tr = r;
6212 		nr = (s != NULL) ? s->nat_rule.ptr : pd.nat_rule;
6213 		if (nr != NULL && r == &V_pf_default_rule)
6214 			tr = nr;
6215 		if (tr->src.addr.type == PF_ADDR_TABLE)
6216 			pfr_update_stats(tr->src.addr.p.tbl,
6217 			    (s == NULL) ? pd.src :
6218 			    &s->key[(s->direction == PF_IN)]->
6219 				addr[(s->direction == PF_OUT)],
6220 			    pd.af, pd.tot_len, dir == PF_OUT,
6221 			    r->action == PF_PASS, tr->src.neg);
6222 		if (tr->dst.addr.type == PF_ADDR_TABLE)
6223 			pfr_update_stats(tr->dst.addr.p.tbl,
6224 			    (s == NULL) ? pd.dst :
6225 			    &s->key[(s->direction == PF_IN)]->
6226 				addr[(s->direction == PF_IN)],
6227 			    pd.af, pd.tot_len, dir == PF_OUT,
6228 			    r->action == PF_PASS, tr->dst.neg);
6229 	}
6230 
6231 	switch (action) {
6232 	case PF_SYNPROXY_DROP:
6233 		m_freem(*m0);
6234 	case PF_DEFER:
6235 		*m0 = NULL;
6236 		action = PF_PASS;
6237 		break;
6238 	default:
6239 		/* pf_route() returns unlocked. */
6240 		if (r->rt) {
6241 			pf_route(m0, r, dir, kif->pfik_ifp, s, &pd);
6242 			return (action);
6243 		}
6244 		break;
6245 	}
6246 	if (s)
6247 		PF_STATE_UNLOCK(s);
6248 
6249 	return (action);
6250 }
6251 #endif /* INET */
6252 
6253 #ifdef INET6
6254 int
6255 pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, struct inpcb *inp)
6256 {
6257 	struct pfi_kif		*kif;
6258 	u_short			 action, reason = 0, log = 0;
6259 	struct mbuf		*m = *m0, *n = NULL;
6260 	struct ip6_hdr		*h = NULL;
6261 	struct pf_rule		*a = NULL, *r = &V_pf_default_rule, *tr, *nr;
6262 	struct pf_state		*s = NULL;
6263 	struct pf_ruleset	*ruleset = NULL;
6264 	struct pf_pdesc		 pd;
6265 	int			 off, terminal = 0, dirndx, rh_cnt = 0;
6266 
6267 	M_ASSERTPKTHDR(m);
6268 
6269 	if (!V_pf_status.running)
6270 		return (PF_PASS);
6271 
6272 	memset(&pd, 0, sizeof(pd));
6273 	pd.pf_mtag = pf_find_mtag(m);
6274 
6275 	if (pd.pf_mtag && pd.pf_mtag->flags & PF_TAG_GENERATED)
6276 		return (PF_PASS);
6277 
6278 	kif = (struct pfi_kif *)ifp->if_pf_kif;
6279 	if (kif == NULL) {
6280 		DPFPRINTF(PF_DEBUG_URGENT,
6281 		    ("pf_test6: kif == NULL, if_xname %s\n", ifp->if_xname));
6282 		return (PF_DROP);
6283 	}
6284 	if (kif->pfik_flags & PFI_IFLAG_SKIP)
6285 		return (PF_PASS);
6286 
6287 	PF_RULES_RLOCK();
6288 
6289 	/* We do IP header normalization and packet reassembly here */
6290 	if (pf_normalize_ip6(m0, dir, kif, &reason, &pd) != PF_PASS) {
6291 		action = PF_DROP;
6292 		goto done;
6293 	}
6294 	m = *m0;	/* pf_normalize messes with m0 */
6295 	h = mtod(m, struct ip6_hdr *);
6296 
6297 #if 1
6298 	/*
6299 	 * we do not support jumbogram yet.  if we keep going, zero ip6_plen
6300 	 * will do something bad, so drop the packet for now.
6301 	 */
6302 	if (htons(h->ip6_plen) == 0) {
6303 		action = PF_DROP;
6304 		REASON_SET(&reason, PFRES_NORM);	/*XXX*/
6305 		goto done;
6306 	}
6307 #endif
6308 
6309 	pd.src = (struct pf_addr *)&h->ip6_src;
6310 	pd.dst = (struct pf_addr *)&h->ip6_dst;
6311 	pd.sport = pd.dport = NULL;
6312 	pd.ip_sum = NULL;
6313 	pd.proto_sum = NULL;
6314 	pd.dir = dir;
6315 	pd.sidx = (dir == PF_IN) ? 0 : 1;
6316 	pd.didx = (dir == PF_IN) ? 1 : 0;
6317 	pd.af = AF_INET6;
6318 	pd.tos = 0;
6319 	pd.tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr);
6320 
6321 	off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr);
6322 	pd.proto = h->ip6_nxt;
6323 	do {
6324 		switch (pd.proto) {
6325 		case IPPROTO_FRAGMENT:
6326 			action = pf_test_fragment(&r, dir, kif, m, h,
6327 			    &pd, &a, &ruleset);
6328 			if (action == PF_DROP)
6329 				REASON_SET(&reason, PFRES_FRAG);
6330 			goto done;
6331 		case IPPROTO_ROUTING: {
6332 			struct ip6_rthdr rthdr;
6333 
6334 			if (rh_cnt++) {
6335 				DPFPRINTF(PF_DEBUG_MISC,
6336 				    ("pf: IPv6 more than one rthdr\n"));
6337 				action = PF_DROP;
6338 				REASON_SET(&reason, PFRES_IPOPTIONS);
6339 				log = 1;
6340 				goto done;
6341 			}
6342 			if (!pf_pull_hdr(m, off, &rthdr, sizeof(rthdr), NULL,
6343 			    &reason, pd.af)) {
6344 				DPFPRINTF(PF_DEBUG_MISC,
6345 				    ("pf: IPv6 short rthdr\n"));
6346 				action = PF_DROP;
6347 				REASON_SET(&reason, PFRES_SHORT);
6348 				log = 1;
6349 				goto done;
6350 			}
6351 			if (rthdr.ip6r_type == IPV6_RTHDR_TYPE_0) {
6352 				DPFPRINTF(PF_DEBUG_MISC,
6353 				    ("pf: IPv6 rthdr0\n"));
6354 				action = PF_DROP;
6355 				REASON_SET(&reason, PFRES_IPOPTIONS);
6356 				log = 1;
6357 				goto done;
6358 			}
6359 			/* FALLTHROUGH */
6360 		}
6361 		case IPPROTO_AH:
6362 		case IPPROTO_HOPOPTS:
6363 		case IPPROTO_DSTOPTS: {
6364 			/* get next header and header length */
6365 			struct ip6_ext	opt6;
6366 
6367 			if (!pf_pull_hdr(m, off, &opt6, sizeof(opt6),
6368 			    NULL, &reason, pd.af)) {
6369 				DPFPRINTF(PF_DEBUG_MISC,
6370 				    ("pf: IPv6 short opt\n"));
6371 				action = PF_DROP;
6372 				log = 1;
6373 				goto done;
6374 			}
6375 			if (pd.proto == IPPROTO_AH)
6376 				off += (opt6.ip6e_len + 2) * 4;
6377 			else
6378 				off += (opt6.ip6e_len + 1) * 8;
6379 			pd.proto = opt6.ip6e_nxt;
6380 			/* goto the next header */
6381 			break;
6382 		}
6383 		default:
6384 			terminal++;
6385 			break;
6386 		}
6387 	} while (!terminal);
6388 
6389 	/* if there's no routing header, use unmodified mbuf for checksumming */
6390 	if (!n)
6391 		n = m;
6392 
6393 	switch (pd.proto) {
6394 
6395 	case IPPROTO_TCP: {
6396 		struct tcphdr	th;
6397 
6398 		pd.hdr.tcp = &th;
6399 		if (!pf_pull_hdr(m, off, &th, sizeof(th),
6400 		    &action, &reason, AF_INET6)) {
6401 			log = action != PF_PASS;
6402 			goto done;
6403 		}
6404 		pd.p_len = pd.tot_len - off - (th.th_off << 2);
6405 		action = pf_normalize_tcp(dir, kif, m, 0, off, h, &pd);
6406 		if (action == PF_DROP)
6407 			goto done;
6408 		action = pf_test_state_tcp(&s, dir, kif, m, off, h, &pd,
6409 		    &reason);
6410 		if (action == PF_PASS) {
6411 			if (pfsync_update_state_ptr != NULL)
6412 				pfsync_update_state_ptr(s);
6413 			r = s->rule.ptr;
6414 			a = s->anchor.ptr;
6415 			log = s->log;
6416 		} else if (s == NULL)
6417 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6418 			    &a, &ruleset, inp);
6419 		break;
6420 	}
6421 
6422 	case IPPROTO_UDP: {
6423 		struct udphdr	uh;
6424 
6425 		pd.hdr.udp = &uh;
6426 		if (!pf_pull_hdr(m, off, &uh, sizeof(uh),
6427 		    &action, &reason, AF_INET6)) {
6428 			log = action != PF_PASS;
6429 			goto done;
6430 		}
6431 		if (uh.uh_dport == 0 ||
6432 		    ntohs(uh.uh_ulen) > m->m_pkthdr.len - off ||
6433 		    ntohs(uh.uh_ulen) < sizeof(struct udphdr)) {
6434 			action = PF_DROP;
6435 			REASON_SET(&reason, PFRES_SHORT);
6436 			goto done;
6437 		}
6438 		action = pf_test_state_udp(&s, dir, kif, m, off, h, &pd);
6439 		if (action == PF_PASS) {
6440 			if (pfsync_update_state_ptr != NULL)
6441 				pfsync_update_state_ptr(s);
6442 			r = s->rule.ptr;
6443 			a = s->anchor.ptr;
6444 			log = s->log;
6445 		} else if (s == NULL)
6446 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6447 			    &a, &ruleset, inp);
6448 		break;
6449 	}
6450 
6451 	case IPPROTO_ICMP: {
6452 		action = PF_DROP;
6453 		DPFPRINTF(PF_DEBUG_MISC,
6454 		    ("pf: dropping IPv6 packet with ICMPv4 payload\n"));
6455 		goto done;
6456 	}
6457 
6458 	case IPPROTO_ICMPV6: {
6459 		union {
6460 			struct icmp6_hdr		icmp6;
6461 			struct mld_hdr			mld;
6462 			struct nd_neighbor_solicit	nd;
6463 		} ih;
6464 		size_t	icmp_hlen = sizeof(struct icmp6_hdr);
6465 
6466 		pd.hdr.icmp6 = &ih.icmp6;
6467 		if (!pf_pull_hdr(m, off, &ih, icmp_hlen,
6468 		    &action, &reason, AF_INET6)) {
6469 			log = action != PF_PASS;
6470 			goto done;
6471 		}
6472 		/* ICMP headers we look further into to match state */
6473 		switch (ih.icmp6.icmp6_type) {
6474 		case MLD_LISTENER_QUERY:
6475 		case MLD_LISTENER_REPORT:
6476 			icmp_hlen = sizeof(struct mld_hdr);
6477 			break;
6478 		case ND_NEIGHBOR_SOLICIT:
6479 		case ND_NEIGHBOR_ADVERT:
6480 			icmp_hlen = sizeof(struct nd_neighbor_solicit);
6481 			break;
6482 		}
6483 		if (icmp_hlen > sizeof(struct icmp6_hdr) &&
6484 		    !pf_pull_hdr(m, off, &ih, icmp_hlen,
6485 		    &action, &reason, AF_INET6)) {
6486 			log = action != PF_PASS;
6487 			goto done;
6488 		}
6489 		action = pf_test_state_icmp(&s, dir, kif,
6490 		    m, off, h, &pd, &reason);
6491 		if (action == PF_PASS) {
6492 			if (pfsync_update_state_ptr != NULL)
6493 				pfsync_update_state_ptr(s);
6494 			r = s->rule.ptr;
6495 			a = s->anchor.ptr;
6496 			log = s->log;
6497 		} else if (s == NULL)
6498 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6499 			    &a, &ruleset, inp);
6500 		break;
6501 	}
6502 
6503 	default:
6504 		action = pf_test_state_other(&s, dir, kif, m, &pd);
6505 		if (action == PF_PASS) {
6506 			if (pfsync_update_state_ptr != NULL)
6507 				pfsync_update_state_ptr(s);
6508 			r = s->rule.ptr;
6509 			a = s->anchor.ptr;
6510 			log = s->log;
6511 		} else if (s == NULL)
6512 			action = pf_test_rule(&r, &s, dir, kif, m, off, &pd,
6513 			    &a, &ruleset, inp);
6514 		break;
6515 	}
6516 
6517 done:
6518 	PF_RULES_RUNLOCK();
6519 	if (n != m) {
6520 		m_freem(n);
6521 		n = NULL;
6522 	}
6523 
6524 	/* handle dangerous IPv6 extension headers. */
6525 	if (action == PF_PASS && rh_cnt &&
6526 	    !((s && s->state_flags & PFSTATE_ALLOWOPTS) || r->allow_opts)) {
6527 		action = PF_DROP;
6528 		REASON_SET(&reason, PFRES_IPOPTIONS);
6529 		log = 1;
6530 		DPFPRINTF(PF_DEBUG_MISC,
6531 		    ("pf: dropping packet with dangerous v6 headers\n"));
6532 	}
6533 
6534 	if (s && s->tag > 0 && pf_tag_packet(m, &pd, s->tag)) {
6535 		action = PF_DROP;
6536 		REASON_SET(&reason, PFRES_MEMORY);
6537 	}
6538 	if (r->rtableid >= 0)
6539 		M_SETFIB(m, r->rtableid);
6540 
6541 #ifdef ALTQ
6542 	if (action == PF_PASS && r->qid) {
6543 		if (pd.pf_mtag == NULL &&
6544 		    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
6545 			action = PF_DROP;
6546 			REASON_SET(&reason, PFRES_MEMORY);
6547 		}
6548 		if (pd.tos & IPTOS_LOWDELAY)
6549 			pd.pf_mtag->qid = r->pqid;
6550 		else
6551 			pd.pf_mtag->qid = r->qid;
6552 		/* add hints for ecn */
6553 		pd.pf_mtag->hdr = h;
6554 	}
6555 #endif /* ALTQ */
6556 
6557 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
6558 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL &&
6559 	    (s->nat_rule.ptr->action == PF_RDR ||
6560 	    s->nat_rule.ptr->action == PF_BINAT) &&
6561 	    IN6_IS_ADDR_LOOPBACK(&pd.dst->v6))
6562 		m->m_flags |= M_SKIP_FIREWALL;
6563 
6564 	/* XXX: Anybody working on it?! */
6565 	if (r->divert.port)
6566 		printf("pf: divert(9) is not supported for IPv6\n");
6567 
6568 	if (log) {
6569 		struct pf_rule *lr;
6570 
6571 		if (s != NULL && s->nat_rule.ptr != NULL &&
6572 		    s->nat_rule.ptr->log & PF_LOG_ALL)
6573 			lr = s->nat_rule.ptr;
6574 		else
6575 			lr = r;
6576 		PFLOG_PACKET(kif, m, AF_INET6, dir, reason, lr, a, ruleset,
6577 		    &pd, (s == NULL));
6578 	}
6579 
6580 	kif->pfik_bytes[1][dir == PF_OUT][action != PF_PASS] += pd.tot_len;
6581 	kif->pfik_packets[1][dir == PF_OUT][action != PF_PASS]++;
6582 
6583 	if (action == PF_PASS || r->action == PF_DROP) {
6584 		dirndx = (dir == PF_OUT);
6585 		r->packets[dirndx]++;
6586 		r->bytes[dirndx] += pd.tot_len;
6587 		if (a != NULL) {
6588 			a->packets[dirndx]++;
6589 			a->bytes[dirndx] += pd.tot_len;
6590 		}
6591 		if (s != NULL) {
6592 			if (s->nat_rule.ptr != NULL) {
6593 				s->nat_rule.ptr->packets[dirndx]++;
6594 				s->nat_rule.ptr->bytes[dirndx] += pd.tot_len;
6595 			}
6596 			if (s->src_node != NULL) {
6597 				s->src_node->packets[dirndx]++;
6598 				s->src_node->bytes[dirndx] += pd.tot_len;
6599 			}
6600 			if (s->nat_src_node != NULL) {
6601 				s->nat_src_node->packets[dirndx]++;
6602 				s->nat_src_node->bytes[dirndx] += pd.tot_len;
6603 			}
6604 			dirndx = (dir == s->direction) ? 0 : 1;
6605 			s->packets[dirndx]++;
6606 			s->bytes[dirndx] += pd.tot_len;
6607 		}
6608 		tr = r;
6609 		nr = (s != NULL) ? s->nat_rule.ptr : pd.nat_rule;
6610 		if (nr != NULL && r == &V_pf_default_rule)
6611 			tr = nr;
6612 		if (tr->src.addr.type == PF_ADDR_TABLE)
6613 			pfr_update_stats(tr->src.addr.p.tbl,
6614 			    (s == NULL) ? pd.src :
6615 			    &s->key[(s->direction == PF_IN)]->addr[0],
6616 			    pd.af, pd.tot_len, dir == PF_OUT,
6617 			    r->action == PF_PASS, tr->src.neg);
6618 		if (tr->dst.addr.type == PF_ADDR_TABLE)
6619 			pfr_update_stats(tr->dst.addr.p.tbl,
6620 			    (s == NULL) ? pd.dst :
6621 			    &s->key[(s->direction == PF_IN)]->addr[1],
6622 			    pd.af, pd.tot_len, dir == PF_OUT,
6623 			    r->action == PF_PASS, tr->dst.neg);
6624 	}
6625 
6626 	switch (action) {
6627 	case PF_SYNPROXY_DROP:
6628 		m_freem(*m0);
6629 	case PF_DEFER:
6630 		*m0 = NULL;
6631 		action = PF_PASS;
6632 		break;
6633 	default:
6634 		/* pf_route6() returns unlocked. */
6635 		if (r->rt) {
6636 			pf_route6(m0, r, dir, kif->pfik_ifp, s, &pd);
6637 			return (action);
6638 		}
6639 		break;
6640 	}
6641 
6642 	if (s)
6643 		PF_STATE_UNLOCK(s);
6644 
6645 	return (action);
6646 }
6647 #endif /* INET6 */
6648