xref: /freebsd/sys/netpfil/pf/pf.c (revision 215fd38e2915d142cb4b0245f137329ef4da5a12)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2001 Daniel Hartmeier
5  * Copyright (c) 2002 - 2008 Henning Brauer
6  * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  *    - Redistributions of source code must retain the above copyright
14  *      notice, this list of conditions and the following disclaimer.
15  *    - Redistributions in binary form must reproduce the above
16  *      copyright notice, this list of conditions and the following
17  *      disclaimer in the documentation and/or other materials provided
18  *      with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGE.
32  *
33  * Effort sponsored in part by the Defense Advanced Research Projects
34  * Agency (DARPA) and Air Force Research Laboratory, Air Force
35  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
36  *
37  *	$OpenBSD: pf.c,v 1.634 2009/02/27 12:37:45 henning Exp $
38  */
39 
40 #include <sys/cdefs.h>
41 #include "opt_bpf.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_pf.h"
45 #include "opt_sctp.h"
46 
47 #include <sys/param.h>
48 #include <sys/bus.h>
49 #include <sys/endian.h>
50 #include <sys/gsb_crc32.h>
51 #include <sys/hash.h>
52 #include <sys/interrupt.h>
53 #include <sys/kernel.h>
54 #include <sys/kthread.h>
55 #include <sys/limits.h>
56 #include <sys/mbuf.h>
57 #include <sys/md5.h>
58 #include <sys/random.h>
59 #include <sys/refcount.h>
60 #include <sys/sdt.h>
61 #include <sys/socket.h>
62 #include <sys/sysctl.h>
63 #include <sys/taskqueue.h>
64 #include <sys/ucred.h>
65 
66 #include <net/if.h>
67 #include <net/if_var.h>
68 #include <net/if_private.h>
69 #include <net/if_types.h>
70 #include <net/if_vlan_var.h>
71 #include <net/route.h>
72 #include <net/route/nhop.h>
73 #include <net/vnet.h>
74 
75 #include <net/pfil.h>
76 #include <net/pfvar.h>
77 #include <net/if_pflog.h>
78 #include <net/if_pfsync.h>
79 
80 #include <netinet/in_pcb.h>
81 #include <netinet/in_var.h>
82 #include <netinet/in_fib.h>
83 #include <netinet/ip.h>
84 #include <netinet/ip_fw.h>
85 #include <netinet/ip_icmp.h>
86 #include <netinet/icmp_var.h>
87 #include <netinet/ip_var.h>
88 #include <netinet/tcp.h>
89 #include <netinet/tcp_fsm.h>
90 #include <netinet/tcp_seq.h>
91 #include <netinet/tcp_timer.h>
92 #include <netinet/tcp_var.h>
93 #include <netinet/udp.h>
94 #include <netinet/udp_var.h>
95 
96 /* dummynet */
97 #include <netinet/ip_dummynet.h>
98 #include <netinet/ip_fw.h>
99 #include <netpfil/ipfw/dn_heap.h>
100 #include <netpfil/ipfw/ip_fw_private.h>
101 #include <netpfil/ipfw/ip_dn_private.h>
102 
103 #ifdef INET6
104 #include <netinet/ip6.h>
105 #include <netinet/icmp6.h>
106 #include <netinet6/nd6.h>
107 #include <netinet6/ip6_var.h>
108 #include <netinet6/in6_pcb.h>
109 #include <netinet6/in6_fib.h>
110 #include <netinet6/scope6_var.h>
111 #endif /* INET6 */
112 
113 #include <netinet/sctp_header.h>
114 #include <netinet/sctp_crc32.h>
115 
116 #include <machine/in_cksum.h>
117 #include <security/mac/mac_framework.h>
118 
119 #define	DPFPRINTF(n, x)	if (V_pf_status.debug >= (n)) printf x
120 
121 SDT_PROVIDER_DEFINE(pf);
122 SDT_PROBE_DEFINE4(pf, ip, test, done, "int", "int", "struct pf_krule *",
123     "struct pf_kstate *");
124 SDT_PROBE_DEFINE5(pf, ip, state, lookup, "struct pfi_kkif *",
125     "struct pf_state_key_cmp *", "int", "struct pf_pdesc *",
126     "struct pf_kstate *");
127 SDT_PROBE_DEFINE2(pf, ip, , bound_iface, "struct pf_kstate *",
128     "struct pfi_kkif *");
129 SDT_PROBE_DEFINE4(pf, sctp, multihome, test, "struct pfi_kkif *",
130     "struct pf_krule *", "struct mbuf *", "int");
131 SDT_PROBE_DEFINE2(pf, sctp, multihome, add, "uint32_t",
132     "struct pf_sctp_source *");
133 SDT_PROBE_DEFINE3(pf, sctp, multihome, remove, "uint32_t",
134     "struct pf_kstate *", "struct pf_sctp_source *");
135 
136 SDT_PROBE_DEFINE3(pf, eth, test_rule, entry, "int", "struct ifnet *",
137     "struct mbuf *");
138 SDT_PROBE_DEFINE2(pf, eth, test_rule, test, "int", "struct pf_keth_rule *");
139 SDT_PROBE_DEFINE3(pf, eth, test_rule, mismatch,
140     "int", "struct pf_keth_rule *", "char *");
141 SDT_PROBE_DEFINE2(pf, eth, test_rule, match, "int", "struct pf_keth_rule *");
142 SDT_PROBE_DEFINE2(pf, eth, test_rule, final_match,
143     "int", "struct pf_keth_rule *");
144 SDT_PROBE_DEFINE2(pf, purge, state, rowcount, "int", "size_t");
145 
146 /*
147  * Global variables
148  */
149 
150 /* state tables */
151 VNET_DEFINE(struct pf_altqqueue,	 pf_altqs[4]);
152 VNET_DEFINE(struct pf_kpalist,		 pf_pabuf);
153 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_active);
154 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_active);
155 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_inactive);
156 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_inactive);
157 VNET_DEFINE(struct pf_kstatus,		 pf_status);
158 
159 VNET_DEFINE(u_int32_t,			 ticket_altqs_active);
160 VNET_DEFINE(u_int32_t,			 ticket_altqs_inactive);
161 VNET_DEFINE(int,			 altqs_inactive_open);
162 VNET_DEFINE(u_int32_t,			 ticket_pabuf);
163 
164 VNET_DEFINE(MD5_CTX,			 pf_tcp_secret_ctx);
165 #define	V_pf_tcp_secret_ctx		 VNET(pf_tcp_secret_ctx)
166 VNET_DEFINE(u_char,			 pf_tcp_secret[16]);
167 #define	V_pf_tcp_secret			 VNET(pf_tcp_secret)
168 VNET_DEFINE(int,			 pf_tcp_secret_init);
169 #define	V_pf_tcp_secret_init		 VNET(pf_tcp_secret_init)
170 VNET_DEFINE(int,			 pf_tcp_iss_off);
171 #define	V_pf_tcp_iss_off		 VNET(pf_tcp_iss_off)
172 VNET_DECLARE(int,			 pf_vnet_active);
173 #define	V_pf_vnet_active		 VNET(pf_vnet_active)
174 
175 VNET_DEFINE_STATIC(uint32_t, pf_purge_idx);
176 #define V_pf_purge_idx	VNET(pf_purge_idx)
177 
178 #ifdef PF_WANT_32_TO_64_COUNTER
179 VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter);
180 #define	V_pf_counter_periodic_iter	VNET(pf_counter_periodic_iter)
181 
182 VNET_DEFINE(struct allrulelist_head, pf_allrulelist);
183 VNET_DEFINE(size_t, pf_allrulecount);
184 VNET_DEFINE(struct pf_krule *, pf_rulemarker);
185 #endif
186 
187 struct pf_sctp_endpoint;
188 RB_HEAD(pf_sctp_endpoints, pf_sctp_endpoint);
189 struct pf_sctp_source {
190 	sa_family_t			af;
191 	struct pf_addr			addr;
192 	TAILQ_ENTRY(pf_sctp_source)	entry;
193 };
194 TAILQ_HEAD(pf_sctp_sources, pf_sctp_source);
195 struct pf_sctp_endpoint
196 {
197 	uint32_t		 v_tag;
198 	struct pf_sctp_sources	 sources;
199 	RB_ENTRY(pf_sctp_endpoint)	entry;
200 };
201 static int
202 pf_sctp_endpoint_compare(struct pf_sctp_endpoint *a, struct pf_sctp_endpoint *b)
203 {
204 	return (a->v_tag - b->v_tag);
205 }
206 RB_PROTOTYPE(pf_sctp_endpoints, pf_sctp_endpoint, entry, pf_sctp_endpoint_compare);
207 RB_GENERATE(pf_sctp_endpoints, pf_sctp_endpoint, entry, pf_sctp_endpoint_compare);
208 VNET_DEFINE_STATIC(struct pf_sctp_endpoints, pf_sctp_endpoints);
209 #define V_pf_sctp_endpoints	VNET(pf_sctp_endpoints)
210 static struct mtx_padalign pf_sctp_endpoints_mtx;
211 MTX_SYSINIT(pf_sctp_endpoints_mtx, &pf_sctp_endpoints_mtx, "SCTP endpoints", MTX_DEF);
212 #define	PF_SCTP_ENDPOINTS_LOCK()	mtx_lock(&pf_sctp_endpoints_mtx)
213 #define	PF_SCTP_ENDPOINTS_UNLOCK()	mtx_unlock(&pf_sctp_endpoints_mtx)
214 
215 /*
216  * Queue for pf_intr() sends.
217  */
218 static MALLOC_DEFINE(M_PFTEMP, "pf_temp", "pf(4) temporary allocations");
219 struct pf_send_entry {
220 	STAILQ_ENTRY(pf_send_entry)	pfse_next;
221 	struct mbuf			*pfse_m;
222 	enum {
223 		PFSE_IP,
224 		PFSE_IP6,
225 		PFSE_ICMP,
226 		PFSE_ICMP6,
227 	}				pfse_type;
228 	struct {
229 		int		type;
230 		int		code;
231 		int		mtu;
232 	} icmpopts;
233 };
234 
235 STAILQ_HEAD(pf_send_head, pf_send_entry);
236 VNET_DEFINE_STATIC(struct pf_send_head, pf_sendqueue);
237 #define	V_pf_sendqueue	VNET(pf_sendqueue)
238 
239 static struct mtx_padalign pf_sendqueue_mtx;
240 MTX_SYSINIT(pf_sendqueue_mtx, &pf_sendqueue_mtx, "pf send queue", MTX_DEF);
241 #define	PF_SENDQ_LOCK()		mtx_lock(&pf_sendqueue_mtx)
242 #define	PF_SENDQ_UNLOCK()	mtx_unlock(&pf_sendqueue_mtx)
243 
244 /*
245  * Queue for pf_overload_task() tasks.
246  */
247 struct pf_overload_entry {
248 	SLIST_ENTRY(pf_overload_entry)	next;
249 	struct pf_addr  		addr;
250 	sa_family_t			af;
251 	uint8_t				dir;
252 	struct pf_krule  		*rule;
253 };
254 
255 SLIST_HEAD(pf_overload_head, pf_overload_entry);
256 VNET_DEFINE_STATIC(struct pf_overload_head, pf_overloadqueue);
257 #define V_pf_overloadqueue	VNET(pf_overloadqueue)
258 VNET_DEFINE_STATIC(struct task, pf_overloadtask);
259 #define	V_pf_overloadtask	VNET(pf_overloadtask)
260 
261 static struct mtx_padalign pf_overloadqueue_mtx;
262 MTX_SYSINIT(pf_overloadqueue_mtx, &pf_overloadqueue_mtx,
263     "pf overload/flush queue", MTX_DEF);
264 #define	PF_OVERLOADQ_LOCK()	mtx_lock(&pf_overloadqueue_mtx)
265 #define	PF_OVERLOADQ_UNLOCK()	mtx_unlock(&pf_overloadqueue_mtx)
266 
267 VNET_DEFINE(struct pf_krulequeue, pf_unlinked_rules);
268 struct mtx_padalign pf_unlnkdrules_mtx;
269 MTX_SYSINIT(pf_unlnkdrules_mtx, &pf_unlnkdrules_mtx, "pf unlinked rules",
270     MTX_DEF);
271 
272 struct sx pf_config_lock;
273 SX_SYSINIT(pf_config_lock, &pf_config_lock, "pf config");
274 
275 struct mtx_padalign pf_table_stats_lock;
276 MTX_SYSINIT(pf_table_stats_lock, &pf_table_stats_lock, "pf table stats",
277     MTX_DEF);
278 
279 VNET_DEFINE_STATIC(uma_zone_t,	pf_sources_z);
280 #define	V_pf_sources_z	VNET(pf_sources_z)
281 uma_zone_t		pf_mtag_z;
282 VNET_DEFINE(uma_zone_t,	 pf_state_z);
283 VNET_DEFINE(uma_zone_t,	 pf_state_key_z);
284 VNET_DEFINE(uma_zone_t,	 pf_udp_mapping_z);
285 
286 VNET_DEFINE(struct unrhdr64, pf_stateid);
287 
288 static void		 pf_src_tree_remove_state(struct pf_kstate *);
289 static void		 pf_init_threshold(struct pf_threshold *, u_int32_t,
290 			    u_int32_t);
291 static void		 pf_add_threshold(struct pf_threshold *);
292 static int		 pf_check_threshold(struct pf_threshold *);
293 
294 static void		 pf_change_ap(struct mbuf *, struct pf_addr *, u_int16_t *,
295 			    u_int16_t *, u_int16_t *, struct pf_addr *,
296 			    u_int16_t, u_int8_t, sa_family_t);
297 static int		 pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *,
298 			    struct tcphdr *, struct pf_state_peer *);
299 int			 pf_icmp_mapping(struct pf_pdesc *, u_int8_t, int *,
300 			    int *, u_int16_t *, u_int16_t *);
301 static void		 pf_change_icmp(struct pf_addr *, u_int16_t *,
302 			    struct pf_addr *, struct pf_addr *, u_int16_t,
303 			    u_int16_t *, u_int16_t *, u_int16_t *,
304 			    u_int16_t *, u_int8_t, sa_family_t);
305 static void		 pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t,
306 			    sa_family_t, struct pf_krule *, int);
307 static void		 pf_detach_state(struct pf_kstate *);
308 static int		 pf_state_key_attach(struct pf_state_key *,
309 			    struct pf_state_key *, struct pf_kstate *);
310 static void		 pf_state_key_detach(struct pf_kstate *, int);
311 static int		 pf_state_key_ctor(void *, int, void *, int);
312 static u_int32_t	 pf_tcp_iss(struct pf_pdesc *);
313 static __inline void	 pf_dummynet_flag_remove(struct mbuf *m,
314 			    struct pf_mtag *pf_mtag);
315 static int		 pf_dummynet(struct pf_pdesc *, struct pf_kstate *,
316 			    struct pf_krule *, struct mbuf **);
317 static int		 pf_dummynet_route(struct pf_pdesc *,
318 			    struct pf_kstate *, struct pf_krule *,
319 			    struct ifnet *, struct sockaddr *, struct mbuf **);
320 static int		 pf_test_eth_rule(int, struct pfi_kkif *,
321 			    struct mbuf **);
322 static int		 pf_test_rule(struct pf_krule **, struct pf_kstate **,
323 			    struct pfi_kkif *, struct mbuf *, int,
324 			    struct pf_pdesc *, struct pf_krule **,
325 			    struct pf_kruleset **, struct inpcb *, int);
326 static int		 pf_create_state(struct pf_krule *, struct pf_krule *,
327 			    struct pf_krule *, struct pf_pdesc *,
328 			    struct pf_ksrc_node *, struct pf_state_key *,
329 			    struct pf_state_key *, struct mbuf *, int,
330 			    u_int16_t, u_int16_t, int *, struct pfi_kkif *,
331 			    struct pf_kstate **, int, u_int16_t, u_int16_t,
332 			    int, struct pf_krule_slist *, struct pf_udp_mapping *);
333 static int		 pf_state_key_addr_setup(struct pf_pdesc *, struct mbuf *,
334 			    int, struct pf_state_key_cmp *, int, struct pf_addr *,
335 			    int, struct pf_addr *, int);
336 static int		 pf_tcp_track_full(struct pf_kstate **,
337 			    struct pfi_kkif *, struct mbuf *, int,
338 			    struct pf_pdesc *, u_short *, int *);
339 static int		 pf_tcp_track_sloppy(struct pf_kstate **,
340 			    struct pf_pdesc *, u_short *);
341 static int		 pf_test_state_tcp(struct pf_kstate **,
342 			    struct pfi_kkif *, struct mbuf *, int,
343 			    struct pf_pdesc *, u_short *);
344 static int		 pf_test_state_udp(struct pf_kstate **,
345 			    struct pfi_kkif *, struct mbuf *, int,
346 			    struct pf_pdesc *);
347 int			 pf_icmp_state_lookup(struct pf_state_key_cmp *,
348 			    struct pf_pdesc *, struct pf_kstate **, struct mbuf *,
349 			    int, int, struct pfi_kkif *, u_int16_t, u_int16_t,
350 			    int, int *, int, int);
351 static int		 pf_test_state_icmp(struct pf_kstate **,
352 			    struct pfi_kkif *, struct mbuf *, int,
353 			    struct pf_pdesc *, u_short *);
354 static void		 pf_sctp_multihome_detach_addr(const struct pf_kstate *);
355 static void		 pf_sctp_multihome_delayed(struct pf_pdesc *, int,
356 			    struct pfi_kkif *, struct pf_kstate *, int);
357 static int		 pf_test_state_sctp(struct pf_kstate **,
358 			    struct pfi_kkif *, struct mbuf *, int,
359 			    struct pf_pdesc *, u_short *);
360 static int		 pf_test_state_other(struct pf_kstate **,
361 			    struct pfi_kkif *, struct mbuf *, struct pf_pdesc *);
362 static u_int16_t	 pf_calc_mss(struct pf_addr *, sa_family_t,
363 				int, u_int16_t);
364 static int		 pf_check_proto_cksum(struct mbuf *, int, int,
365 			    u_int8_t, sa_family_t);
366 static void		 pf_print_state_parts(struct pf_kstate *,
367 			    struct pf_state_key *, struct pf_state_key *);
368 static void		 pf_patch_8(struct mbuf *, u_int16_t *, u_int8_t *, u_int8_t,
369 			    bool, u_int8_t);
370 static struct pf_kstate	*pf_find_state(struct pfi_kkif *,
371 			    const struct pf_state_key_cmp *, u_int);
372 static int		 pf_src_connlimit(struct pf_kstate **);
373 static int		 pf_match_rcvif(struct mbuf *, struct pf_krule *);
374 static void		 pf_counters_inc(int,
375 			    struct pf_pdesc *, struct pfi_kkif *,
376 			    struct pf_kstate *, struct pf_krule *,
377 			    struct pf_krule *);
378 static void		 pf_overload_task(void *v, int pending);
379 static u_short		 pf_insert_src_node(struct pf_ksrc_node **,
380 			    struct pf_krule *, struct pf_addr *, sa_family_t);
381 static u_int		 pf_purge_expired_states(u_int, int);
382 static void		 pf_purge_unlinked_rules(void);
383 static int		 pf_mtag_uminit(void *, int, int);
384 static void		 pf_mtag_free(struct m_tag *);
385 static void		 pf_packet_rework_nat(struct mbuf *, struct pf_pdesc *,
386 			    int, struct pf_state_key *);
387 #ifdef INET
388 static void		 pf_route(struct mbuf **, struct pf_krule *,
389 			    struct ifnet *, struct pf_kstate *,
390 			    struct pf_pdesc *, struct inpcb *);
391 #endif /* INET */
392 #ifdef INET6
393 static void		 pf_change_a6(struct pf_addr *, u_int16_t *,
394 			    struct pf_addr *, u_int8_t);
395 static void		 pf_route6(struct mbuf **, struct pf_krule *,
396 			    struct ifnet *, struct pf_kstate *,
397 			    struct pf_pdesc *, struct inpcb *);
398 #endif /* INET6 */
399 static __inline void pf_set_protostate(struct pf_kstate *, int, u_int8_t);
400 
401 int in4_cksum(struct mbuf *m, u_int8_t nxt, int off, int len);
402 
403 extern int pf_end_threads;
404 extern struct proc *pf_purge_proc;
405 
406 VNET_DEFINE(struct pf_limit, pf_limits[PF_LIMIT_MAX]);
407 
408 enum { PF_ICMP_MULTI_NONE, PF_ICMP_MULTI_LINK };
409 
410 #define	PACKET_UNDO_NAT(_m, _pd, _off, _s)		\
411 	do {								\
412 		struct pf_state_key *nk;				\
413 		if ((pd->dir) == PF_OUT)					\
414 			nk = (_s)->key[PF_SK_STACK];			\
415 		else							\
416 			nk = (_s)->key[PF_SK_WIRE];			\
417 		pf_packet_rework_nat(_m, _pd, _off, nk);		\
418 	} while (0)
419 
420 #define	PACKET_LOOPED(pd)	((pd)->pf_mtag &&			\
421 				 (pd)->pf_mtag->flags & PF_MTAG_FLAG_PACKET_LOOPED)
422 
423 #define	STATE_LOOKUP(i, k, s, pd)					\
424 	do {								\
425 		(s) = pf_find_state((i), (k), (pd->dir));			\
426 		SDT_PROBE5(pf, ip, state, lookup, i, k, (pd->dir), pd, (s));	\
427 		if ((s) == NULL)					\
428 			return (PF_DROP);				\
429 		if (PACKET_LOOPED(pd))					\
430 			return (PF_PASS);				\
431 	} while (0)
432 
433 static struct pfi_kkif *
434 BOUND_IFACE(struct pf_kstate *st, struct pfi_kkif *k)
435 {
436 	SDT_PROBE2(pf, ip, , bound_iface, st, k);
437 
438 	/* Floating unless otherwise specified. */
439 	if (! (st->rule.ptr->rule_flag & PFRULE_IFBOUND))
440 		return (V_pfi_all);
441 
442 	/*
443 	 * Initially set to all, because we don't know what interface we'll be
444 	 * sending this out when we create the state.
445 	 */
446 	if (st->rule.ptr->rt == PF_REPLYTO)
447 		return (V_pfi_all);
448 
449 	/* Don't overrule the interface for states created on incoming packets. */
450 	if (st->direction == PF_IN)
451 		return (k);
452 
453 	/* No route-to, so don't overrule. */
454 	if (st->rt != PF_ROUTETO)
455 		return (k);
456 
457 	/* Bind to the route-to interface. */
458 	return (st->rt_kif);
459 }
460 
461 #define	STATE_INC_COUNTERS(s)						\
462 	do {								\
463 		struct pf_krule_item *mrm;				\
464 		counter_u64_add(s->rule.ptr->states_cur, 1);		\
465 		counter_u64_add(s->rule.ptr->states_tot, 1);		\
466 		if (s->anchor.ptr != NULL) {				\
467 			counter_u64_add(s->anchor.ptr->states_cur, 1);	\
468 			counter_u64_add(s->anchor.ptr->states_tot, 1);	\
469 		}							\
470 		if (s->nat_rule.ptr != NULL) {				\
471 			counter_u64_add(s->nat_rule.ptr->states_cur, 1);\
472 			counter_u64_add(s->nat_rule.ptr->states_tot, 1);\
473 		}							\
474 		SLIST_FOREACH(mrm, &s->match_rules, entry) {		\
475 			counter_u64_add(mrm->r->states_cur, 1);		\
476 			counter_u64_add(mrm->r->states_tot, 1);		\
477 		}							\
478 	} while (0)
479 
480 #define	STATE_DEC_COUNTERS(s)						\
481 	do {								\
482 		struct pf_krule_item *mrm;				\
483 		if (s->nat_rule.ptr != NULL)				\
484 			counter_u64_add(s->nat_rule.ptr->states_cur, -1);\
485 		if (s->anchor.ptr != NULL)				\
486 			counter_u64_add(s->anchor.ptr->states_cur, -1);	\
487 		counter_u64_add(s->rule.ptr->states_cur, -1);		\
488 		SLIST_FOREACH(mrm, &s->match_rules, entry)		\
489 			counter_u64_add(mrm->r->states_cur, -1);	\
490 	} while (0)
491 
492 MALLOC_DEFINE(M_PFHASH, "pf_hash", "pf(4) hash header structures");
493 MALLOC_DEFINE(M_PF_RULE_ITEM, "pf_krule_item", "pf(4) rule items");
494 VNET_DEFINE(struct pf_keyhash *, pf_keyhash);
495 VNET_DEFINE(struct pf_idhash *, pf_idhash);
496 VNET_DEFINE(struct pf_srchash *, pf_srchash);
497 VNET_DEFINE(struct pf_udpendpointhash *, pf_udpendpointhash);
498 VNET_DEFINE(struct pf_udpendpointmapping *, pf_udpendpointmapping);
499 
500 SYSCTL_NODE(_net, OID_AUTO, pf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
501     "pf(4)");
502 
503 VNET_DEFINE(u_long, pf_hashmask);
504 VNET_DEFINE(u_long, pf_srchashmask);
505 VNET_DEFINE(u_long, pf_udpendpointhashmask);
506 VNET_DEFINE_STATIC(u_long, pf_hashsize);
507 #define V_pf_hashsize	VNET(pf_hashsize)
508 VNET_DEFINE_STATIC(u_long, pf_srchashsize);
509 #define V_pf_srchashsize	VNET(pf_srchashsize)
510 VNET_DEFINE_STATIC(u_long, pf_udpendpointhashsize);
511 #define V_pf_udpendpointhashsize	VNET(pf_udpendpointhashsize)
512 u_long	pf_ioctl_maxcount = 65535;
513 
514 SYSCTL_ULONG(_net_pf, OID_AUTO, states_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
515     &VNET_NAME(pf_hashsize), 0, "Size of pf(4) states hashtable");
516 SYSCTL_ULONG(_net_pf, OID_AUTO, source_nodes_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
517     &VNET_NAME(pf_srchashsize), 0, "Size of pf(4) source nodes hashtable");
518 SYSCTL_ULONG(_net_pf, OID_AUTO, udpendpoint_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
519     &VNET_NAME(pf_udpendpointhashsize), 0, "Size of pf(4) endpoint hashtable");
520 SYSCTL_ULONG(_net_pf, OID_AUTO, request_maxcount, CTLFLAG_RWTUN,
521     &pf_ioctl_maxcount, 0, "Maximum number of tables, addresses, ... in a single ioctl() call");
522 
523 VNET_DEFINE(void *, pf_swi_cookie);
524 VNET_DEFINE(struct intr_event *, pf_swi_ie);
525 
526 VNET_DEFINE(uint32_t, pf_hashseed);
527 #define	V_pf_hashseed	VNET(pf_hashseed)
528 
529 static void
530 pf_sctp_checksum(struct mbuf *m, int off)
531 {
532 	uint32_t sum = 0;
533 
534 	/* Zero out the checksum, to enable recalculation. */
535 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
536 	    sizeof(sum), (caddr_t)&sum);
537 
538 	sum = sctp_calculate_cksum(m, off);
539 
540 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
541 	    sizeof(sum), (caddr_t)&sum);
542 }
543 
544 int
545 pf_addr_cmp(struct pf_addr *a, struct pf_addr *b, sa_family_t af)
546 {
547 
548 	switch (af) {
549 #ifdef INET
550 	case AF_INET:
551 		if (a->addr32[0] > b->addr32[0])
552 			return (1);
553 		if (a->addr32[0] < b->addr32[0])
554 			return (-1);
555 		break;
556 #endif /* INET */
557 #ifdef INET6
558 	case AF_INET6:
559 		if (a->addr32[3] > b->addr32[3])
560 			return (1);
561 		if (a->addr32[3] < b->addr32[3])
562 			return (-1);
563 		if (a->addr32[2] > b->addr32[2])
564 			return (1);
565 		if (a->addr32[2] < b->addr32[2])
566 			return (-1);
567 		if (a->addr32[1] > b->addr32[1])
568 			return (1);
569 		if (a->addr32[1] < b->addr32[1])
570 			return (-1);
571 		if (a->addr32[0] > b->addr32[0])
572 			return (1);
573 		if (a->addr32[0] < b->addr32[0])
574 			return (-1);
575 		break;
576 #endif /* INET6 */
577 	default:
578 		panic("%s: unknown address family %u", __func__, af);
579 	}
580 	return (0);
581 }
582 
583 static bool
584 pf_is_loopback(sa_family_t af, struct pf_addr *addr)
585 {
586 	switch (af) {
587 	case AF_INET:
588 		return IN_LOOPBACK(ntohl(addr->v4.s_addr));
589 	case AF_INET6:
590 		return IN6_IS_ADDR_LOOPBACK(&addr->v6);
591 	default:
592 		panic("Unknown af %d", af);
593 	}
594 }
595 
596 static void
597 pf_packet_rework_nat(struct mbuf *m, struct pf_pdesc *pd, int off,
598 	struct pf_state_key *nk)
599 {
600 
601 	switch (pd->proto) {
602 	case IPPROTO_TCP: {
603 		struct tcphdr *th = &pd->hdr.tcp;
604 
605 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
606 			pf_change_ap(m, pd->src, &th->th_sport, pd->ip_sum,
607 			    &th->th_sum, &nk->addr[pd->sidx],
608 			    nk->port[pd->sidx], 0, pd->af);
609 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
610 			pf_change_ap(m, pd->dst, &th->th_dport, pd->ip_sum,
611 			    &th->th_sum, &nk->addr[pd->didx],
612 			    nk->port[pd->didx], 0, pd->af);
613 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
614 		break;
615 	}
616 	case IPPROTO_UDP: {
617 		struct udphdr *uh = &pd->hdr.udp;
618 
619 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
620 			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
621 			    &uh->uh_sum, &nk->addr[pd->sidx],
622 			    nk->port[pd->sidx], 1, pd->af);
623 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
624 			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
625 			    &uh->uh_sum, &nk->addr[pd->didx],
626 			    nk->port[pd->didx], 1, pd->af);
627 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
628 		break;
629 	}
630 	case IPPROTO_SCTP: {
631 		struct sctphdr *sh = &pd->hdr.sctp;
632 		uint16_t checksum = 0;
633 
634 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
635 			pf_change_ap(m, pd->src, &sh->src_port, pd->ip_sum,
636 			    &checksum, &nk->addr[pd->sidx],
637 			    nk->port[pd->sidx], 1, pd->af);
638 		}
639 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
640 			pf_change_ap(m, pd->dst, &sh->dest_port, pd->ip_sum,
641 			    &checksum, &nk->addr[pd->didx],
642 			    nk->port[pd->didx], 1, pd->af);
643 		}
644 
645 		break;
646 	}
647 	case IPPROTO_ICMP: {
648 		struct icmp *ih = &pd->hdr.icmp;
649 
650 		if (nk->port[pd->sidx] != ih->icmp_id) {
651 			pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
652 			    ih->icmp_cksum, ih->icmp_id,
653 			    nk->port[pd->sidx], 0);
654 			ih->icmp_id = nk->port[pd->sidx];
655 			pd->sport = &ih->icmp_id;
656 
657 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)ih);
658 		}
659 		/* FALLTHROUGH */
660 	}
661 	default:
662 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
663 			switch (pd->af) {
664 			case AF_INET:
665 				pf_change_a(&pd->src->v4.s_addr,
666 				    pd->ip_sum, nk->addr[pd->sidx].v4.s_addr,
667 				    0);
668 				break;
669 			case AF_INET6:
670 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
671 				break;
672 			}
673 		}
674 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
675 			switch (pd->af) {
676 			case AF_INET:
677 				pf_change_a(&pd->dst->v4.s_addr,
678 				    pd->ip_sum, nk->addr[pd->didx].v4.s_addr,
679 				    0);
680 				break;
681 			case AF_INET6:
682 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
683 				break;
684 			}
685 		}
686 		break;
687 	}
688 }
689 
690 static __inline uint32_t
691 pf_hashkey(const struct pf_state_key *sk)
692 {
693 	uint32_t h;
694 
695 	h = murmur3_32_hash32((const uint32_t *)sk,
696 	    sizeof(struct pf_state_key_cmp)/sizeof(uint32_t),
697 	    V_pf_hashseed);
698 
699 	return (h & V_pf_hashmask);
700 }
701 
702 static __inline uint32_t
703 pf_hashsrc(struct pf_addr *addr, sa_family_t af)
704 {
705 	uint32_t h;
706 
707 	switch (af) {
708 	case AF_INET:
709 		h = murmur3_32_hash32((uint32_t *)&addr->v4,
710 		    sizeof(addr->v4)/sizeof(uint32_t), V_pf_hashseed);
711 		break;
712 	case AF_INET6:
713 		h = murmur3_32_hash32((uint32_t *)&addr->v6,
714 		    sizeof(addr->v6)/sizeof(uint32_t), V_pf_hashseed);
715 		break;
716 	default:
717 		panic("%s: unknown address family %u", __func__, af);
718 	}
719 
720 	return (h & V_pf_srchashmask);
721 }
722 
723 static inline uint32_t
724 pf_hashudpendpoint(struct pf_udp_endpoint *endpoint)
725 {
726 	uint32_t h;
727 
728 	h = murmur3_32_hash32((uint32_t *)endpoint,
729 	    sizeof(struct pf_udp_endpoint_cmp)/sizeof(uint32_t),
730 	    V_pf_hashseed);
731 	return (h & V_pf_udpendpointhashmask);
732 }
733 
734 #ifdef ALTQ
735 static int
736 pf_state_hash(struct pf_kstate *s)
737 {
738 	u_int32_t hv = (intptr_t)s / sizeof(*s);
739 
740 	hv ^= crc32(&s->src, sizeof(s->src));
741 	hv ^= crc32(&s->dst, sizeof(s->dst));
742 	if (hv == 0)
743 		hv = 1;
744 	return (hv);
745 }
746 #endif
747 
748 static __inline void
749 pf_set_protostate(struct pf_kstate *s, int which, u_int8_t newstate)
750 {
751 	if (which == PF_PEER_DST || which == PF_PEER_BOTH)
752 		s->dst.state = newstate;
753 	if (which == PF_PEER_DST)
754 		return;
755 	if (s->src.state == newstate)
756 		return;
757 	if (s->creatorid == V_pf_status.hostid &&
758 	    s->key[PF_SK_STACK] != NULL &&
759 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP &&
760 	    !(TCPS_HAVEESTABLISHED(s->src.state) ||
761 	    s->src.state == TCPS_CLOSED) &&
762 	    (TCPS_HAVEESTABLISHED(newstate) || newstate == TCPS_CLOSED))
763 		atomic_add_32(&V_pf_status.states_halfopen, -1);
764 
765 	s->src.state = newstate;
766 }
767 
768 #ifdef INET6
769 void
770 pf_addrcpy(struct pf_addr *dst, struct pf_addr *src, sa_family_t af)
771 {
772 	switch (af) {
773 #ifdef INET
774 	case AF_INET:
775 		memcpy(&dst->v4, &src->v4, sizeof(dst->v4));
776 		break;
777 #endif /* INET */
778 	case AF_INET6:
779 		memcpy(&dst->v6, &src->v6, sizeof(dst->v6));
780 		break;
781 	}
782 }
783 #endif /* INET6 */
784 
785 static void
786 pf_init_threshold(struct pf_threshold *threshold,
787     u_int32_t limit, u_int32_t seconds)
788 {
789 	threshold->limit = limit * PF_THRESHOLD_MULT;
790 	threshold->seconds = seconds;
791 	threshold->count = 0;
792 	threshold->last = time_uptime;
793 }
794 
795 static void
796 pf_add_threshold(struct pf_threshold *threshold)
797 {
798 	u_int32_t t = time_uptime, diff = t - threshold->last;
799 
800 	if (diff >= threshold->seconds)
801 		threshold->count = 0;
802 	else
803 		threshold->count -= threshold->count * diff /
804 		    threshold->seconds;
805 	threshold->count += PF_THRESHOLD_MULT;
806 	threshold->last = t;
807 }
808 
809 static int
810 pf_check_threshold(struct pf_threshold *threshold)
811 {
812 	return (threshold->count > threshold->limit);
813 }
814 
815 static int
816 pf_src_connlimit(struct pf_kstate **state)
817 {
818 	struct pf_overload_entry *pfoe;
819 	int bad = 0;
820 
821 	PF_STATE_LOCK_ASSERT(*state);
822 	/*
823 	 * XXXKS: The src node is accessed unlocked!
824 	 * PF_SRC_NODE_LOCK_ASSERT((*state)->src_node);
825 	 */
826 
827 	(*state)->src_node->conn++;
828 	(*state)->src.tcp_est = 1;
829 	pf_add_threshold(&(*state)->src_node->conn_rate);
830 
831 	if ((*state)->rule.ptr->max_src_conn &&
832 	    (*state)->rule.ptr->max_src_conn <
833 	    (*state)->src_node->conn) {
834 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONN], 1);
835 		bad++;
836 	}
837 
838 	if ((*state)->rule.ptr->max_src_conn_rate.limit &&
839 	    pf_check_threshold(&(*state)->src_node->conn_rate)) {
840 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONNRATE], 1);
841 		bad++;
842 	}
843 
844 	if (!bad)
845 		return (0);
846 
847 	/* Kill this state. */
848 	(*state)->timeout = PFTM_PURGE;
849 	pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
850 
851 	if ((*state)->rule.ptr->overload_tbl == NULL)
852 		return (1);
853 
854 	/* Schedule overloading and flushing task. */
855 	pfoe = malloc(sizeof(*pfoe), M_PFTEMP, M_NOWAIT);
856 	if (pfoe == NULL)
857 		return (1);	/* too bad :( */
858 
859 	bcopy(&(*state)->src_node->addr, &pfoe->addr, sizeof(pfoe->addr));
860 	pfoe->af = (*state)->key[PF_SK_WIRE]->af;
861 	pfoe->rule = (*state)->rule.ptr;
862 	pfoe->dir = (*state)->direction;
863 	PF_OVERLOADQ_LOCK();
864 	SLIST_INSERT_HEAD(&V_pf_overloadqueue, pfoe, next);
865 	PF_OVERLOADQ_UNLOCK();
866 	taskqueue_enqueue(taskqueue_swi, &V_pf_overloadtask);
867 
868 	return (1);
869 }
870 
871 static void
872 pf_overload_task(void *v, int pending)
873 {
874 	struct pf_overload_head queue;
875 	struct pfr_addr p;
876 	struct pf_overload_entry *pfoe, *pfoe1;
877 	uint32_t killed = 0;
878 
879 	CURVNET_SET((struct vnet *)v);
880 
881 	PF_OVERLOADQ_LOCK();
882 	queue = V_pf_overloadqueue;
883 	SLIST_INIT(&V_pf_overloadqueue);
884 	PF_OVERLOADQ_UNLOCK();
885 
886 	bzero(&p, sizeof(p));
887 	SLIST_FOREACH(pfoe, &queue, next) {
888 		counter_u64_add(V_pf_status.lcounters[LCNT_OVERLOAD_TABLE], 1);
889 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
890 			printf("%s: blocking address ", __func__);
891 			pf_print_host(&pfoe->addr, 0, pfoe->af);
892 			printf("\n");
893 		}
894 
895 		p.pfra_af = pfoe->af;
896 		switch (pfoe->af) {
897 #ifdef INET
898 		case AF_INET:
899 			p.pfra_net = 32;
900 			p.pfra_ip4addr = pfoe->addr.v4;
901 			break;
902 #endif
903 #ifdef INET6
904 		case AF_INET6:
905 			p.pfra_net = 128;
906 			p.pfra_ip6addr = pfoe->addr.v6;
907 			break;
908 #endif
909 		}
910 
911 		PF_RULES_WLOCK();
912 		pfr_insert_kentry(pfoe->rule->overload_tbl, &p, time_second);
913 		PF_RULES_WUNLOCK();
914 	}
915 
916 	/*
917 	 * Remove those entries, that don't need flushing.
918 	 */
919 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
920 		if (pfoe->rule->flush == 0) {
921 			SLIST_REMOVE(&queue, pfoe, pf_overload_entry, next);
922 			free(pfoe, M_PFTEMP);
923 		} else
924 			counter_u64_add(
925 			    V_pf_status.lcounters[LCNT_OVERLOAD_FLUSH], 1);
926 
927 	/* If nothing to flush, return. */
928 	if (SLIST_EMPTY(&queue)) {
929 		CURVNET_RESTORE();
930 		return;
931 	}
932 
933 	for (int i = 0; i <= V_pf_hashmask; i++) {
934 		struct pf_idhash *ih = &V_pf_idhash[i];
935 		struct pf_state_key *sk;
936 		struct pf_kstate *s;
937 
938 		PF_HASHROW_LOCK(ih);
939 		LIST_FOREACH(s, &ih->states, entry) {
940 		    sk = s->key[PF_SK_WIRE];
941 		    SLIST_FOREACH(pfoe, &queue, next)
942 			if (sk->af == pfoe->af &&
943 			    ((pfoe->rule->flush & PF_FLUSH_GLOBAL) ||
944 			    pfoe->rule == s->rule.ptr) &&
945 			    ((pfoe->dir == PF_OUT &&
946 			    PF_AEQ(&pfoe->addr, &sk->addr[1], sk->af)) ||
947 			    (pfoe->dir == PF_IN &&
948 			    PF_AEQ(&pfoe->addr, &sk->addr[0], sk->af)))) {
949 				s->timeout = PFTM_PURGE;
950 				pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
951 				killed++;
952 			}
953 		}
954 		PF_HASHROW_UNLOCK(ih);
955 	}
956 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
957 		free(pfoe, M_PFTEMP);
958 	if (V_pf_status.debug >= PF_DEBUG_MISC)
959 		printf("%s: %u states killed", __func__, killed);
960 
961 	CURVNET_RESTORE();
962 }
963 
964 /*
965  * Can return locked on failure, so that we can consistently
966  * allocate and insert a new one.
967  */
968 struct pf_ksrc_node *
969 pf_find_src_node(struct pf_addr *src, struct pf_krule *rule, sa_family_t af,
970 	struct pf_srchash **sh, bool returnlocked)
971 {
972 	struct pf_ksrc_node *n;
973 
974 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_SEARCH], 1);
975 
976 	*sh = &V_pf_srchash[pf_hashsrc(src, af)];
977 	PF_HASHROW_LOCK(*sh);
978 	LIST_FOREACH(n, &(*sh)->nodes, entry)
979 		if (n->rule.ptr == rule && n->af == af &&
980 		    ((af == AF_INET && n->addr.v4.s_addr == src->v4.s_addr) ||
981 		    (af == AF_INET6 && bcmp(&n->addr, src, sizeof(*src)) == 0)))
982 			break;
983 
984 	if (n != NULL) {
985 		n->states++;
986 		PF_HASHROW_UNLOCK(*sh);
987 	} else if (returnlocked == false)
988 		PF_HASHROW_UNLOCK(*sh);
989 
990 	return (n);
991 }
992 
993 static void
994 pf_free_src_node(struct pf_ksrc_node *sn)
995 {
996 
997 	for (int i = 0; i < 2; i++) {
998 		counter_u64_free(sn->bytes[i]);
999 		counter_u64_free(sn->packets[i]);
1000 	}
1001 	uma_zfree(V_pf_sources_z, sn);
1002 }
1003 
1004 static u_short
1005 pf_insert_src_node(struct pf_ksrc_node **sn, struct pf_krule *rule,
1006     struct pf_addr *src, sa_family_t af)
1007 {
1008 	u_short			 reason = 0;
1009 	struct pf_srchash	*sh = NULL;
1010 
1011 	KASSERT((rule->rule_flag & PFRULE_SRCTRACK ||
1012 	    rule->rpool.opts & PF_POOL_STICKYADDR),
1013 	    ("%s for non-tracking rule %p", __func__, rule));
1014 
1015 	if (*sn == NULL)
1016 		*sn = pf_find_src_node(src, rule, af, &sh, true);
1017 
1018 	if (*sn == NULL) {
1019 		PF_HASHROW_ASSERT(sh);
1020 
1021 		if (rule->max_src_nodes &&
1022 		    counter_u64_fetch(rule->src_nodes) >= rule->max_src_nodes) {
1023 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCNODES], 1);
1024 			PF_HASHROW_UNLOCK(sh);
1025 			reason = PFRES_SRCLIMIT;
1026 			goto done;
1027 		}
1028 
1029 		(*sn) = uma_zalloc(V_pf_sources_z, M_NOWAIT | M_ZERO);
1030 		if ((*sn) == NULL) {
1031 			PF_HASHROW_UNLOCK(sh);
1032 			reason = PFRES_MEMORY;
1033 			goto done;
1034 		}
1035 
1036 		for (int i = 0; i < 2; i++) {
1037 			(*sn)->bytes[i] = counter_u64_alloc(M_NOWAIT);
1038 			(*sn)->packets[i] = counter_u64_alloc(M_NOWAIT);
1039 
1040 			if ((*sn)->bytes[i] == NULL || (*sn)->packets[i] == NULL) {
1041 				pf_free_src_node(*sn);
1042 				PF_HASHROW_UNLOCK(sh);
1043 				reason = PFRES_MEMORY;
1044 				goto done;
1045 			}
1046 		}
1047 
1048 		pf_init_threshold(&(*sn)->conn_rate,
1049 		    rule->max_src_conn_rate.limit,
1050 		    rule->max_src_conn_rate.seconds);
1051 
1052 		MPASS((*sn)->lock == NULL);
1053 		(*sn)->lock = &sh->lock;
1054 
1055 		(*sn)->af = af;
1056 		(*sn)->rule.ptr = rule;
1057 		PF_ACPY(&(*sn)->addr, src, af);
1058 		LIST_INSERT_HEAD(&sh->nodes, *sn, entry);
1059 		(*sn)->creation = time_uptime;
1060 		(*sn)->ruletype = rule->action;
1061 		(*sn)->states = 1;
1062 		if ((*sn)->rule.ptr != NULL)
1063 			counter_u64_add((*sn)->rule.ptr->src_nodes, 1);
1064 		PF_HASHROW_UNLOCK(sh);
1065 		counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_INSERT], 1);
1066 	} else {
1067 		if (rule->max_src_states &&
1068 		    (*sn)->states >= rule->max_src_states) {
1069 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCSTATES],
1070 			    1);
1071 			reason = PFRES_SRCLIMIT;
1072 			goto done;
1073 		}
1074 	}
1075 done:
1076 	return (reason);
1077 }
1078 
1079 void
1080 pf_unlink_src_node(struct pf_ksrc_node *src)
1081 {
1082 	PF_SRC_NODE_LOCK_ASSERT(src);
1083 
1084 	LIST_REMOVE(src, entry);
1085 	if (src->rule.ptr)
1086 		counter_u64_add(src->rule.ptr->src_nodes, -1);
1087 }
1088 
1089 u_int
1090 pf_free_src_nodes(struct pf_ksrc_node_list *head)
1091 {
1092 	struct pf_ksrc_node *sn, *tmp;
1093 	u_int count = 0;
1094 
1095 	LIST_FOREACH_SAFE(sn, head, entry, tmp) {
1096 		pf_free_src_node(sn);
1097 		count++;
1098 	}
1099 
1100 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], count);
1101 
1102 	return (count);
1103 }
1104 
1105 void
1106 pf_mtag_initialize(void)
1107 {
1108 
1109 	pf_mtag_z = uma_zcreate("pf mtags", sizeof(struct m_tag) +
1110 	    sizeof(struct pf_mtag), NULL, NULL, pf_mtag_uminit, NULL,
1111 	    UMA_ALIGN_PTR, 0);
1112 }
1113 
1114 /* Per-vnet data storage structures initialization. */
1115 void
1116 pf_initialize(void)
1117 {
1118 	struct pf_keyhash	*kh;
1119 	struct pf_idhash	*ih;
1120 	struct pf_srchash	*sh;
1121 	struct pf_udpendpointhash	*uh;
1122 	u_int i;
1123 
1124 	if (V_pf_hashsize == 0 || !powerof2(V_pf_hashsize))
1125 		V_pf_hashsize = PF_HASHSIZ;
1126 	if (V_pf_srchashsize == 0 || !powerof2(V_pf_srchashsize))
1127 		V_pf_srchashsize = PF_SRCHASHSIZ;
1128 	if (V_pf_udpendpointhashsize == 0 || !powerof2(V_pf_udpendpointhashsize))
1129 		V_pf_udpendpointhashsize = PF_UDPENDHASHSIZ;
1130 
1131 	V_pf_hashseed = arc4random();
1132 
1133 	/* States and state keys storage. */
1134 	V_pf_state_z = uma_zcreate("pf states", sizeof(struct pf_kstate),
1135 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1136 	V_pf_limits[PF_LIMIT_STATES].zone = V_pf_state_z;
1137 	uma_zone_set_max(V_pf_state_z, PFSTATE_HIWAT);
1138 	uma_zone_set_warning(V_pf_state_z, "PF states limit reached");
1139 
1140 	V_pf_state_key_z = uma_zcreate("pf state keys",
1141 	    sizeof(struct pf_state_key), pf_state_key_ctor, NULL, NULL, NULL,
1142 	    UMA_ALIGN_PTR, 0);
1143 
1144 	V_pf_keyhash = mallocarray(V_pf_hashsize, sizeof(struct pf_keyhash),
1145 	    M_PFHASH, M_NOWAIT | M_ZERO);
1146 	V_pf_idhash = mallocarray(V_pf_hashsize, sizeof(struct pf_idhash),
1147 	    M_PFHASH, M_NOWAIT | M_ZERO);
1148 	if (V_pf_keyhash == NULL || V_pf_idhash == NULL) {
1149 		printf("pf: Unable to allocate memory for "
1150 		    "state_hashsize %lu.\n", V_pf_hashsize);
1151 
1152 		free(V_pf_keyhash, M_PFHASH);
1153 		free(V_pf_idhash, M_PFHASH);
1154 
1155 		V_pf_hashsize = PF_HASHSIZ;
1156 		V_pf_keyhash = mallocarray(V_pf_hashsize,
1157 		    sizeof(struct pf_keyhash), M_PFHASH, M_WAITOK | M_ZERO);
1158 		V_pf_idhash = mallocarray(V_pf_hashsize,
1159 		    sizeof(struct pf_idhash), M_PFHASH, M_WAITOK | M_ZERO);
1160 	}
1161 
1162 	V_pf_hashmask = V_pf_hashsize - 1;
1163 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= V_pf_hashmask;
1164 	    i++, kh++, ih++) {
1165 		mtx_init(&kh->lock, "pf_keyhash", NULL, MTX_DEF | MTX_DUPOK);
1166 		mtx_init(&ih->lock, "pf_idhash", NULL, MTX_DEF);
1167 	}
1168 
1169 	/* Source nodes. */
1170 	V_pf_sources_z = uma_zcreate("pf source nodes",
1171 	    sizeof(struct pf_ksrc_node), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
1172 	    0);
1173 	V_pf_limits[PF_LIMIT_SRC_NODES].zone = V_pf_sources_z;
1174 	uma_zone_set_max(V_pf_sources_z, PFSNODE_HIWAT);
1175 	uma_zone_set_warning(V_pf_sources_z, "PF source nodes limit reached");
1176 
1177 	V_pf_srchash = mallocarray(V_pf_srchashsize,
1178 	    sizeof(struct pf_srchash), M_PFHASH, M_NOWAIT | M_ZERO);
1179 	if (V_pf_srchash == NULL) {
1180 		printf("pf: Unable to allocate memory for "
1181 		    "source_hashsize %lu.\n", V_pf_srchashsize);
1182 
1183 		V_pf_srchashsize = PF_SRCHASHSIZ;
1184 		V_pf_srchash = mallocarray(V_pf_srchashsize,
1185 		    sizeof(struct pf_srchash), M_PFHASH, M_WAITOK | M_ZERO);
1186 	}
1187 
1188 	V_pf_srchashmask = V_pf_srchashsize - 1;
1189 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++)
1190 		mtx_init(&sh->lock, "pf_srchash", NULL, MTX_DEF);
1191 
1192 
1193 	/* UDP endpoint mappings. */
1194 	V_pf_udp_mapping_z = uma_zcreate("pf UDP mappings",
1195 	    sizeof(struct pf_udp_mapping), NULL, NULL, NULL, NULL,
1196 	    UMA_ALIGN_PTR, 0);
1197 	V_pf_udpendpointhash = mallocarray(V_pf_udpendpointhashsize,
1198 	    sizeof(struct pf_udpendpointhash), M_PFHASH, M_NOWAIT | M_ZERO);
1199 	if (V_pf_udpendpointhash == NULL) {
1200 		printf("pf: Unable to allocate memory for "
1201 		    "udpendpoint_hashsize %lu.\n", V_pf_udpendpointhashsize);
1202 
1203 		V_pf_udpendpointhashsize = PF_UDPENDHASHSIZ;
1204 		V_pf_udpendpointhash = mallocarray(V_pf_udpendpointhashsize,
1205 		    sizeof(struct pf_udpendpointhash), M_PFHASH, M_WAITOK | M_ZERO);
1206 	}
1207 
1208 	V_pf_udpendpointhashmask = V_pf_udpendpointhashsize - 1;
1209 	for (i = 0, uh = V_pf_udpendpointhash;
1210 	    i <= V_pf_udpendpointhashmask;
1211 	    i++, uh++) {
1212 		mtx_init(&uh->lock, "pf_udpendpointhash", NULL,
1213 		    MTX_DEF | MTX_DUPOK);
1214 	}
1215 
1216 	/* ALTQ */
1217 	TAILQ_INIT(&V_pf_altqs[0]);
1218 	TAILQ_INIT(&V_pf_altqs[1]);
1219 	TAILQ_INIT(&V_pf_altqs[2]);
1220 	TAILQ_INIT(&V_pf_altqs[3]);
1221 	TAILQ_INIT(&V_pf_pabuf);
1222 	V_pf_altqs_active = &V_pf_altqs[0];
1223 	V_pf_altq_ifs_active = &V_pf_altqs[1];
1224 	V_pf_altqs_inactive = &V_pf_altqs[2];
1225 	V_pf_altq_ifs_inactive = &V_pf_altqs[3];
1226 
1227 	/* Send & overload+flush queues. */
1228 	STAILQ_INIT(&V_pf_sendqueue);
1229 	SLIST_INIT(&V_pf_overloadqueue);
1230 	TASK_INIT(&V_pf_overloadtask, 0, pf_overload_task, curvnet);
1231 
1232 	/* Unlinked, but may be referenced rules. */
1233 	TAILQ_INIT(&V_pf_unlinked_rules);
1234 }
1235 
1236 void
1237 pf_mtag_cleanup(void)
1238 {
1239 
1240 	uma_zdestroy(pf_mtag_z);
1241 }
1242 
1243 void
1244 pf_cleanup(void)
1245 {
1246 	struct pf_keyhash	*kh;
1247 	struct pf_idhash	*ih;
1248 	struct pf_srchash	*sh;
1249 	struct pf_udpendpointhash	*uh;
1250 	struct pf_send_entry	*pfse, *next;
1251 	u_int i;
1252 
1253 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash;
1254 	    i <= V_pf_hashmask;
1255 	    i++, kh++, ih++) {
1256 		KASSERT(LIST_EMPTY(&kh->keys), ("%s: key hash not empty",
1257 		    __func__));
1258 		KASSERT(LIST_EMPTY(&ih->states), ("%s: id hash not empty",
1259 		    __func__));
1260 		mtx_destroy(&kh->lock);
1261 		mtx_destroy(&ih->lock);
1262 	}
1263 	free(V_pf_keyhash, M_PFHASH);
1264 	free(V_pf_idhash, M_PFHASH);
1265 
1266 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
1267 		KASSERT(LIST_EMPTY(&sh->nodes),
1268 		    ("%s: source node hash not empty", __func__));
1269 		mtx_destroy(&sh->lock);
1270 	}
1271 	free(V_pf_srchash, M_PFHASH);
1272 
1273 	for (i = 0, uh = V_pf_udpendpointhash;
1274 	    i <= V_pf_udpendpointhashmask;
1275 	    i++, uh++) {
1276 		KASSERT(LIST_EMPTY(&uh->endpoints),
1277 		    ("%s: udp endpoint hash not empty", __func__));
1278 		mtx_destroy(&uh->lock);
1279 	}
1280 	free(V_pf_udpendpointhash, M_PFHASH);
1281 
1282 	STAILQ_FOREACH_SAFE(pfse, &V_pf_sendqueue, pfse_next, next) {
1283 		m_freem(pfse->pfse_m);
1284 		free(pfse, M_PFTEMP);
1285 	}
1286 	MPASS(RB_EMPTY(&V_pf_sctp_endpoints));
1287 
1288 	uma_zdestroy(V_pf_sources_z);
1289 	uma_zdestroy(V_pf_state_z);
1290 	uma_zdestroy(V_pf_state_key_z);
1291 	uma_zdestroy(V_pf_udp_mapping_z);
1292 }
1293 
1294 static int
1295 pf_mtag_uminit(void *mem, int size, int how)
1296 {
1297 	struct m_tag *t;
1298 
1299 	t = (struct m_tag *)mem;
1300 	t->m_tag_cookie = MTAG_ABI_COMPAT;
1301 	t->m_tag_id = PACKET_TAG_PF;
1302 	t->m_tag_len = sizeof(struct pf_mtag);
1303 	t->m_tag_free = pf_mtag_free;
1304 
1305 	return (0);
1306 }
1307 
1308 static void
1309 pf_mtag_free(struct m_tag *t)
1310 {
1311 
1312 	uma_zfree(pf_mtag_z, t);
1313 }
1314 
1315 struct pf_mtag *
1316 pf_get_mtag(struct mbuf *m)
1317 {
1318 	struct m_tag *mtag;
1319 
1320 	if ((mtag = m_tag_find(m, PACKET_TAG_PF, NULL)) != NULL)
1321 		return ((struct pf_mtag *)(mtag + 1));
1322 
1323 	mtag = uma_zalloc(pf_mtag_z, M_NOWAIT);
1324 	if (mtag == NULL)
1325 		return (NULL);
1326 	bzero(mtag + 1, sizeof(struct pf_mtag));
1327 	m_tag_prepend(m, mtag);
1328 
1329 	return ((struct pf_mtag *)(mtag + 1));
1330 }
1331 
1332 static int
1333 pf_state_key_attach(struct pf_state_key *skw, struct pf_state_key *sks,
1334     struct pf_kstate *s)
1335 {
1336 	struct pf_keyhash	*khs, *khw, *kh;
1337 	struct pf_state_key	*sk, *cur;
1338 	struct pf_kstate	*si, *olds = NULL;
1339 	int idx;
1340 
1341 	NET_EPOCH_ASSERT();
1342 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1343 	KASSERT(s->key[PF_SK_WIRE] == NULL, ("%s: state has key", __func__));
1344 	KASSERT(s->key[PF_SK_STACK] == NULL, ("%s: state has key", __func__));
1345 
1346 	/*
1347 	 * We need to lock hash slots of both keys. To avoid deadlock
1348 	 * we always lock the slot with lower address first. Unlock order
1349 	 * isn't important.
1350 	 *
1351 	 * We also need to lock ID hash slot before dropping key
1352 	 * locks. On success we return with ID hash slot locked.
1353 	 */
1354 
1355 	if (skw == sks) {
1356 		khs = khw = &V_pf_keyhash[pf_hashkey(skw)];
1357 		PF_HASHROW_LOCK(khs);
1358 	} else {
1359 		khs = &V_pf_keyhash[pf_hashkey(sks)];
1360 		khw = &V_pf_keyhash[pf_hashkey(skw)];
1361 		if (khs == khw) {
1362 			PF_HASHROW_LOCK(khs);
1363 		} else if (khs < khw) {
1364 			PF_HASHROW_LOCK(khs);
1365 			PF_HASHROW_LOCK(khw);
1366 		} else {
1367 			PF_HASHROW_LOCK(khw);
1368 			PF_HASHROW_LOCK(khs);
1369 		}
1370 	}
1371 
1372 #define	KEYS_UNLOCK()	do {			\
1373 	if (khs != khw) {			\
1374 		PF_HASHROW_UNLOCK(khs);		\
1375 		PF_HASHROW_UNLOCK(khw);		\
1376 	} else					\
1377 		PF_HASHROW_UNLOCK(khs);		\
1378 } while (0)
1379 
1380 	/*
1381 	 * First run: start with wire key.
1382 	 */
1383 	sk = skw;
1384 	kh = khw;
1385 	idx = PF_SK_WIRE;
1386 
1387 	MPASS(s->lock == NULL);
1388 	s->lock = &V_pf_idhash[PF_IDHASH(s)].lock;
1389 
1390 keyattach:
1391 	LIST_FOREACH(cur, &kh->keys, entry)
1392 		if (bcmp(cur, sk, sizeof(struct pf_state_key_cmp)) == 0)
1393 			break;
1394 
1395 	if (cur != NULL) {
1396 		/* Key exists. Check for same kif, if none, add to key. */
1397 		TAILQ_FOREACH(si, &cur->states[idx], key_list[idx]) {
1398 			struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(si)];
1399 
1400 			PF_HASHROW_LOCK(ih);
1401 			if (si->kif == s->kif &&
1402 			    si->direction == s->direction) {
1403 				if (sk->proto == IPPROTO_TCP &&
1404 				    si->src.state >= TCPS_FIN_WAIT_2 &&
1405 				    si->dst.state >= TCPS_FIN_WAIT_2) {
1406 					/*
1407 					 * New state matches an old >FIN_WAIT_2
1408 					 * state. We can't drop key hash locks,
1409 					 * thus we can't unlink it properly.
1410 					 *
1411 					 * As a workaround we drop it into
1412 					 * TCPS_CLOSED state, schedule purge
1413 					 * ASAP and push it into the very end
1414 					 * of the slot TAILQ, so that it won't
1415 					 * conflict with our new state.
1416 					 */
1417 					pf_set_protostate(si, PF_PEER_BOTH,
1418 					    TCPS_CLOSED);
1419 					si->timeout = PFTM_PURGE;
1420 					olds = si;
1421 				} else {
1422 					if (V_pf_status.debug >= PF_DEBUG_MISC) {
1423 						printf("pf: %s key attach "
1424 						    "failed on %s: ",
1425 						    (idx == PF_SK_WIRE) ?
1426 						    "wire" : "stack",
1427 						    s->kif->pfik_name);
1428 						pf_print_state_parts(s,
1429 						    (idx == PF_SK_WIRE) ?
1430 						    sk : NULL,
1431 						    (idx == PF_SK_STACK) ?
1432 						    sk : NULL);
1433 						printf(", existing: ");
1434 						pf_print_state_parts(si,
1435 						    (idx == PF_SK_WIRE) ?
1436 						    sk : NULL,
1437 						    (idx == PF_SK_STACK) ?
1438 						    sk : NULL);
1439 						printf("\n");
1440 					}
1441 					s->timeout = PFTM_UNLINKED;
1442 					PF_HASHROW_UNLOCK(ih);
1443 					KEYS_UNLOCK();
1444 					uma_zfree(V_pf_state_key_z, sk);
1445 					if (idx == PF_SK_STACK)
1446 						pf_detach_state(s);
1447 					return (EEXIST); /* collision! */
1448 				}
1449 			}
1450 			PF_HASHROW_UNLOCK(ih);
1451 		}
1452 		uma_zfree(V_pf_state_key_z, sk);
1453 		s->key[idx] = cur;
1454 	} else {
1455 		LIST_INSERT_HEAD(&kh->keys, sk, entry);
1456 		s->key[idx] = sk;
1457 	}
1458 
1459 stateattach:
1460 	/* List is sorted, if-bound states before floating. */
1461 	if (s->kif == V_pfi_all)
1462 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], s, key_list[idx]);
1463 	else
1464 		TAILQ_INSERT_HEAD(&s->key[idx]->states[idx], s, key_list[idx]);
1465 
1466 	if (olds) {
1467 		TAILQ_REMOVE(&s->key[idx]->states[idx], olds, key_list[idx]);
1468 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], olds,
1469 		    key_list[idx]);
1470 		olds = NULL;
1471 	}
1472 
1473 	/*
1474 	 * Attach done. See how should we (or should not?)
1475 	 * attach a second key.
1476 	 */
1477 	if (sks == skw) {
1478 		s->key[PF_SK_STACK] = s->key[PF_SK_WIRE];
1479 		idx = PF_SK_STACK;
1480 		sks = NULL;
1481 		goto stateattach;
1482 	} else if (sks != NULL) {
1483 		/*
1484 		 * Continue attaching with stack key.
1485 		 */
1486 		sk = sks;
1487 		kh = khs;
1488 		idx = PF_SK_STACK;
1489 		sks = NULL;
1490 		goto keyattach;
1491 	}
1492 
1493 	PF_STATE_LOCK(s);
1494 	KEYS_UNLOCK();
1495 
1496 	KASSERT(s->key[PF_SK_WIRE] != NULL && s->key[PF_SK_STACK] != NULL,
1497 	    ("%s failure", __func__));
1498 
1499 	return (0);
1500 #undef	KEYS_UNLOCK
1501 }
1502 
1503 static void
1504 pf_detach_state(struct pf_kstate *s)
1505 {
1506 	struct pf_state_key *sks = s->key[PF_SK_STACK];
1507 	struct pf_keyhash *kh;
1508 
1509 	NET_EPOCH_ASSERT();
1510 	MPASS(s->timeout >= PFTM_MAX);
1511 
1512 	pf_sctp_multihome_detach_addr(s);
1513 
1514 	if ((s->state_flags & PFSTATE_PFLOW) && V_pflow_export_state_ptr)
1515 		V_pflow_export_state_ptr(s);
1516 
1517 	if (sks != NULL) {
1518 		kh = &V_pf_keyhash[pf_hashkey(sks)];
1519 		PF_HASHROW_LOCK(kh);
1520 		if (s->key[PF_SK_STACK] != NULL)
1521 			pf_state_key_detach(s, PF_SK_STACK);
1522 		/*
1523 		 * If both point to same key, then we are done.
1524 		 */
1525 		if (sks == s->key[PF_SK_WIRE]) {
1526 			pf_state_key_detach(s, PF_SK_WIRE);
1527 			PF_HASHROW_UNLOCK(kh);
1528 			return;
1529 		}
1530 		PF_HASHROW_UNLOCK(kh);
1531 	}
1532 
1533 	if (s->key[PF_SK_WIRE] != NULL) {
1534 		kh = &V_pf_keyhash[pf_hashkey(s->key[PF_SK_WIRE])];
1535 		PF_HASHROW_LOCK(kh);
1536 		if (s->key[PF_SK_WIRE] != NULL)
1537 			pf_state_key_detach(s, PF_SK_WIRE);
1538 		PF_HASHROW_UNLOCK(kh);
1539 	}
1540 }
1541 
1542 static void
1543 pf_state_key_detach(struct pf_kstate *s, int idx)
1544 {
1545 	struct pf_state_key *sk = s->key[idx];
1546 #ifdef INVARIANTS
1547 	struct pf_keyhash *kh = &V_pf_keyhash[pf_hashkey(sk)];
1548 
1549 	PF_HASHROW_ASSERT(kh);
1550 #endif
1551 	TAILQ_REMOVE(&sk->states[idx], s, key_list[idx]);
1552 	s->key[idx] = NULL;
1553 
1554 	if (TAILQ_EMPTY(&sk->states[0]) && TAILQ_EMPTY(&sk->states[1])) {
1555 		LIST_REMOVE(sk, entry);
1556 		uma_zfree(V_pf_state_key_z, sk);
1557 	}
1558 }
1559 
1560 static int
1561 pf_state_key_ctor(void *mem, int size, void *arg, int flags)
1562 {
1563 	struct pf_state_key *sk = mem;
1564 
1565 	bzero(sk, sizeof(struct pf_state_key_cmp));
1566 	TAILQ_INIT(&sk->states[PF_SK_WIRE]);
1567 	TAILQ_INIT(&sk->states[PF_SK_STACK]);
1568 
1569 	return (0);
1570 }
1571 
1572 static int
1573 pf_state_key_addr_setup(struct pf_pdesc *pd, struct mbuf *m, int off,
1574     struct pf_state_key_cmp *key, int sidx, struct pf_addr *saddr,
1575     int didx, struct pf_addr *daddr, int multi)
1576 {
1577 #ifdef INET6
1578 	struct nd_neighbor_solicit nd;
1579 	struct pf_addr *target;
1580 	u_short action, reason;
1581 
1582 	if (pd->af == AF_INET || pd->proto != IPPROTO_ICMPV6)
1583 		goto copy;
1584 
1585 	switch (pd->hdr.icmp6.icmp6_type) {
1586 	case ND_NEIGHBOR_SOLICIT:
1587 		if (multi)
1588 			return (-1);
1589 		if (!pf_pull_hdr(m, off, &nd, sizeof(nd), &action, &reason, pd->af))
1590 			return (-1);
1591 		target = (struct pf_addr *)&nd.nd_ns_target;
1592 		daddr = target;
1593 		break;
1594 	case ND_NEIGHBOR_ADVERT:
1595 		if (multi)
1596 			return (-1);
1597 		if (!pf_pull_hdr(m, off, &nd, sizeof(nd), &action, &reason, pd->af))
1598 			return (-1);
1599 		target = (struct pf_addr *)&nd.nd_ns_target;
1600 		saddr = target;
1601 		if (IN6_IS_ADDR_MULTICAST(&pd->dst->v6)) {
1602 			key->addr[didx].addr32[0] = 0;
1603 			key->addr[didx].addr32[1] = 0;
1604 			key->addr[didx].addr32[2] = 0;
1605 			key->addr[didx].addr32[3] = 0;
1606 			daddr = NULL; /* overwritten */
1607 		}
1608 		break;
1609 	default:
1610 		if (multi == PF_ICMP_MULTI_LINK) {
1611 			key->addr[sidx].addr32[0] = IPV6_ADDR_INT32_MLL;
1612 			key->addr[sidx].addr32[1] = 0;
1613 			key->addr[sidx].addr32[2] = 0;
1614 			key->addr[sidx].addr32[3] = IPV6_ADDR_INT32_ONE;
1615 			saddr = NULL; /* overwritten */
1616 		}
1617 	}
1618 copy:
1619 #endif
1620 	if (saddr)
1621 		PF_ACPY(&key->addr[sidx], saddr, pd->af);
1622 	if (daddr)
1623 		PF_ACPY(&key->addr[didx], daddr, pd->af);
1624 
1625 	return (0);
1626 }
1627 
1628 struct pf_state_key *
1629 pf_state_key_setup(struct pf_pdesc *pd, struct mbuf *m, int off,
1630     struct pf_addr *saddr, struct pf_addr *daddr, u_int16_t sport,
1631     u_int16_t dport)
1632 {
1633 	struct pf_state_key *sk;
1634 
1635 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1636 	if (sk == NULL)
1637 		return (NULL);
1638 
1639 	if (pf_state_key_addr_setup(pd, m, off, (struct pf_state_key_cmp *)sk,
1640 	    pd->sidx, pd->src, pd->didx, pd->dst, 0)) {
1641 		uma_zfree(V_pf_state_key_z, sk);
1642 		return (NULL);
1643 	}
1644 
1645 	sk->port[pd->sidx] = sport;
1646 	sk->port[pd->didx] = dport;
1647 	sk->proto = pd->proto;
1648 	sk->af = pd->af;
1649 
1650 	return (sk);
1651 }
1652 
1653 struct pf_state_key *
1654 pf_state_key_clone(const struct pf_state_key *orig)
1655 {
1656 	struct pf_state_key *sk;
1657 
1658 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1659 	if (sk == NULL)
1660 		return (NULL);
1661 
1662 	bcopy(orig, sk, sizeof(struct pf_state_key_cmp));
1663 
1664 	return (sk);
1665 }
1666 
1667 int
1668 pf_state_insert(struct pfi_kkif *kif, struct pfi_kkif *orig_kif,
1669     struct pf_state_key *skw, struct pf_state_key *sks, struct pf_kstate *s)
1670 {
1671 	struct pf_idhash *ih;
1672 	struct pf_kstate *cur;
1673 	int error;
1674 
1675 	NET_EPOCH_ASSERT();
1676 
1677 	KASSERT(TAILQ_EMPTY(&sks->states[0]) && TAILQ_EMPTY(&sks->states[1]),
1678 	    ("%s: sks not pristine", __func__));
1679 	KASSERT(TAILQ_EMPTY(&skw->states[0]) && TAILQ_EMPTY(&skw->states[1]),
1680 	    ("%s: skw not pristine", __func__));
1681 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1682 
1683 	s->kif = kif;
1684 	s->orig_kif = orig_kif;
1685 
1686 	if (s->id == 0 && s->creatorid == 0) {
1687 		s->id = alloc_unr64(&V_pf_stateid);
1688 		s->id = htobe64(s->id);
1689 		s->creatorid = V_pf_status.hostid;
1690 	}
1691 
1692 	/* Returns with ID locked on success. */
1693 	if ((error = pf_state_key_attach(skw, sks, s)) != 0)
1694 		return (error);
1695 
1696 	ih = &V_pf_idhash[PF_IDHASH(s)];
1697 	PF_HASHROW_ASSERT(ih);
1698 	LIST_FOREACH(cur, &ih->states, entry)
1699 		if (cur->id == s->id && cur->creatorid == s->creatorid)
1700 			break;
1701 
1702 	if (cur != NULL) {
1703 		s->timeout = PFTM_UNLINKED;
1704 		PF_HASHROW_UNLOCK(ih);
1705 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
1706 			printf("pf: state ID collision: "
1707 			    "id: %016llx creatorid: %08x\n",
1708 			    (unsigned long long)be64toh(s->id),
1709 			    ntohl(s->creatorid));
1710 		}
1711 		pf_detach_state(s);
1712 		return (EEXIST);
1713 	}
1714 	LIST_INSERT_HEAD(&ih->states, s, entry);
1715 	/* One for keys, one for ID hash. */
1716 	refcount_init(&s->refs, 2);
1717 
1718 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_INSERT], 1);
1719 	if (V_pfsync_insert_state_ptr != NULL)
1720 		V_pfsync_insert_state_ptr(s);
1721 
1722 	/* Returns locked. */
1723 	return (0);
1724 }
1725 
1726 /*
1727  * Find state by ID: returns with locked row on success.
1728  */
1729 struct pf_kstate *
1730 pf_find_state_byid(uint64_t id, uint32_t creatorid)
1731 {
1732 	struct pf_idhash *ih;
1733 	struct pf_kstate *s;
1734 
1735 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1736 
1737 	ih = &V_pf_idhash[(be64toh(id) % (V_pf_hashmask + 1))];
1738 
1739 	PF_HASHROW_LOCK(ih);
1740 	LIST_FOREACH(s, &ih->states, entry)
1741 		if (s->id == id && s->creatorid == creatorid)
1742 			break;
1743 
1744 	if (s == NULL)
1745 		PF_HASHROW_UNLOCK(ih);
1746 
1747 	return (s);
1748 }
1749 
1750 /*
1751  * Find state by key.
1752  * Returns with ID hash slot locked on success.
1753  */
1754 static struct pf_kstate *
1755 pf_find_state(struct pfi_kkif *kif, const struct pf_state_key_cmp *key,
1756     u_int dir)
1757 {
1758 	struct pf_keyhash	*kh;
1759 	struct pf_state_key	*sk;
1760 	struct pf_kstate	*s;
1761 	int idx;
1762 
1763 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1764 
1765 	kh = &V_pf_keyhash[pf_hashkey((const struct pf_state_key *)key)];
1766 
1767 	PF_HASHROW_LOCK(kh);
1768 	LIST_FOREACH(sk, &kh->keys, entry)
1769 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1770 			break;
1771 	if (sk == NULL) {
1772 		PF_HASHROW_UNLOCK(kh);
1773 		return (NULL);
1774 	}
1775 
1776 	idx = (dir == PF_IN ? PF_SK_WIRE : PF_SK_STACK);
1777 
1778 	/* List is sorted, if-bound states before floating ones. */
1779 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx])
1780 		if (s->kif == V_pfi_all || s->kif == kif || s->orig_kif == kif) {
1781 			PF_STATE_LOCK(s);
1782 			PF_HASHROW_UNLOCK(kh);
1783 			if (__predict_false(s->timeout >= PFTM_MAX)) {
1784 				/*
1785 				 * State is either being processed by
1786 				 * pf_unlink_state() in an other thread, or
1787 				 * is scheduled for immediate expiry.
1788 				 */
1789 				PF_STATE_UNLOCK(s);
1790 				return (NULL);
1791 			}
1792 			return (s);
1793 		}
1794 	PF_HASHROW_UNLOCK(kh);
1795 
1796 	return (NULL);
1797 }
1798 
1799 /*
1800  * Returns with ID hash slot locked on success.
1801  */
1802 struct pf_kstate *
1803 pf_find_state_all(const struct pf_state_key_cmp *key, u_int dir, int *more)
1804 {
1805 	struct pf_keyhash	*kh;
1806 	struct pf_state_key	*sk;
1807 	struct pf_kstate	*s, *ret = NULL;
1808 	int			 idx, inout = 0;
1809 
1810 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1811 
1812 	kh = &V_pf_keyhash[pf_hashkey((const struct pf_state_key *)key)];
1813 
1814 	PF_HASHROW_LOCK(kh);
1815 	LIST_FOREACH(sk, &kh->keys, entry)
1816 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1817 			break;
1818 	if (sk == NULL) {
1819 		PF_HASHROW_UNLOCK(kh);
1820 		return (NULL);
1821 	}
1822 	switch (dir) {
1823 	case PF_IN:
1824 		idx = PF_SK_WIRE;
1825 		break;
1826 	case PF_OUT:
1827 		idx = PF_SK_STACK;
1828 		break;
1829 	case PF_INOUT:
1830 		idx = PF_SK_WIRE;
1831 		inout = 1;
1832 		break;
1833 	default:
1834 		panic("%s: dir %u", __func__, dir);
1835 	}
1836 second_run:
1837 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx]) {
1838 		if (more == NULL) {
1839 			PF_STATE_LOCK(s);
1840 			PF_HASHROW_UNLOCK(kh);
1841 			return (s);
1842 		}
1843 
1844 		if (ret)
1845 			(*more)++;
1846 		else {
1847 			ret = s;
1848 			PF_STATE_LOCK(s);
1849 		}
1850 	}
1851 	if (inout == 1) {
1852 		inout = 0;
1853 		idx = PF_SK_STACK;
1854 		goto second_run;
1855 	}
1856 	PF_HASHROW_UNLOCK(kh);
1857 
1858 	return (ret);
1859 }
1860 
1861 /*
1862  * FIXME
1863  * This routine is inefficient -- locks the state only to unlock immediately on
1864  * return.
1865  * It is racy -- after the state is unlocked nothing stops other threads from
1866  * removing it.
1867  */
1868 bool
1869 pf_find_state_all_exists(const struct pf_state_key_cmp *key, u_int dir)
1870 {
1871 	struct pf_kstate *s;
1872 
1873 	s = pf_find_state_all(key, dir, NULL);
1874 	if (s != NULL) {
1875 		PF_STATE_UNLOCK(s);
1876 		return (true);
1877 	}
1878 	return (false);
1879 }
1880 
1881 struct pf_udp_mapping *
1882 pf_udp_mapping_create(sa_family_t af, struct pf_addr *src_addr, uint16_t src_port,
1883     struct pf_addr *nat_addr, uint16_t nat_port)
1884 {
1885 	struct pf_udp_mapping *mapping;
1886 
1887 	mapping = uma_zalloc(V_pf_udp_mapping_z, M_NOWAIT | M_ZERO);
1888 	if (mapping == NULL)
1889 		return (NULL);
1890 	PF_ACPY(&mapping->endpoints[0].addr, src_addr, af);
1891 	mapping->endpoints[0].port = src_port;
1892 	mapping->endpoints[0].af = af;
1893 	mapping->endpoints[0].mapping = mapping;
1894 	PF_ACPY(&mapping->endpoints[1].addr, nat_addr, af);
1895 	mapping->endpoints[1].port = nat_port;
1896 	mapping->endpoints[1].af = af;
1897 	mapping->endpoints[1].mapping = mapping;
1898 	refcount_init(&mapping->refs, 1);
1899 	return (mapping);
1900 }
1901 
1902 int
1903 pf_udp_mapping_insert(struct pf_udp_mapping *mapping)
1904 {
1905 	struct pf_udpendpointhash *h0, *h1;
1906 	struct pf_udp_endpoint *endpoint;
1907 	int ret = EEXIST;
1908 
1909 	h0 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[0])];
1910 	h1 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[1])];
1911 	if (h0 == h1) {
1912 		PF_HASHROW_LOCK(h0);
1913 	} else if (h0 < h1) {
1914 		PF_HASHROW_LOCK(h0);
1915 		PF_HASHROW_LOCK(h1);
1916 	} else {
1917 		PF_HASHROW_LOCK(h1);
1918 		PF_HASHROW_LOCK(h0);
1919 	}
1920 
1921 	LIST_FOREACH(endpoint, &h0->endpoints, entry) {
1922 		if (bcmp(endpoint, &mapping->endpoints[0],
1923 		    sizeof(struct pf_udp_endpoint_cmp)) == 0)
1924 			break;
1925 	}
1926 	if (endpoint != NULL)
1927 		goto cleanup;
1928 	LIST_FOREACH(endpoint, &h1->endpoints, entry) {
1929 		if (bcmp(endpoint, &mapping->endpoints[1],
1930 		    sizeof(struct pf_udp_endpoint_cmp)) == 0)
1931 			break;
1932 	}
1933 	if (endpoint != NULL)
1934 		goto cleanup;
1935 	LIST_INSERT_HEAD(&h0->endpoints, &mapping->endpoints[0], entry);
1936 	LIST_INSERT_HEAD(&h1->endpoints, &mapping->endpoints[1], entry);
1937 	ret = 0;
1938 
1939 cleanup:
1940 	if (h0 != h1) {
1941 		PF_HASHROW_UNLOCK(h0);
1942 		PF_HASHROW_UNLOCK(h1);
1943 	} else {
1944 		PF_HASHROW_UNLOCK(h0);
1945 	}
1946 	return (ret);
1947 }
1948 
1949 void
1950 pf_udp_mapping_release(struct pf_udp_mapping *mapping)
1951 {
1952 	/* refcount is synchronized on the source endpoint's row lock */
1953 	struct pf_udpendpointhash *h0, *h1;
1954 
1955 	if (mapping == NULL)
1956 		return;
1957 
1958 	h0 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[0])];
1959 	PF_HASHROW_LOCK(h0);
1960 	if (refcount_release(&mapping->refs)) {
1961 		LIST_REMOVE(&mapping->endpoints[0], entry);
1962 		PF_HASHROW_UNLOCK(h0);
1963 		h1 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[1])];
1964 		PF_HASHROW_LOCK(h1);
1965 		LIST_REMOVE(&mapping->endpoints[1], entry);
1966 		PF_HASHROW_UNLOCK(h1);
1967 
1968 		uma_zfree(V_pf_udp_mapping_z, mapping);
1969 	} else {
1970 			PF_HASHROW_UNLOCK(h0);
1971 	}
1972 }
1973 
1974 
1975 struct pf_udp_mapping *
1976 pf_udp_mapping_find(struct pf_udp_endpoint_cmp *key)
1977 {
1978 	struct pf_udpendpointhash *uh;
1979 	struct pf_udp_endpoint *endpoint;
1980 
1981 	uh = &V_pf_udpendpointhash[pf_hashudpendpoint((struct pf_udp_endpoint*)key)];
1982 
1983 	PF_HASHROW_LOCK(uh);
1984 	LIST_FOREACH(endpoint, &uh->endpoints, entry) {
1985 		if (bcmp(endpoint, key, sizeof(struct pf_udp_endpoint_cmp)) == 0 &&
1986 			bcmp(endpoint, &endpoint->mapping->endpoints[0],
1987 			    sizeof(struct pf_udp_endpoint_cmp)) == 0)
1988 			break;
1989 	}
1990 	if (endpoint == NULL) {
1991 		PF_HASHROW_UNLOCK(uh);
1992 		return (NULL);
1993 	}
1994 	refcount_acquire(&endpoint->mapping->refs);
1995 	PF_HASHROW_UNLOCK(uh);
1996 	return (endpoint->mapping);
1997 }
1998 /* END state table stuff */
1999 
2000 static void
2001 pf_send(struct pf_send_entry *pfse)
2002 {
2003 
2004 	PF_SENDQ_LOCK();
2005 	STAILQ_INSERT_TAIL(&V_pf_sendqueue, pfse, pfse_next);
2006 	PF_SENDQ_UNLOCK();
2007 	swi_sched(V_pf_swi_cookie, 0);
2008 }
2009 
2010 static bool
2011 pf_isforlocal(struct mbuf *m, int af)
2012 {
2013 	switch (af) {
2014 #ifdef INET
2015 	case AF_INET: {
2016 		struct ip *ip = mtod(m, struct ip *);
2017 
2018 		return (in_localip(ip->ip_dst));
2019 	}
2020 #endif
2021 #ifdef INET6
2022 	case AF_INET6: {
2023 		struct ip6_hdr *ip6;
2024 		struct in6_ifaddr *ia;
2025 		ip6 = mtod(m, struct ip6_hdr *);
2026 		ia = in6ifa_ifwithaddr(&ip6->ip6_dst, 0 /* XXX */, false);
2027 		if (ia == NULL)
2028 			return (false);
2029 		return (! (ia->ia6_flags & IN6_IFF_NOTREADY));
2030 	}
2031 #endif
2032 	default:
2033 		panic("Unsupported af %d", af);
2034 	}
2035 
2036 	return (false);
2037 }
2038 
2039 int
2040 pf_icmp_mapping(struct pf_pdesc *pd, u_int8_t type,
2041     int *icmp_dir, int *multi, u_int16_t *virtual_id, u_int16_t *virtual_type)
2042 {
2043 	/*
2044 	 * ICMP types marked with PF_OUT are typically responses to
2045 	 * PF_IN, and will match states in the opposite direction.
2046 	 * PF_IN ICMP types need to match a state with that type.
2047 	 */
2048 	*icmp_dir = PF_OUT;
2049 	*multi = PF_ICMP_MULTI_LINK;
2050 	/* Queries (and responses) */
2051 	switch (pd->af) {
2052 #ifdef INET
2053 	case AF_INET:
2054 		switch (type) {
2055 		case ICMP_ECHO:
2056 			*icmp_dir = PF_IN;
2057 		case ICMP_ECHOREPLY:
2058 			*virtual_type = ICMP_ECHO;
2059 			*virtual_id = pd->hdr.icmp.icmp_id;
2060 			break;
2061 
2062 		case ICMP_TSTAMP:
2063 			*icmp_dir = PF_IN;
2064 		case ICMP_TSTAMPREPLY:
2065 			*virtual_type = ICMP_TSTAMP;
2066 			*virtual_id = pd->hdr.icmp.icmp_id;
2067 			break;
2068 
2069 		case ICMP_IREQ:
2070 			*icmp_dir = PF_IN;
2071 		case ICMP_IREQREPLY:
2072 			*virtual_type = ICMP_IREQ;
2073 			*virtual_id = pd->hdr.icmp.icmp_id;
2074 			break;
2075 
2076 		case ICMP_MASKREQ:
2077 			*icmp_dir = PF_IN;
2078 		case ICMP_MASKREPLY:
2079 			*virtual_type = ICMP_MASKREQ;
2080 			*virtual_id = pd->hdr.icmp.icmp_id;
2081 			break;
2082 
2083 		case ICMP_IPV6_WHEREAREYOU:
2084 			*icmp_dir = PF_IN;
2085 		case ICMP_IPV6_IAMHERE:
2086 			*virtual_type = ICMP_IPV6_WHEREAREYOU;
2087 			*virtual_id = 0; /* Nothing sane to match on! */
2088 			break;
2089 
2090 		case ICMP_MOBILE_REGREQUEST:
2091 			*icmp_dir = PF_IN;
2092 		case ICMP_MOBILE_REGREPLY:
2093 			*virtual_type = ICMP_MOBILE_REGREQUEST;
2094 			*virtual_id = 0; /* Nothing sane to match on! */
2095 			break;
2096 
2097 		case ICMP_ROUTERSOLICIT:
2098 			*icmp_dir = PF_IN;
2099 		case ICMP_ROUTERADVERT:
2100 			*virtual_type = ICMP_ROUTERSOLICIT;
2101 			*virtual_id = 0; /* Nothing sane to match on! */
2102 			break;
2103 
2104 		/* These ICMP types map to other connections */
2105 		case ICMP_UNREACH:
2106 		case ICMP_SOURCEQUENCH:
2107 		case ICMP_REDIRECT:
2108 		case ICMP_TIMXCEED:
2109 		case ICMP_PARAMPROB:
2110 			/* These will not be used, but set them anyway */
2111 			*icmp_dir = PF_IN;
2112 			*virtual_type = type;
2113 			*virtual_id = 0;
2114 			HTONS(*virtual_type);
2115 			return (1);  /* These types match to another state */
2116 
2117 		/*
2118 		 * All remaining ICMP types get their own states,
2119 		 * and will only match in one direction.
2120 		 */
2121 		default:
2122 			*icmp_dir = PF_IN;
2123 			*virtual_type = type;
2124 			*virtual_id = 0;
2125 			break;
2126 		}
2127 		break;
2128 #endif /* INET */
2129 #ifdef INET6
2130 	case AF_INET6:
2131 		switch (type) {
2132 		case ICMP6_ECHO_REQUEST:
2133 			*icmp_dir = PF_IN;
2134 		case ICMP6_ECHO_REPLY:
2135 			*virtual_type = ICMP6_ECHO_REQUEST;
2136 			*virtual_id = pd->hdr.icmp6.icmp6_id;
2137 			break;
2138 
2139 		case MLD_LISTENER_QUERY:
2140 		case MLD_LISTENER_REPORT: {
2141 			/*
2142 			 * Listener Report can be sent by clients
2143 			 * without an associated Listener Query.
2144 			 * In addition to that, when Report is sent as a
2145 			 * reply to a Query its source and destination
2146 			 * address are different.
2147 			 */
2148 			*icmp_dir = PF_IN;
2149 			*virtual_type = MLD_LISTENER_QUERY;
2150 			*virtual_id = 0;
2151 			break;
2152 		}
2153 		case MLD_MTRACE:
2154 			*icmp_dir = PF_IN;
2155 		case MLD_MTRACE_RESP:
2156 			*virtual_type = MLD_MTRACE;
2157 			*virtual_id = 0; /* Nothing sane to match on! */
2158 			break;
2159 
2160 		case ND_NEIGHBOR_SOLICIT:
2161 			*icmp_dir = PF_IN;
2162 		case ND_NEIGHBOR_ADVERT: {
2163 			*virtual_type = ND_NEIGHBOR_SOLICIT;
2164 			*virtual_id = 0;
2165 			break;
2166 		}
2167 
2168 		/*
2169 		 * These ICMP types map to other connections.
2170 		 * ND_REDIRECT can't be in this list because the triggering
2171 		 * packet header is optional.
2172 		 */
2173 		case ICMP6_DST_UNREACH:
2174 		case ICMP6_PACKET_TOO_BIG:
2175 		case ICMP6_TIME_EXCEEDED:
2176 		case ICMP6_PARAM_PROB:
2177 			/* These will not be used, but set them anyway */
2178 			*icmp_dir = PF_IN;
2179 			*virtual_type = type;
2180 			*virtual_id = 0;
2181 			HTONS(*virtual_type);
2182 			return (1);  /* These types match to another state */
2183 		/*
2184 		 * All remaining ICMP6 types get their own states,
2185 		 * and will only match in one direction.
2186 		 */
2187 		default:
2188 			*icmp_dir = PF_IN;
2189 			*virtual_type = type;
2190 			*virtual_id = 0;
2191 			break;
2192 		}
2193 		break;
2194 #endif /* INET6 */
2195 	default:
2196 		*icmp_dir = PF_IN;
2197 		*virtual_type = type;
2198 		*virtual_id = 0;
2199 		break;
2200 	}
2201 	HTONS(*virtual_type);
2202 	return (0);  /* These types match to their own state */
2203 }
2204 
2205 void
2206 pf_intr(void *v)
2207 {
2208 	struct epoch_tracker et;
2209 	struct pf_send_head queue;
2210 	struct pf_send_entry *pfse, *next;
2211 
2212 	CURVNET_SET((struct vnet *)v);
2213 
2214 	PF_SENDQ_LOCK();
2215 	queue = V_pf_sendqueue;
2216 	STAILQ_INIT(&V_pf_sendqueue);
2217 	PF_SENDQ_UNLOCK();
2218 
2219 	NET_EPOCH_ENTER(et);
2220 
2221 	STAILQ_FOREACH_SAFE(pfse, &queue, pfse_next, next) {
2222 		switch (pfse->pfse_type) {
2223 #ifdef INET
2224 		case PFSE_IP: {
2225 			if (pf_isforlocal(pfse->pfse_m, AF_INET)) {
2226 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
2227 				pfse->pfse_m->m_pkthdr.csum_flags |=
2228 				    CSUM_IP_VALID | CSUM_IP_CHECKED;
2229 				ip_input(pfse->pfse_m);
2230 			} else {
2231 				ip_output(pfse->pfse_m, NULL, NULL, 0, NULL,
2232 				    NULL);
2233 			}
2234 			break;
2235 		}
2236 		case PFSE_ICMP:
2237 			icmp_error(pfse->pfse_m, pfse->icmpopts.type,
2238 			    pfse->icmpopts.code, 0, pfse->icmpopts.mtu);
2239 			break;
2240 #endif /* INET */
2241 #ifdef INET6
2242 		case PFSE_IP6:
2243 			if (pf_isforlocal(pfse->pfse_m, AF_INET6)) {
2244 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
2245 				ip6_input(pfse->pfse_m);
2246 			} else {
2247 				ip6_output(pfse->pfse_m, NULL, NULL, 0, NULL,
2248 				    NULL, NULL);
2249 			}
2250 			break;
2251 		case PFSE_ICMP6:
2252 			icmp6_error(pfse->pfse_m, pfse->icmpopts.type,
2253 			    pfse->icmpopts.code, pfse->icmpopts.mtu);
2254 			break;
2255 #endif /* INET6 */
2256 		default:
2257 			panic("%s: unknown type", __func__);
2258 		}
2259 		free(pfse, M_PFTEMP);
2260 	}
2261 	NET_EPOCH_EXIT(et);
2262 	CURVNET_RESTORE();
2263 }
2264 
2265 #define	pf_purge_thread_period	(hz / 10)
2266 
2267 #ifdef PF_WANT_32_TO_64_COUNTER
2268 static void
2269 pf_status_counter_u64_periodic(void)
2270 {
2271 
2272 	PF_RULES_RASSERT();
2273 
2274 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 60)) != 0) {
2275 		return;
2276 	}
2277 
2278 	for (int i = 0; i < FCNT_MAX; i++) {
2279 		pf_counter_u64_periodic(&V_pf_status.fcounters[i]);
2280 	}
2281 }
2282 
2283 static void
2284 pf_kif_counter_u64_periodic(void)
2285 {
2286 	struct pfi_kkif *kif;
2287 	size_t r, run;
2288 
2289 	PF_RULES_RASSERT();
2290 
2291 	if (__predict_false(V_pf_allkifcount == 0)) {
2292 		return;
2293 	}
2294 
2295 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
2296 		return;
2297 	}
2298 
2299 	run = V_pf_allkifcount / 10;
2300 	if (run < 5)
2301 		run = 5;
2302 
2303 	for (r = 0; r < run; r++) {
2304 		kif = LIST_NEXT(V_pf_kifmarker, pfik_allkiflist);
2305 		if (kif == NULL) {
2306 			LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
2307 			LIST_INSERT_HEAD(&V_pf_allkiflist, V_pf_kifmarker, pfik_allkiflist);
2308 			break;
2309 		}
2310 
2311 		LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
2312 		LIST_INSERT_AFTER(kif, V_pf_kifmarker, pfik_allkiflist);
2313 
2314 		for (int i = 0; i < 2; i++) {
2315 			for (int j = 0; j < 2; j++) {
2316 				for (int k = 0; k < 2; k++) {
2317 					pf_counter_u64_periodic(&kif->pfik_packets[i][j][k]);
2318 					pf_counter_u64_periodic(&kif->pfik_bytes[i][j][k]);
2319 				}
2320 			}
2321 		}
2322 	}
2323 }
2324 
2325 static void
2326 pf_rule_counter_u64_periodic(void)
2327 {
2328 	struct pf_krule *rule;
2329 	size_t r, run;
2330 
2331 	PF_RULES_RASSERT();
2332 
2333 	if (__predict_false(V_pf_allrulecount == 0)) {
2334 		return;
2335 	}
2336 
2337 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
2338 		return;
2339 	}
2340 
2341 	run = V_pf_allrulecount / 10;
2342 	if (run < 5)
2343 		run = 5;
2344 
2345 	for (r = 0; r < run; r++) {
2346 		rule = LIST_NEXT(V_pf_rulemarker, allrulelist);
2347 		if (rule == NULL) {
2348 			LIST_REMOVE(V_pf_rulemarker, allrulelist);
2349 			LIST_INSERT_HEAD(&V_pf_allrulelist, V_pf_rulemarker, allrulelist);
2350 			break;
2351 		}
2352 
2353 		LIST_REMOVE(V_pf_rulemarker, allrulelist);
2354 		LIST_INSERT_AFTER(rule, V_pf_rulemarker, allrulelist);
2355 
2356 		pf_counter_u64_periodic(&rule->evaluations);
2357 		for (int i = 0; i < 2; i++) {
2358 			pf_counter_u64_periodic(&rule->packets[i]);
2359 			pf_counter_u64_periodic(&rule->bytes[i]);
2360 		}
2361 	}
2362 }
2363 
2364 static void
2365 pf_counter_u64_periodic_main(void)
2366 {
2367 	PF_RULES_RLOCK_TRACKER;
2368 
2369 	V_pf_counter_periodic_iter++;
2370 
2371 	PF_RULES_RLOCK();
2372 	pf_counter_u64_critical_enter();
2373 	pf_status_counter_u64_periodic();
2374 	pf_kif_counter_u64_periodic();
2375 	pf_rule_counter_u64_periodic();
2376 	pf_counter_u64_critical_exit();
2377 	PF_RULES_RUNLOCK();
2378 }
2379 #else
2380 #define	pf_counter_u64_periodic_main()	do { } while (0)
2381 #endif
2382 
2383 void
2384 pf_purge_thread(void *unused __unused)
2385 {
2386 	struct epoch_tracker	 et;
2387 
2388 	VNET_ITERATOR_DECL(vnet_iter);
2389 
2390 	sx_xlock(&pf_end_lock);
2391 	while (pf_end_threads == 0) {
2392 		sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
2393 
2394 		VNET_LIST_RLOCK();
2395 		NET_EPOCH_ENTER(et);
2396 		VNET_FOREACH(vnet_iter) {
2397 			CURVNET_SET(vnet_iter);
2398 
2399 			/* Wait until V_pf_default_rule is initialized. */
2400 			if (V_pf_vnet_active == 0) {
2401 				CURVNET_RESTORE();
2402 				continue;
2403 			}
2404 
2405 			pf_counter_u64_periodic_main();
2406 
2407 			/*
2408 			 *  Process 1/interval fraction of the state
2409 			 * table every run.
2410 			 */
2411 			V_pf_purge_idx =
2412 			    pf_purge_expired_states(V_pf_purge_idx, V_pf_hashmask /
2413 			    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
2414 
2415 			/*
2416 			 * Purge other expired types every
2417 			 * PFTM_INTERVAL seconds.
2418 			 */
2419 			if (V_pf_purge_idx == 0) {
2420 				/*
2421 				 * Order is important:
2422 				 * - states and src nodes reference rules
2423 				 * - states and rules reference kifs
2424 				 */
2425 				pf_purge_expired_fragments();
2426 				pf_purge_expired_src_nodes();
2427 				pf_purge_unlinked_rules();
2428 				pfi_kkif_purge();
2429 			}
2430 			CURVNET_RESTORE();
2431 		}
2432 		NET_EPOCH_EXIT(et);
2433 		VNET_LIST_RUNLOCK();
2434 	}
2435 
2436 	pf_end_threads++;
2437 	sx_xunlock(&pf_end_lock);
2438 	kproc_exit(0);
2439 }
2440 
2441 void
2442 pf_unload_vnet_purge(void)
2443 {
2444 
2445 	/*
2446 	 * To cleanse up all kifs and rules we need
2447 	 * two runs: first one clears reference flags,
2448 	 * then pf_purge_expired_states() doesn't
2449 	 * raise them, and then second run frees.
2450 	 */
2451 	pf_purge_unlinked_rules();
2452 	pfi_kkif_purge();
2453 
2454 	/*
2455 	 * Now purge everything.
2456 	 */
2457 	pf_purge_expired_states(0, V_pf_hashmask);
2458 	pf_purge_fragments(UINT_MAX);
2459 	pf_purge_expired_src_nodes();
2460 
2461 	/*
2462 	 * Now all kifs & rules should be unreferenced,
2463 	 * thus should be successfully freed.
2464 	 */
2465 	pf_purge_unlinked_rules();
2466 	pfi_kkif_purge();
2467 }
2468 
2469 u_int32_t
2470 pf_state_expires(const struct pf_kstate *state)
2471 {
2472 	u_int32_t	timeout;
2473 	u_int32_t	start;
2474 	u_int32_t	end;
2475 	u_int32_t	states;
2476 
2477 	/* handle all PFTM_* > PFTM_MAX here */
2478 	if (state->timeout == PFTM_PURGE)
2479 		return (time_uptime);
2480 	KASSERT(state->timeout != PFTM_UNLINKED,
2481 	    ("pf_state_expires: timeout == PFTM_UNLINKED"));
2482 	KASSERT((state->timeout < PFTM_MAX),
2483 	    ("pf_state_expires: timeout > PFTM_MAX"));
2484 	timeout = state->rule.ptr->timeout[state->timeout];
2485 	if (!timeout)
2486 		timeout = V_pf_default_rule.timeout[state->timeout];
2487 	start = state->rule.ptr->timeout[PFTM_ADAPTIVE_START];
2488 	if (start && state->rule.ptr != &V_pf_default_rule) {
2489 		end = state->rule.ptr->timeout[PFTM_ADAPTIVE_END];
2490 		states = counter_u64_fetch(state->rule.ptr->states_cur);
2491 	} else {
2492 		start = V_pf_default_rule.timeout[PFTM_ADAPTIVE_START];
2493 		end = V_pf_default_rule.timeout[PFTM_ADAPTIVE_END];
2494 		states = V_pf_status.states;
2495 	}
2496 	if (end && states > start && start < end) {
2497 		if (states < end) {
2498 			timeout = (u_int64_t)timeout * (end - states) /
2499 			    (end - start);
2500 			return ((state->expire / 1000) + timeout);
2501 		}
2502 		else
2503 			return (time_uptime);
2504 	}
2505 	return ((state->expire / 1000) + timeout);
2506 }
2507 
2508 void
2509 pf_purge_expired_src_nodes(void)
2510 {
2511 	struct pf_ksrc_node_list	 freelist;
2512 	struct pf_srchash	*sh;
2513 	struct pf_ksrc_node	*cur, *next;
2514 	int i;
2515 
2516 	LIST_INIT(&freelist);
2517 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
2518 	    PF_HASHROW_LOCK(sh);
2519 	    LIST_FOREACH_SAFE(cur, &sh->nodes, entry, next)
2520 		if (cur->states == 0 && cur->expire <= time_uptime) {
2521 			pf_unlink_src_node(cur);
2522 			LIST_INSERT_HEAD(&freelist, cur, entry);
2523 		} else if (cur->rule.ptr != NULL)
2524 			cur->rule.ptr->rule_ref |= PFRULE_REFS;
2525 	    PF_HASHROW_UNLOCK(sh);
2526 	}
2527 
2528 	pf_free_src_nodes(&freelist);
2529 
2530 	V_pf_status.src_nodes = uma_zone_get_cur(V_pf_sources_z);
2531 }
2532 
2533 static void
2534 pf_src_tree_remove_state(struct pf_kstate *s)
2535 {
2536 	struct pf_ksrc_node *sn;
2537 	uint32_t timeout;
2538 
2539 	timeout = s->rule.ptr->timeout[PFTM_SRC_NODE] ?
2540 	    s->rule.ptr->timeout[PFTM_SRC_NODE] :
2541 	    V_pf_default_rule.timeout[PFTM_SRC_NODE];
2542 
2543 	if (s->src_node != NULL) {
2544 		sn = s->src_node;
2545 		PF_SRC_NODE_LOCK(sn);
2546 		if (s->src.tcp_est)
2547 			--sn->conn;
2548 		if (--sn->states == 0)
2549 			sn->expire = time_uptime + timeout;
2550 		PF_SRC_NODE_UNLOCK(sn);
2551 	}
2552 	if (s->nat_src_node != s->src_node && s->nat_src_node != NULL) {
2553 		sn = s->nat_src_node;
2554 		PF_SRC_NODE_LOCK(sn);
2555 		if (--sn->states == 0)
2556 			sn->expire = time_uptime + timeout;
2557 		PF_SRC_NODE_UNLOCK(sn);
2558 	}
2559 	s->src_node = s->nat_src_node = NULL;
2560 }
2561 
2562 /*
2563  * Unlink and potentilly free a state. Function may be
2564  * called with ID hash row locked, but always returns
2565  * unlocked, since it needs to go through key hash locking.
2566  */
2567 int
2568 pf_unlink_state(struct pf_kstate *s)
2569 {
2570 	struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)];
2571 
2572 	NET_EPOCH_ASSERT();
2573 	PF_HASHROW_ASSERT(ih);
2574 
2575 	if (s->timeout == PFTM_UNLINKED) {
2576 		/*
2577 		 * State is being processed
2578 		 * by pf_unlink_state() in
2579 		 * an other thread.
2580 		 */
2581 		PF_HASHROW_UNLOCK(ih);
2582 		return (0);	/* XXXGL: undefined actually */
2583 	}
2584 
2585 	if (s->src.state == PF_TCPS_PROXY_DST) {
2586 		/* XXX wire key the right one? */
2587 		pf_send_tcp(s->rule.ptr, s->key[PF_SK_WIRE]->af,
2588 		    &s->key[PF_SK_WIRE]->addr[1],
2589 		    &s->key[PF_SK_WIRE]->addr[0],
2590 		    s->key[PF_SK_WIRE]->port[1],
2591 		    s->key[PF_SK_WIRE]->port[0],
2592 		    s->src.seqhi, s->src.seqlo + 1,
2593 		    TH_RST|TH_ACK, 0, 0, 0, true, s->tag, 0, s->act.rtableid);
2594 	}
2595 
2596 	LIST_REMOVE(s, entry);
2597 	pf_src_tree_remove_state(s);
2598 
2599 	if (V_pfsync_delete_state_ptr != NULL)
2600 		V_pfsync_delete_state_ptr(s);
2601 
2602 	STATE_DEC_COUNTERS(s);
2603 
2604 	s->timeout = PFTM_UNLINKED;
2605 
2606 	/* Ensure we remove it from the list of halfopen states, if needed. */
2607 	if (s->key[PF_SK_STACK] != NULL &&
2608 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP)
2609 		pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
2610 
2611 	PF_HASHROW_UNLOCK(ih);
2612 
2613 	pf_detach_state(s);
2614 
2615 	pf_udp_mapping_release(s->udp_mapping);
2616 
2617 	/* pf_state_insert() initialises refs to 2 */
2618 	return (pf_release_staten(s, 2));
2619 }
2620 
2621 struct pf_kstate *
2622 pf_alloc_state(int flags)
2623 {
2624 
2625 	return (uma_zalloc(V_pf_state_z, flags | M_ZERO));
2626 }
2627 
2628 void
2629 pf_free_state(struct pf_kstate *cur)
2630 {
2631 	struct pf_krule_item *ri;
2632 
2633 	KASSERT(cur->refs == 0, ("%s: %p has refs", __func__, cur));
2634 	KASSERT(cur->timeout == PFTM_UNLINKED, ("%s: timeout %u", __func__,
2635 	    cur->timeout));
2636 
2637 	while ((ri = SLIST_FIRST(&cur->match_rules))) {
2638 		SLIST_REMOVE_HEAD(&cur->match_rules, entry);
2639 		free(ri, M_PF_RULE_ITEM);
2640 	}
2641 
2642 	pf_normalize_tcp_cleanup(cur);
2643 	uma_zfree(V_pf_state_z, cur);
2644 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_REMOVALS], 1);
2645 }
2646 
2647 /*
2648  * Called only from pf_purge_thread(), thus serialized.
2649  */
2650 static u_int
2651 pf_purge_expired_states(u_int i, int maxcheck)
2652 {
2653 	struct pf_idhash *ih;
2654 	struct pf_kstate *s;
2655 	struct pf_krule_item *mrm;
2656 	size_t count __unused;
2657 
2658 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2659 
2660 	/*
2661 	 * Go through hash and unlink states that expire now.
2662 	 */
2663 	while (maxcheck > 0) {
2664 		count = 0;
2665 		ih = &V_pf_idhash[i];
2666 
2667 		/* only take the lock if we expect to do work */
2668 		if (!LIST_EMPTY(&ih->states)) {
2669 relock:
2670 			PF_HASHROW_LOCK(ih);
2671 			LIST_FOREACH(s, &ih->states, entry) {
2672 				if (pf_state_expires(s) <= time_uptime) {
2673 					V_pf_status.states -=
2674 					    pf_unlink_state(s);
2675 					goto relock;
2676 				}
2677 				s->rule.ptr->rule_ref |= PFRULE_REFS;
2678 				if (s->nat_rule.ptr != NULL)
2679 					s->nat_rule.ptr->rule_ref |= PFRULE_REFS;
2680 				if (s->anchor.ptr != NULL)
2681 					s->anchor.ptr->rule_ref |= PFRULE_REFS;
2682 				s->kif->pfik_flags |= PFI_IFLAG_REFS;
2683 				SLIST_FOREACH(mrm, &s->match_rules, entry)
2684 					mrm->r->rule_ref |= PFRULE_REFS;
2685 				if (s->rt_kif)
2686 					s->rt_kif->pfik_flags |= PFI_IFLAG_REFS;
2687 				count++;
2688 			}
2689 			PF_HASHROW_UNLOCK(ih);
2690 		}
2691 
2692 		SDT_PROBE2(pf, purge, state, rowcount, i, count);
2693 
2694 		/* Return when we hit end of hash. */
2695 		if (++i > V_pf_hashmask) {
2696 			V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2697 			return (0);
2698 		}
2699 
2700 		maxcheck--;
2701 	}
2702 
2703 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2704 
2705 	return (i);
2706 }
2707 
2708 static void
2709 pf_purge_unlinked_rules(void)
2710 {
2711 	struct pf_krulequeue tmpq;
2712 	struct pf_krule *r, *r1;
2713 
2714 	/*
2715 	 * If we have overloading task pending, then we'd
2716 	 * better skip purging this time. There is a tiny
2717 	 * probability that overloading task references
2718 	 * an already unlinked rule.
2719 	 */
2720 	PF_OVERLOADQ_LOCK();
2721 	if (!SLIST_EMPTY(&V_pf_overloadqueue)) {
2722 		PF_OVERLOADQ_UNLOCK();
2723 		return;
2724 	}
2725 	PF_OVERLOADQ_UNLOCK();
2726 
2727 	/*
2728 	 * Do naive mark-and-sweep garbage collecting of old rules.
2729 	 * Reference flag is raised by pf_purge_expired_states()
2730 	 * and pf_purge_expired_src_nodes().
2731 	 *
2732 	 * To avoid LOR between PF_UNLNKDRULES_LOCK/PF_RULES_WLOCK,
2733 	 * use a temporary queue.
2734 	 */
2735 	TAILQ_INIT(&tmpq);
2736 	PF_UNLNKDRULES_LOCK();
2737 	TAILQ_FOREACH_SAFE(r, &V_pf_unlinked_rules, entries, r1) {
2738 		if (!(r->rule_ref & PFRULE_REFS)) {
2739 			TAILQ_REMOVE(&V_pf_unlinked_rules, r, entries);
2740 			TAILQ_INSERT_TAIL(&tmpq, r, entries);
2741 		} else
2742 			r->rule_ref &= ~PFRULE_REFS;
2743 	}
2744 	PF_UNLNKDRULES_UNLOCK();
2745 
2746 	if (!TAILQ_EMPTY(&tmpq)) {
2747 		PF_CONFIG_LOCK();
2748 		PF_RULES_WLOCK();
2749 		TAILQ_FOREACH_SAFE(r, &tmpq, entries, r1) {
2750 			TAILQ_REMOVE(&tmpq, r, entries);
2751 			pf_free_rule(r);
2752 		}
2753 		PF_RULES_WUNLOCK();
2754 		PF_CONFIG_UNLOCK();
2755 	}
2756 }
2757 
2758 void
2759 pf_print_host(struct pf_addr *addr, u_int16_t p, sa_family_t af)
2760 {
2761 	switch (af) {
2762 #ifdef INET
2763 	case AF_INET: {
2764 		u_int32_t a = ntohl(addr->addr32[0]);
2765 		printf("%u.%u.%u.%u", (a>>24)&255, (a>>16)&255,
2766 		    (a>>8)&255, a&255);
2767 		if (p) {
2768 			p = ntohs(p);
2769 			printf(":%u", p);
2770 		}
2771 		break;
2772 	}
2773 #endif /* INET */
2774 #ifdef INET6
2775 	case AF_INET6: {
2776 		u_int16_t b;
2777 		u_int8_t i, curstart, curend, maxstart, maxend;
2778 		curstart = curend = maxstart = maxend = 255;
2779 		for (i = 0; i < 8; i++) {
2780 			if (!addr->addr16[i]) {
2781 				if (curstart == 255)
2782 					curstart = i;
2783 				curend = i;
2784 			} else {
2785 				if ((curend - curstart) >
2786 				    (maxend - maxstart)) {
2787 					maxstart = curstart;
2788 					maxend = curend;
2789 				}
2790 				curstart = curend = 255;
2791 			}
2792 		}
2793 		if ((curend - curstart) >
2794 		    (maxend - maxstart)) {
2795 			maxstart = curstart;
2796 			maxend = curend;
2797 		}
2798 		for (i = 0; i < 8; i++) {
2799 			if (i >= maxstart && i <= maxend) {
2800 				if (i == 0)
2801 					printf(":");
2802 				if (i == maxend)
2803 					printf(":");
2804 			} else {
2805 				b = ntohs(addr->addr16[i]);
2806 				printf("%x", b);
2807 				if (i < 7)
2808 					printf(":");
2809 			}
2810 		}
2811 		if (p) {
2812 			p = ntohs(p);
2813 			printf("[%u]", p);
2814 		}
2815 		break;
2816 	}
2817 #endif /* INET6 */
2818 	}
2819 }
2820 
2821 void
2822 pf_print_state(struct pf_kstate *s)
2823 {
2824 	pf_print_state_parts(s, NULL, NULL);
2825 }
2826 
2827 static void
2828 pf_print_state_parts(struct pf_kstate *s,
2829     struct pf_state_key *skwp, struct pf_state_key *sksp)
2830 {
2831 	struct pf_state_key *skw, *sks;
2832 	u_int8_t proto, dir;
2833 
2834 	/* Do our best to fill these, but they're skipped if NULL */
2835 	skw = skwp ? skwp : (s ? s->key[PF_SK_WIRE] : NULL);
2836 	sks = sksp ? sksp : (s ? s->key[PF_SK_STACK] : NULL);
2837 	proto = skw ? skw->proto : (sks ? sks->proto : 0);
2838 	dir = s ? s->direction : 0;
2839 
2840 	switch (proto) {
2841 	case IPPROTO_IPV4:
2842 		printf("IPv4");
2843 		break;
2844 	case IPPROTO_IPV6:
2845 		printf("IPv6");
2846 		break;
2847 	case IPPROTO_TCP:
2848 		printf("TCP");
2849 		break;
2850 	case IPPROTO_UDP:
2851 		printf("UDP");
2852 		break;
2853 	case IPPROTO_ICMP:
2854 		printf("ICMP");
2855 		break;
2856 	case IPPROTO_ICMPV6:
2857 		printf("ICMPv6");
2858 		break;
2859 	default:
2860 		printf("%u", proto);
2861 		break;
2862 	}
2863 	switch (dir) {
2864 	case PF_IN:
2865 		printf(" in");
2866 		break;
2867 	case PF_OUT:
2868 		printf(" out");
2869 		break;
2870 	}
2871 	if (skw) {
2872 		printf(" wire: ");
2873 		pf_print_host(&skw->addr[0], skw->port[0], skw->af);
2874 		printf(" ");
2875 		pf_print_host(&skw->addr[1], skw->port[1], skw->af);
2876 	}
2877 	if (sks) {
2878 		printf(" stack: ");
2879 		if (sks != skw) {
2880 			pf_print_host(&sks->addr[0], sks->port[0], sks->af);
2881 			printf(" ");
2882 			pf_print_host(&sks->addr[1], sks->port[1], sks->af);
2883 		} else
2884 			printf("-");
2885 	}
2886 	if (s) {
2887 		if (proto == IPPROTO_TCP) {
2888 			printf(" [lo=%u high=%u win=%u modulator=%u",
2889 			    s->src.seqlo, s->src.seqhi,
2890 			    s->src.max_win, s->src.seqdiff);
2891 			if (s->src.wscale && s->dst.wscale)
2892 				printf(" wscale=%u",
2893 				    s->src.wscale & PF_WSCALE_MASK);
2894 			printf("]");
2895 			printf(" [lo=%u high=%u win=%u modulator=%u",
2896 			    s->dst.seqlo, s->dst.seqhi,
2897 			    s->dst.max_win, s->dst.seqdiff);
2898 			if (s->src.wscale && s->dst.wscale)
2899 				printf(" wscale=%u",
2900 				s->dst.wscale & PF_WSCALE_MASK);
2901 			printf("]");
2902 		}
2903 		printf(" %u:%u", s->src.state, s->dst.state);
2904 		if (s->rule.ptr)
2905 			printf(" @%d", s->rule.ptr->nr);
2906 	}
2907 }
2908 
2909 void
2910 pf_print_flags(u_int8_t f)
2911 {
2912 	if (f)
2913 		printf(" ");
2914 	if (f & TH_FIN)
2915 		printf("F");
2916 	if (f & TH_SYN)
2917 		printf("S");
2918 	if (f & TH_RST)
2919 		printf("R");
2920 	if (f & TH_PUSH)
2921 		printf("P");
2922 	if (f & TH_ACK)
2923 		printf("A");
2924 	if (f & TH_URG)
2925 		printf("U");
2926 	if (f & TH_ECE)
2927 		printf("E");
2928 	if (f & TH_CWR)
2929 		printf("W");
2930 }
2931 
2932 #define	PF_SET_SKIP_STEPS(i)					\
2933 	do {							\
2934 		while (head[i] != cur) {			\
2935 			head[i]->skip[i].ptr = cur;		\
2936 			head[i] = TAILQ_NEXT(head[i], entries);	\
2937 		}						\
2938 	} while (0)
2939 
2940 void
2941 pf_calc_skip_steps(struct pf_krulequeue *rules)
2942 {
2943 	struct pf_krule *cur, *prev, *head[PF_SKIP_COUNT];
2944 	int i;
2945 
2946 	cur = TAILQ_FIRST(rules);
2947 	prev = cur;
2948 	for (i = 0; i < PF_SKIP_COUNT; ++i)
2949 		head[i] = cur;
2950 	while (cur != NULL) {
2951 		if (cur->kif != prev->kif || cur->ifnot != prev->ifnot)
2952 			PF_SET_SKIP_STEPS(PF_SKIP_IFP);
2953 		if (cur->direction != prev->direction)
2954 			PF_SET_SKIP_STEPS(PF_SKIP_DIR);
2955 		if (cur->af != prev->af)
2956 			PF_SET_SKIP_STEPS(PF_SKIP_AF);
2957 		if (cur->proto != prev->proto)
2958 			PF_SET_SKIP_STEPS(PF_SKIP_PROTO);
2959 		if (cur->src.neg != prev->src.neg ||
2960 		    pf_addr_wrap_neq(&cur->src.addr, &prev->src.addr))
2961 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_ADDR);
2962 		if (cur->dst.neg != prev->dst.neg ||
2963 		    pf_addr_wrap_neq(&cur->dst.addr, &prev->dst.addr))
2964 			PF_SET_SKIP_STEPS(PF_SKIP_DST_ADDR);
2965 		if (cur->src.port[0] != prev->src.port[0] ||
2966 		    cur->src.port[1] != prev->src.port[1] ||
2967 		    cur->src.port_op != prev->src.port_op)
2968 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_PORT);
2969 		if (cur->dst.port[0] != prev->dst.port[0] ||
2970 		    cur->dst.port[1] != prev->dst.port[1] ||
2971 		    cur->dst.port_op != prev->dst.port_op)
2972 			PF_SET_SKIP_STEPS(PF_SKIP_DST_PORT);
2973 
2974 		prev = cur;
2975 		cur = TAILQ_NEXT(cur, entries);
2976 	}
2977 	for (i = 0; i < PF_SKIP_COUNT; ++i)
2978 		PF_SET_SKIP_STEPS(i);
2979 }
2980 
2981 int
2982 pf_addr_wrap_neq(struct pf_addr_wrap *aw1, struct pf_addr_wrap *aw2)
2983 {
2984 	if (aw1->type != aw2->type)
2985 		return (1);
2986 	switch (aw1->type) {
2987 	case PF_ADDR_ADDRMASK:
2988 	case PF_ADDR_RANGE:
2989 		if (PF_ANEQ(&aw1->v.a.addr, &aw2->v.a.addr, AF_INET6))
2990 			return (1);
2991 		if (PF_ANEQ(&aw1->v.a.mask, &aw2->v.a.mask, AF_INET6))
2992 			return (1);
2993 		return (0);
2994 	case PF_ADDR_DYNIFTL:
2995 		return (aw1->p.dyn->pfid_kt != aw2->p.dyn->pfid_kt);
2996 	case PF_ADDR_NOROUTE:
2997 	case PF_ADDR_URPFFAILED:
2998 		return (0);
2999 	case PF_ADDR_TABLE:
3000 		return (aw1->p.tbl != aw2->p.tbl);
3001 	default:
3002 		printf("invalid address type: %d\n", aw1->type);
3003 		return (1);
3004 	}
3005 }
3006 
3007 /**
3008  * Checksum updates are a little complicated because the checksum in the TCP/UDP
3009  * header isn't always a full checksum. In some cases (i.e. output) it's a
3010  * pseudo-header checksum, which is a partial checksum over src/dst IP
3011  * addresses, protocol number and length.
3012  *
3013  * That means we have the following cases:
3014  *  * Input or forwarding: we don't have TSO, the checksum fields are full
3015  *  	checksums, we need to update the checksum whenever we change anything.
3016  *  * Output (i.e. the checksum is a pseudo-header checksum):
3017  *  	x The field being updated is src/dst address or affects the length of
3018  *  	the packet. We need to update the pseudo-header checksum (note that this
3019  *  	checksum is not ones' complement).
3020  *  	x Some other field is being modified (e.g. src/dst port numbers): We
3021  *  	don't have to update anything.
3022  **/
3023 u_int16_t
3024 pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
3025 {
3026 	u_int32_t x;
3027 
3028 	x = cksum + old - new;
3029 	x = (x + (x >> 16)) & 0xffff;
3030 
3031 	/* optimise: eliminate a branch when not udp */
3032 	if (udp && cksum == 0x0000)
3033 		return cksum;
3034 	if (udp && x == 0x0000)
3035 		x = 0xffff;
3036 
3037 	return (u_int16_t)(x);
3038 }
3039 
3040 static void
3041 pf_patch_8(struct mbuf *m, u_int16_t *cksum, u_int8_t *f, u_int8_t v, bool hi,
3042     u_int8_t udp)
3043 {
3044 	u_int16_t old = htons(hi ? (*f << 8) : *f);
3045 	u_int16_t new = htons(hi ? ( v << 8) :  v);
3046 
3047 	if (*f == v)
3048 		return;
3049 
3050 	*f = v;
3051 
3052 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3053 		return;
3054 
3055 	*cksum = pf_cksum_fixup(*cksum, old, new, udp);
3056 }
3057 
3058 void
3059 pf_patch_16_unaligned(struct mbuf *m, u_int16_t *cksum, void *f, u_int16_t v,
3060     bool hi, u_int8_t udp)
3061 {
3062 	u_int8_t *fb = (u_int8_t *)f;
3063 	u_int8_t *vb = (u_int8_t *)&v;
3064 
3065 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
3066 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
3067 }
3068 
3069 void
3070 pf_patch_32_unaligned(struct mbuf *m, u_int16_t *cksum, void *f, u_int32_t v,
3071     bool hi, u_int8_t udp)
3072 {
3073 	u_int8_t *fb = (u_int8_t *)f;
3074 	u_int8_t *vb = (u_int8_t *)&v;
3075 
3076 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
3077 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
3078 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
3079 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
3080 }
3081 
3082 u_int16_t
3083 pf_proto_cksum_fixup(struct mbuf *m, u_int16_t cksum, u_int16_t old,
3084         u_int16_t new, u_int8_t udp)
3085 {
3086 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3087 		return (cksum);
3088 
3089 	return (pf_cksum_fixup(cksum, old, new, udp));
3090 }
3091 
3092 static void
3093 pf_change_ap(struct mbuf *m, struct pf_addr *a, u_int16_t *p, u_int16_t *ic,
3094         u_int16_t *pc, struct pf_addr *an, u_int16_t pn, u_int8_t u,
3095         sa_family_t af)
3096 {
3097 	struct pf_addr	ao;
3098 	u_int16_t	po = *p;
3099 
3100 	PF_ACPY(&ao, a, af);
3101 	PF_ACPY(a, an, af);
3102 
3103 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3104 		*pc = ~*pc;
3105 
3106 	*p = pn;
3107 
3108 	switch (af) {
3109 #ifdef INET
3110 	case AF_INET:
3111 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
3112 		    ao.addr16[0], an->addr16[0], 0),
3113 		    ao.addr16[1], an->addr16[1], 0);
3114 		*p = pn;
3115 
3116 		*pc = pf_cksum_fixup(pf_cksum_fixup(*pc,
3117 		    ao.addr16[0], an->addr16[0], u),
3118 		    ao.addr16[1], an->addr16[1], u);
3119 
3120 		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
3121 		break;
3122 #endif /* INET */
3123 #ifdef INET6
3124 	case AF_INET6:
3125 		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3126 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3127 		    pf_cksum_fixup(pf_cksum_fixup(*pc,
3128 		    ao.addr16[0], an->addr16[0], u),
3129 		    ao.addr16[1], an->addr16[1], u),
3130 		    ao.addr16[2], an->addr16[2], u),
3131 		    ao.addr16[3], an->addr16[3], u),
3132 		    ao.addr16[4], an->addr16[4], u),
3133 		    ao.addr16[5], an->addr16[5], u),
3134 		    ao.addr16[6], an->addr16[6], u),
3135 		    ao.addr16[7], an->addr16[7], u);
3136 
3137 		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
3138 		break;
3139 #endif /* INET6 */
3140 	}
3141 
3142 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA |
3143 	    CSUM_DELAY_DATA_IPV6)) {
3144 		*pc = ~*pc;
3145 		if (! *pc)
3146 			*pc = 0xffff;
3147 	}
3148 }
3149 
3150 /* Changes a u_int32_t.  Uses a void * so there are no align restrictions */
3151 void
3152 pf_change_a(void *a, u_int16_t *c, u_int32_t an, u_int8_t u)
3153 {
3154 	u_int32_t	ao;
3155 
3156 	memcpy(&ao, a, sizeof(ao));
3157 	memcpy(a, &an, sizeof(u_int32_t));
3158 	*c = pf_cksum_fixup(pf_cksum_fixup(*c, ao / 65536, an / 65536, u),
3159 	    ao % 65536, an % 65536, u);
3160 }
3161 
3162 void
3163 pf_change_proto_a(struct mbuf *m, void *a, u_int16_t *c, u_int32_t an, u_int8_t udp)
3164 {
3165 	u_int32_t	ao;
3166 
3167 	memcpy(&ao, a, sizeof(ao));
3168 	memcpy(a, &an, sizeof(u_int32_t));
3169 
3170 	*c = pf_proto_cksum_fixup(m,
3171 	    pf_proto_cksum_fixup(m, *c, ao / 65536, an / 65536, udp),
3172 	    ao % 65536, an % 65536, udp);
3173 }
3174 
3175 #ifdef INET6
3176 static void
3177 pf_change_a6(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u)
3178 {
3179 	struct pf_addr	ao;
3180 
3181 	PF_ACPY(&ao, a, AF_INET6);
3182 	PF_ACPY(a, an, AF_INET6);
3183 
3184 	*c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3185 	    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3186 	    pf_cksum_fixup(pf_cksum_fixup(*c,
3187 	    ao.addr16[0], an->addr16[0], u),
3188 	    ao.addr16[1], an->addr16[1], u),
3189 	    ao.addr16[2], an->addr16[2], u),
3190 	    ao.addr16[3], an->addr16[3], u),
3191 	    ao.addr16[4], an->addr16[4], u),
3192 	    ao.addr16[5], an->addr16[5], u),
3193 	    ao.addr16[6], an->addr16[6], u),
3194 	    ao.addr16[7], an->addr16[7], u);
3195 }
3196 #endif /* INET6 */
3197 
3198 static void
3199 pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa,
3200     struct pf_addr *na, u_int16_t np, u_int16_t *pc, u_int16_t *h2c,
3201     u_int16_t *ic, u_int16_t *hc, u_int8_t u, sa_family_t af)
3202 {
3203 	struct pf_addr	oia, ooa;
3204 
3205 	PF_ACPY(&oia, ia, af);
3206 	if (oa)
3207 		PF_ACPY(&ooa, oa, af);
3208 
3209 	/* Change inner protocol port, fix inner protocol checksum. */
3210 	if (ip != NULL) {
3211 		u_int16_t	oip = *ip;
3212 		u_int32_t	opc;
3213 
3214 		if (pc != NULL)
3215 			opc = *pc;
3216 		*ip = np;
3217 		if (pc != NULL)
3218 			*pc = pf_cksum_fixup(*pc, oip, *ip, u);
3219 		*ic = pf_cksum_fixup(*ic, oip, *ip, 0);
3220 		if (pc != NULL)
3221 			*ic = pf_cksum_fixup(*ic, opc, *pc, 0);
3222 	}
3223 	/* Change inner ip address, fix inner ip and icmp checksums. */
3224 	PF_ACPY(ia, na, af);
3225 	switch (af) {
3226 #ifdef INET
3227 	case AF_INET: {
3228 		u_int32_t	 oh2c = *h2c;
3229 
3230 		*h2c = pf_cksum_fixup(pf_cksum_fixup(*h2c,
3231 		    oia.addr16[0], ia->addr16[0], 0),
3232 		    oia.addr16[1], ia->addr16[1], 0);
3233 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
3234 		    oia.addr16[0], ia->addr16[0], 0),
3235 		    oia.addr16[1], ia->addr16[1], 0);
3236 		*ic = pf_cksum_fixup(*ic, oh2c, *h2c, 0);
3237 		break;
3238 	}
3239 #endif /* INET */
3240 #ifdef INET6
3241 	case AF_INET6:
3242 		*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3243 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3244 		    pf_cksum_fixup(pf_cksum_fixup(*ic,
3245 		    oia.addr16[0], ia->addr16[0], u),
3246 		    oia.addr16[1], ia->addr16[1], u),
3247 		    oia.addr16[2], ia->addr16[2], u),
3248 		    oia.addr16[3], ia->addr16[3], u),
3249 		    oia.addr16[4], ia->addr16[4], u),
3250 		    oia.addr16[5], ia->addr16[5], u),
3251 		    oia.addr16[6], ia->addr16[6], u),
3252 		    oia.addr16[7], ia->addr16[7], u);
3253 		break;
3254 #endif /* INET6 */
3255 	}
3256 	/* Outer ip address, fix outer ip or icmpv6 checksum, if necessary. */
3257 	if (oa) {
3258 		PF_ACPY(oa, na, af);
3259 		switch (af) {
3260 #ifdef INET
3261 		case AF_INET:
3262 			*hc = pf_cksum_fixup(pf_cksum_fixup(*hc,
3263 			    ooa.addr16[0], oa->addr16[0], 0),
3264 			    ooa.addr16[1], oa->addr16[1], 0);
3265 			break;
3266 #endif /* INET */
3267 #ifdef INET6
3268 		case AF_INET6:
3269 			*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3270 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3271 			    pf_cksum_fixup(pf_cksum_fixup(*ic,
3272 			    ooa.addr16[0], oa->addr16[0], u),
3273 			    ooa.addr16[1], oa->addr16[1], u),
3274 			    ooa.addr16[2], oa->addr16[2], u),
3275 			    ooa.addr16[3], oa->addr16[3], u),
3276 			    ooa.addr16[4], oa->addr16[4], u),
3277 			    ooa.addr16[5], oa->addr16[5], u),
3278 			    ooa.addr16[6], oa->addr16[6], u),
3279 			    ooa.addr16[7], oa->addr16[7], u);
3280 			break;
3281 #endif /* INET6 */
3282 		}
3283 	}
3284 }
3285 
3286 /*
3287  * Need to modulate the sequence numbers in the TCP SACK option
3288  * (credits to Krzysztof Pfaff for report and patch)
3289  */
3290 static int
3291 pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd,
3292     struct tcphdr *th, struct pf_state_peer *dst)
3293 {
3294 	int hlen = (th->th_off << 2) - sizeof(*th), thoptlen = hlen;
3295 	u_int8_t opts[TCP_MAXOLEN], *opt = opts;
3296 	int copyback = 0, i, olen;
3297 	struct sackblk sack;
3298 
3299 #define	TCPOLEN_SACKLEN	(TCPOLEN_SACK + 2)
3300 	if (hlen < TCPOLEN_SACKLEN ||
3301 	    !pf_pull_hdr(m, off + sizeof(*th), opts, hlen, NULL, NULL, pd->af))
3302 		return 0;
3303 
3304 	while (hlen >= TCPOLEN_SACKLEN) {
3305 		size_t startoff = opt - opts;
3306 		olen = opt[1];
3307 		switch (*opt) {
3308 		case TCPOPT_EOL:	/* FALLTHROUGH */
3309 		case TCPOPT_NOP:
3310 			opt++;
3311 			hlen--;
3312 			break;
3313 		case TCPOPT_SACK:
3314 			if (olen > hlen)
3315 				olen = hlen;
3316 			if (olen >= TCPOLEN_SACKLEN) {
3317 				for (i = 2; i + TCPOLEN_SACK <= olen;
3318 				    i += TCPOLEN_SACK) {
3319 					memcpy(&sack, &opt[i], sizeof(sack));
3320 					pf_patch_32_unaligned(m,
3321 					    &th->th_sum, &sack.start,
3322 					    htonl(ntohl(sack.start) - dst->seqdiff),
3323 					    PF_ALGNMNT(startoff),
3324 					    0);
3325 					pf_patch_32_unaligned(m, &th->th_sum,
3326 					    &sack.end,
3327 					    htonl(ntohl(sack.end) - dst->seqdiff),
3328 					    PF_ALGNMNT(startoff),
3329 					    0);
3330 					memcpy(&opt[i], &sack, sizeof(sack));
3331 				}
3332 				copyback = 1;
3333 			}
3334 			/* FALLTHROUGH */
3335 		default:
3336 			if (olen < 2)
3337 				olen = 2;
3338 			hlen -= olen;
3339 			opt += olen;
3340 		}
3341 	}
3342 
3343 	if (copyback)
3344 		m_copyback(m, off + sizeof(*th), thoptlen, (caddr_t)opts);
3345 	return (copyback);
3346 }
3347 
3348 struct mbuf *
3349 pf_build_tcp(const struct pf_krule *r, sa_family_t af,
3350     const struct pf_addr *saddr, const struct pf_addr *daddr,
3351     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
3352     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
3353     bool skip_firewall, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
3354 {
3355 	struct mbuf	*m;
3356 	int		 len, tlen;
3357 #ifdef INET
3358 	struct ip	*h = NULL;
3359 #endif /* INET */
3360 #ifdef INET6
3361 	struct ip6_hdr	*h6 = NULL;
3362 #endif /* INET6 */
3363 	struct tcphdr	*th;
3364 	char		*opt;
3365 	struct pf_mtag  *pf_mtag;
3366 
3367 	len = 0;
3368 	th = NULL;
3369 
3370 	/* maximum segment size tcp option */
3371 	tlen = sizeof(struct tcphdr);
3372 	if (mss)
3373 		tlen += 4;
3374 
3375 	switch (af) {
3376 #ifdef INET
3377 	case AF_INET:
3378 		len = sizeof(struct ip) + tlen;
3379 		break;
3380 #endif /* INET */
3381 #ifdef INET6
3382 	case AF_INET6:
3383 		len = sizeof(struct ip6_hdr) + tlen;
3384 		break;
3385 #endif /* INET6 */
3386 	default:
3387 		panic("%s: unsupported af %d", __func__, af);
3388 	}
3389 
3390 	m = m_gethdr(M_NOWAIT, MT_DATA);
3391 	if (m == NULL)
3392 		return (NULL);
3393 
3394 #ifdef MAC
3395 	mac_netinet_firewall_send(m);
3396 #endif
3397 	if ((pf_mtag = pf_get_mtag(m)) == NULL) {
3398 		m_freem(m);
3399 		return (NULL);
3400 	}
3401 	if (skip_firewall)
3402 		m->m_flags |= M_SKIP_FIREWALL;
3403 	pf_mtag->tag = mtag_tag;
3404 	pf_mtag->flags = mtag_flags;
3405 
3406 	if (rtableid >= 0)
3407 		M_SETFIB(m, rtableid);
3408 
3409 #ifdef ALTQ
3410 	if (r != NULL && r->qid) {
3411 		pf_mtag->qid = r->qid;
3412 
3413 		/* add hints for ecn */
3414 		pf_mtag->hdr = mtod(m, struct ip *);
3415 	}
3416 #endif /* ALTQ */
3417 	m->m_data += max_linkhdr;
3418 	m->m_pkthdr.len = m->m_len = len;
3419 	/* The rest of the stack assumes a rcvif, so provide one.
3420 	 * This is a locally generated packet, so .. close enough. */
3421 	m->m_pkthdr.rcvif = V_loif;
3422 	bzero(m->m_data, len);
3423 	switch (af) {
3424 #ifdef INET
3425 	case AF_INET:
3426 		h = mtod(m, struct ip *);
3427 
3428 		/* IP header fields included in the TCP checksum */
3429 		h->ip_p = IPPROTO_TCP;
3430 		h->ip_len = htons(tlen);
3431 		h->ip_src.s_addr = saddr->v4.s_addr;
3432 		h->ip_dst.s_addr = daddr->v4.s_addr;
3433 
3434 		th = (struct tcphdr *)((caddr_t)h + sizeof(struct ip));
3435 		break;
3436 #endif /* INET */
3437 #ifdef INET6
3438 	case AF_INET6:
3439 		h6 = mtod(m, struct ip6_hdr *);
3440 
3441 		/* IP header fields included in the TCP checksum */
3442 		h6->ip6_nxt = IPPROTO_TCP;
3443 		h6->ip6_plen = htons(tlen);
3444 		memcpy(&h6->ip6_src, &saddr->v6, sizeof(struct in6_addr));
3445 		memcpy(&h6->ip6_dst, &daddr->v6, sizeof(struct in6_addr));
3446 
3447 		th = (struct tcphdr *)((caddr_t)h6 + sizeof(struct ip6_hdr));
3448 		break;
3449 #endif /* INET6 */
3450 	}
3451 
3452 	/* TCP header */
3453 	th->th_sport = sport;
3454 	th->th_dport = dport;
3455 	th->th_seq = htonl(seq);
3456 	th->th_ack = htonl(ack);
3457 	th->th_off = tlen >> 2;
3458 	th->th_flags = tcp_flags;
3459 	th->th_win = htons(win);
3460 
3461 	if (mss) {
3462 		opt = (char *)(th + 1);
3463 		opt[0] = TCPOPT_MAXSEG;
3464 		opt[1] = 4;
3465 		HTONS(mss);
3466 		bcopy((caddr_t)&mss, (caddr_t)(opt + 2), 2);
3467 	}
3468 
3469 	switch (af) {
3470 #ifdef INET
3471 	case AF_INET:
3472 		/* TCP checksum */
3473 		th->th_sum = in_cksum(m, len);
3474 
3475 		/* Finish the IP header */
3476 		h->ip_v = 4;
3477 		h->ip_hl = sizeof(*h) >> 2;
3478 		h->ip_tos = IPTOS_LOWDELAY;
3479 		h->ip_off = htons(V_path_mtu_discovery ? IP_DF : 0);
3480 		h->ip_len = htons(len);
3481 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
3482 		h->ip_sum = 0;
3483 		break;
3484 #endif /* INET */
3485 #ifdef INET6
3486 	case AF_INET6:
3487 		/* TCP checksum */
3488 		th->th_sum = in6_cksum(m, IPPROTO_TCP,
3489 		    sizeof(struct ip6_hdr), tlen);
3490 
3491 		h6->ip6_vfc |= IPV6_VERSION;
3492 		h6->ip6_hlim = IPV6_DEFHLIM;
3493 		break;
3494 #endif /* INET6 */
3495 	}
3496 
3497 	return (m);
3498 }
3499 
3500 static void
3501 pf_send_sctp_abort(sa_family_t af, struct pf_pdesc *pd,
3502     uint8_t ttl, int rtableid)
3503 {
3504 	struct mbuf		*m;
3505 #ifdef INET
3506 	struct ip		*h = NULL;
3507 #endif /* INET */
3508 #ifdef INET6
3509 	struct ip6_hdr		*h6 = NULL;
3510 #endif /* INET6 */
3511 	struct sctphdr		*hdr;
3512 	struct sctp_chunkhdr	*chunk;
3513 	struct pf_send_entry	*pfse;
3514 	int			 off = 0;
3515 
3516 	MPASS(af == pd->af);
3517 
3518 	m = m_gethdr(M_NOWAIT, MT_DATA);
3519 	if (m == NULL)
3520 		return;
3521 
3522 	m->m_data += max_linkhdr;
3523 	m->m_flags |= M_SKIP_FIREWALL;
3524 	/* The rest of the stack assumes a rcvif, so provide one.
3525 	 * This is a locally generated packet, so .. close enough. */
3526 	m->m_pkthdr.rcvif = V_loif;
3527 
3528 	/* IPv4|6 header */
3529 	switch (af) {
3530 #ifdef INET
3531 	case AF_INET:
3532 		bzero(m->m_data, sizeof(struct ip) + sizeof(*hdr) + sizeof(*chunk));
3533 
3534 		h = mtod(m, struct ip *);
3535 
3536 		/* IP header fields included in the TCP checksum */
3537 
3538 		h->ip_p = IPPROTO_SCTP;
3539 		h->ip_len = htons(sizeof(*h) + sizeof(*hdr) + sizeof(*chunk));
3540 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
3541 		h->ip_src = pd->dst->v4;
3542 		h->ip_dst = pd->src->v4;
3543 
3544 		off += sizeof(struct ip);
3545 		break;
3546 #endif /* INET */
3547 #ifdef INET6
3548 	case AF_INET6:
3549 		bzero(m->m_data, sizeof(struct ip6_hdr) + sizeof(*hdr) + sizeof(*chunk));
3550 
3551 		h6 = mtod(m, struct ip6_hdr *);
3552 
3553 		/* IP header fields included in the TCP checksum */
3554 		h6->ip6_vfc |= IPV6_VERSION;
3555 		h6->ip6_nxt = IPPROTO_SCTP;
3556 		h6->ip6_plen = htons(sizeof(*h6) + sizeof(*hdr) + sizeof(*chunk));
3557 		h6->ip6_hlim = ttl ? ttl : V_ip6_defhlim;
3558 		memcpy(&h6->ip6_src, &pd->dst->v6, sizeof(struct in6_addr));
3559 		memcpy(&h6->ip6_dst, &pd->src->v6, sizeof(struct in6_addr));
3560 
3561 		off += sizeof(struct ip6_hdr);
3562 		break;
3563 #endif /* INET6 */
3564 	}
3565 
3566 	/* SCTP header */
3567 	hdr = mtodo(m, off);
3568 
3569 	hdr->src_port = pd->hdr.sctp.dest_port;
3570 	hdr->dest_port = pd->hdr.sctp.src_port;
3571 	hdr->v_tag = pd->sctp_initiate_tag;
3572 	hdr->checksum = 0;
3573 
3574 	/* Abort chunk. */
3575 	off += sizeof(struct sctphdr);
3576 	chunk = mtodo(m, off);
3577 
3578 	chunk->chunk_type = SCTP_ABORT_ASSOCIATION;
3579 	chunk->chunk_length = htons(sizeof(*chunk));
3580 
3581 	/* SCTP checksum */
3582 	off += sizeof(*chunk);
3583 	m->m_pkthdr.len = m->m_len = off;
3584 
3585 	pf_sctp_checksum(m, off - sizeof(*hdr) - sizeof(*chunk));
3586 
3587 	if (rtableid >= 0)
3588 		M_SETFIB(m, rtableid);
3589 
3590 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3591 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3592 	if (pfse == NULL) {
3593 		m_freem(m);
3594 		return;
3595 	}
3596 
3597 	switch (af) {
3598 #ifdef INET
3599 	case AF_INET:
3600 		pfse->pfse_type = PFSE_IP;
3601 		break;
3602 #endif /* INET */
3603 #ifdef INET6
3604 	case AF_INET6:
3605 		pfse->pfse_type = PFSE_IP6;
3606 		break;
3607 #endif /* INET6 */
3608 	}
3609 
3610 	pfse->pfse_m = m;
3611 	pf_send(pfse);
3612 }
3613 
3614 void
3615 pf_send_tcp(const struct pf_krule *r, sa_family_t af,
3616     const struct pf_addr *saddr, const struct pf_addr *daddr,
3617     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
3618     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
3619     bool skip_firewall, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
3620 {
3621 	struct pf_send_entry *pfse;
3622 	struct mbuf	*m;
3623 
3624 	m = pf_build_tcp(r, af, saddr, daddr, sport, dport, seq, ack, tcp_flags,
3625 	    win, mss, ttl, skip_firewall, mtag_tag, mtag_flags, rtableid);
3626 	if (m == NULL)
3627 		return;
3628 
3629 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3630 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3631 	if (pfse == NULL) {
3632 		m_freem(m);
3633 		return;
3634 	}
3635 
3636 	switch (af) {
3637 #ifdef INET
3638 	case AF_INET:
3639 		pfse->pfse_type = PFSE_IP;
3640 		break;
3641 #endif /* INET */
3642 #ifdef INET6
3643 	case AF_INET6:
3644 		pfse->pfse_type = PFSE_IP6;
3645 		break;
3646 #endif /* INET6 */
3647 	}
3648 
3649 	pfse->pfse_m = m;
3650 	pf_send(pfse);
3651 }
3652 
3653 static void
3654 pf_return(struct pf_krule *r, struct pf_krule *nr, struct pf_pdesc *pd,
3655     struct pf_state_key *sk, int off, struct mbuf *m, struct tcphdr *th,
3656     struct pfi_kkif *kif, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen,
3657     u_short *reason, int rtableid)
3658 {
3659 	struct pf_addr	* const saddr = pd->src;
3660 	struct pf_addr	* const daddr = pd->dst;
3661 	sa_family_t	 af = pd->af;
3662 
3663 	/* undo NAT changes, if they have taken place */
3664 	if (nr != NULL) {
3665 		PF_ACPY(saddr, &sk->addr[pd->sidx], af);
3666 		PF_ACPY(daddr, &sk->addr[pd->didx], af);
3667 		if (pd->sport)
3668 			*pd->sport = sk->port[pd->sidx];
3669 		if (pd->dport)
3670 			*pd->dport = sk->port[pd->didx];
3671 		if (pd->proto_sum)
3672 			*pd->proto_sum = bproto_sum;
3673 		if (pd->ip_sum)
3674 			*pd->ip_sum = bip_sum;
3675 		m_copyback(m, off, hdrlen, pd->hdr.any);
3676 	}
3677 	if (pd->proto == IPPROTO_TCP &&
3678 	    ((r->rule_flag & PFRULE_RETURNRST) ||
3679 	    (r->rule_flag & PFRULE_RETURN)) &&
3680 	    !(th->th_flags & TH_RST)) {
3681 		u_int32_t	 ack = ntohl(th->th_seq) + pd->p_len;
3682 		int		 len = 0;
3683 #ifdef INET
3684 		struct ip	*h4;
3685 #endif
3686 #ifdef INET6
3687 		struct ip6_hdr	*h6;
3688 #endif
3689 
3690 		switch (af) {
3691 #ifdef INET
3692 		case AF_INET:
3693 			h4 = mtod(m, struct ip *);
3694 			len = ntohs(h4->ip_len) - off;
3695 			break;
3696 #endif
3697 #ifdef INET6
3698 		case AF_INET6:
3699 			h6 = mtod(m, struct ip6_hdr *);
3700 			len = ntohs(h6->ip6_plen) - (off - sizeof(*h6));
3701 			break;
3702 #endif
3703 		}
3704 
3705 		if (pf_check_proto_cksum(m, off, len, IPPROTO_TCP, af))
3706 			REASON_SET(reason, PFRES_PROTCKSUM);
3707 		else {
3708 			if (th->th_flags & TH_SYN)
3709 				ack++;
3710 			if (th->th_flags & TH_FIN)
3711 				ack++;
3712 			pf_send_tcp(r, af, pd->dst,
3713 				pd->src, th->th_dport, th->th_sport,
3714 				ntohl(th->th_ack), ack, TH_RST|TH_ACK, 0, 0,
3715 				r->return_ttl, true, 0, 0, rtableid);
3716 		}
3717 	} else if (pd->proto == IPPROTO_SCTP &&
3718 	    (r->rule_flag & PFRULE_RETURN)) {
3719 		pf_send_sctp_abort(af, pd, r->return_ttl, rtableid);
3720 	} else if (pd->proto != IPPROTO_ICMP && af == AF_INET &&
3721 		r->return_icmp)
3722 		pf_send_icmp(m, r->return_icmp >> 8,
3723 			r->return_icmp & 255, af, r, rtableid);
3724 	else if (pd->proto != IPPROTO_ICMPV6 && af == AF_INET6 &&
3725 		r->return_icmp6)
3726 		pf_send_icmp(m, r->return_icmp6 >> 8,
3727 			r->return_icmp6 & 255, af, r, rtableid);
3728 }
3729 
3730 static int
3731 pf_match_ieee8021q_pcp(u_int8_t prio, struct mbuf *m)
3732 {
3733 	struct m_tag *mtag;
3734 	u_int8_t mpcp;
3735 
3736 	mtag = m_tag_locate(m, MTAG_8021Q, MTAG_8021Q_PCP_IN, NULL);
3737 	if (mtag == NULL)
3738 		return (0);
3739 
3740 	if (prio == PF_PRIO_ZERO)
3741 		prio = 0;
3742 
3743 	mpcp = *(uint8_t *)(mtag + 1);
3744 
3745 	return (mpcp == prio);
3746 }
3747 
3748 static int
3749 pf_icmp_to_bandlim(uint8_t type)
3750 {
3751 	switch (type) {
3752 		case ICMP_ECHO:
3753 		case ICMP_ECHOREPLY:
3754 			return (BANDLIM_ICMP_ECHO);
3755 		case ICMP_TSTAMP:
3756 		case ICMP_TSTAMPREPLY:
3757 			return (BANDLIM_ICMP_TSTAMP);
3758 		case ICMP_UNREACH:
3759 		default:
3760 			return (BANDLIM_ICMP_UNREACH);
3761 	}
3762 }
3763 
3764 static void
3765 pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af,
3766     struct pf_krule *r, int rtableid)
3767 {
3768 	struct pf_send_entry *pfse;
3769 	struct mbuf *m0;
3770 	struct pf_mtag *pf_mtag;
3771 
3772 	/* ICMP packet rate limitation. */
3773 #ifdef INET6
3774 	if (af == AF_INET6) {
3775 		if (icmp6_ratelimit(NULL, type, code))
3776 			return;
3777 	}
3778 #endif
3779 #ifdef INET
3780 	if (af == AF_INET) {
3781 		if (badport_bandlim(pf_icmp_to_bandlim(type)) != 0)
3782 			return;
3783 	}
3784 #endif
3785 
3786 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3787 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3788 	if (pfse == NULL)
3789 		return;
3790 
3791 	if ((m0 = m_copypacket(m, M_NOWAIT)) == NULL) {
3792 		free(pfse, M_PFTEMP);
3793 		return;
3794 	}
3795 
3796 	if ((pf_mtag = pf_get_mtag(m0)) == NULL) {
3797 		free(pfse, M_PFTEMP);
3798 		return;
3799 	}
3800 	/* XXX: revisit */
3801 	m0->m_flags |= M_SKIP_FIREWALL;
3802 
3803 	if (rtableid >= 0)
3804 		M_SETFIB(m0, rtableid);
3805 
3806 #ifdef ALTQ
3807 	if (r->qid) {
3808 		pf_mtag->qid = r->qid;
3809 		/* add hints for ecn */
3810 		pf_mtag->hdr = mtod(m0, struct ip *);
3811 	}
3812 #endif /* ALTQ */
3813 
3814 	switch (af) {
3815 #ifdef INET
3816 	case AF_INET:
3817 		pfse->pfse_type = PFSE_ICMP;
3818 		break;
3819 #endif /* INET */
3820 #ifdef INET6
3821 	case AF_INET6:
3822 		pfse->pfse_type = PFSE_ICMP6;
3823 		break;
3824 #endif /* INET6 */
3825 	}
3826 	pfse->pfse_m = m0;
3827 	pfse->icmpopts.type = type;
3828 	pfse->icmpopts.code = code;
3829 	pf_send(pfse);
3830 }
3831 
3832 /*
3833  * Return 1 if the addresses a and b match (with mask m), otherwise return 0.
3834  * If n is 0, they match if they are equal. If n is != 0, they match if they
3835  * are different.
3836  */
3837 int
3838 pf_match_addr(u_int8_t n, struct pf_addr *a, struct pf_addr *m,
3839     struct pf_addr *b, sa_family_t af)
3840 {
3841 	int	match = 0;
3842 
3843 	switch (af) {
3844 #ifdef INET
3845 	case AF_INET:
3846 		if (IN_ARE_MASKED_ADDR_EQUAL(a->v4, b->v4, m->v4))
3847 			match++;
3848 		break;
3849 #endif /* INET */
3850 #ifdef INET6
3851 	case AF_INET6:
3852 		if (IN6_ARE_MASKED_ADDR_EQUAL(&a->v6, &b->v6, &m->v6))
3853 			match++;
3854 		break;
3855 #endif /* INET6 */
3856 	}
3857 	if (match) {
3858 		if (n)
3859 			return (0);
3860 		else
3861 			return (1);
3862 	} else {
3863 		if (n)
3864 			return (1);
3865 		else
3866 			return (0);
3867 	}
3868 }
3869 
3870 /*
3871  * Return 1 if b <= a <= e, otherwise return 0.
3872  */
3873 int
3874 pf_match_addr_range(struct pf_addr *b, struct pf_addr *e,
3875     struct pf_addr *a, sa_family_t af)
3876 {
3877 	switch (af) {
3878 #ifdef INET
3879 	case AF_INET:
3880 		if ((ntohl(a->addr32[0]) < ntohl(b->addr32[0])) ||
3881 		    (ntohl(a->addr32[0]) > ntohl(e->addr32[0])))
3882 			return (0);
3883 		break;
3884 #endif /* INET */
3885 #ifdef INET6
3886 	case AF_INET6: {
3887 		int	i;
3888 
3889 		/* check a >= b */
3890 		for (i = 0; i < 4; ++i)
3891 			if (ntohl(a->addr32[i]) > ntohl(b->addr32[i]))
3892 				break;
3893 			else if (ntohl(a->addr32[i]) < ntohl(b->addr32[i]))
3894 				return (0);
3895 		/* check a <= e */
3896 		for (i = 0; i < 4; ++i)
3897 			if (ntohl(a->addr32[i]) < ntohl(e->addr32[i]))
3898 				break;
3899 			else if (ntohl(a->addr32[i]) > ntohl(e->addr32[i]))
3900 				return (0);
3901 		break;
3902 	}
3903 #endif /* INET6 */
3904 	}
3905 	return (1);
3906 }
3907 
3908 static int
3909 pf_match(u_int8_t op, u_int32_t a1, u_int32_t a2, u_int32_t p)
3910 {
3911 	switch (op) {
3912 	case PF_OP_IRG:
3913 		return ((p > a1) && (p < a2));
3914 	case PF_OP_XRG:
3915 		return ((p < a1) || (p > a2));
3916 	case PF_OP_RRG:
3917 		return ((p >= a1) && (p <= a2));
3918 	case PF_OP_EQ:
3919 		return (p == a1);
3920 	case PF_OP_NE:
3921 		return (p != a1);
3922 	case PF_OP_LT:
3923 		return (p < a1);
3924 	case PF_OP_LE:
3925 		return (p <= a1);
3926 	case PF_OP_GT:
3927 		return (p > a1);
3928 	case PF_OP_GE:
3929 		return (p >= a1);
3930 	}
3931 	return (0); /* never reached */
3932 }
3933 
3934 int
3935 pf_match_port(u_int8_t op, u_int16_t a1, u_int16_t a2, u_int16_t p)
3936 {
3937 	NTOHS(a1);
3938 	NTOHS(a2);
3939 	NTOHS(p);
3940 	return (pf_match(op, a1, a2, p));
3941 }
3942 
3943 static int
3944 pf_match_uid(u_int8_t op, uid_t a1, uid_t a2, uid_t u)
3945 {
3946 	if (u == UID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
3947 		return (0);
3948 	return (pf_match(op, a1, a2, u));
3949 }
3950 
3951 static int
3952 pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g)
3953 {
3954 	if (g == GID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
3955 		return (0);
3956 	return (pf_match(op, a1, a2, g));
3957 }
3958 
3959 int
3960 pf_match_tag(struct mbuf *m, struct pf_krule *r, int *tag, int mtag)
3961 {
3962 	if (*tag == -1)
3963 		*tag = mtag;
3964 
3965 	return ((!r->match_tag_not && r->match_tag == *tag) ||
3966 	    (r->match_tag_not && r->match_tag != *tag));
3967 }
3968 
3969 static int
3970 pf_match_rcvif(struct mbuf *m, struct pf_krule *r)
3971 {
3972 	struct ifnet *ifp = m->m_pkthdr.rcvif;
3973 	struct pfi_kkif *kif;
3974 
3975 	if (ifp == NULL)
3976 		return (0);
3977 
3978 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
3979 
3980 	if (kif == NULL) {
3981 		DPFPRINTF(PF_DEBUG_URGENT,
3982 		    ("pf_test_via: kif == NULL, @%d via %s\n", r->nr,
3983 			r->rcv_ifname));
3984 		return (0);
3985 	}
3986 
3987 	return (pfi_kkif_match(r->rcv_kif, kif));
3988 }
3989 
3990 int
3991 pf_tag_packet(struct mbuf *m, struct pf_pdesc *pd, int tag)
3992 {
3993 
3994 	KASSERT(tag > 0, ("%s: tag %d", __func__, tag));
3995 
3996 	if (pd->pf_mtag == NULL && ((pd->pf_mtag = pf_get_mtag(m)) == NULL))
3997 		return (ENOMEM);
3998 
3999 	pd->pf_mtag->tag = tag;
4000 
4001 	return (0);
4002 }
4003 
4004 #define	PF_ANCHOR_STACKSIZE	32
4005 struct pf_kanchor_stackframe {
4006 	struct pf_kruleset	*rs;
4007 	struct pf_krule		*r;	/* XXX: + match bit */
4008 	struct pf_kanchor	*child;
4009 };
4010 
4011 /*
4012  * XXX: We rely on malloc(9) returning pointer aligned addresses.
4013  */
4014 #define	PF_ANCHORSTACK_MATCH	0x00000001
4015 #define	PF_ANCHORSTACK_MASK	(PF_ANCHORSTACK_MATCH)
4016 
4017 #define	PF_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
4018 #define	PF_ANCHOR_RULE(f)	(struct pf_krule *)			\
4019 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
4020 #define	PF_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 			\
4021 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
4022 } while (0)
4023 
4024 void
4025 pf_step_into_anchor(struct pf_kanchor_stackframe *stack, int *depth,
4026     struct pf_kruleset **rs, int n, struct pf_krule **r, struct pf_krule **a,
4027     int *match)
4028 {
4029 	struct pf_kanchor_stackframe	*f;
4030 
4031 	PF_RULES_RASSERT();
4032 
4033 	if (match)
4034 		*match = 0;
4035 	if (*depth >= PF_ANCHOR_STACKSIZE) {
4036 		printf("%s: anchor stack overflow on %s\n",
4037 		    __func__, (*r)->anchor->name);
4038 		*r = TAILQ_NEXT(*r, entries);
4039 		return;
4040 	} else if (*depth == 0 && a != NULL)
4041 		*a = *r;
4042 	f = stack + (*depth)++;
4043 	f->rs = *rs;
4044 	f->r = *r;
4045 	if ((*r)->anchor_wildcard) {
4046 		struct pf_kanchor_node *parent = &(*r)->anchor->children;
4047 
4048 		if ((f->child = RB_MIN(pf_kanchor_node, parent)) == NULL) {
4049 			*r = NULL;
4050 			return;
4051 		}
4052 		*rs = &f->child->ruleset;
4053 	} else {
4054 		f->child = NULL;
4055 		*rs = &(*r)->anchor->ruleset;
4056 	}
4057 	*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
4058 }
4059 
4060 int
4061 pf_step_out_of_anchor(struct pf_kanchor_stackframe *stack, int *depth,
4062     struct pf_kruleset **rs, int n, struct pf_krule **r, struct pf_krule **a,
4063     int *match)
4064 {
4065 	struct pf_kanchor_stackframe	*f;
4066 	struct pf_krule *fr;
4067 	int quick = 0;
4068 
4069 	PF_RULES_RASSERT();
4070 
4071 	do {
4072 		if (*depth <= 0)
4073 			break;
4074 		f = stack + *depth - 1;
4075 		fr = PF_ANCHOR_RULE(f);
4076 		if (f->child != NULL) {
4077 			/*
4078 			 * This block traverses through
4079 			 * a wildcard anchor.
4080 			 */
4081 			if (match != NULL && *match) {
4082 				/*
4083 				 * If any of "*" matched, then
4084 				 * "foo/ *" matched, mark frame
4085 				 * appropriately.
4086 				 */
4087 				PF_ANCHOR_SET_MATCH(f);
4088 				*match = 0;
4089 			}
4090 			f->child = RB_NEXT(pf_kanchor_node,
4091 			    &fr->anchor->children, f->child);
4092 			if (f->child != NULL) {
4093 				*rs = &f->child->ruleset;
4094 				*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
4095 				if (*r == NULL)
4096 					continue;
4097 				else
4098 					break;
4099 			}
4100 		}
4101 		(*depth)--;
4102 		if (*depth == 0 && a != NULL)
4103 			*a = NULL;
4104 		*rs = f->rs;
4105 		if (PF_ANCHOR_MATCH(f) || (match != NULL && *match))
4106 			quick = fr->quick;
4107 		*r = TAILQ_NEXT(fr, entries);
4108 	} while (*r == NULL);
4109 
4110 	return (quick);
4111 }
4112 
4113 struct pf_keth_anchor_stackframe {
4114 	struct pf_keth_ruleset	*rs;
4115 	struct pf_keth_rule	*r;	/* XXX: + match bit */
4116 	struct pf_keth_anchor	*child;
4117 };
4118 
4119 #define	PF_ETH_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
4120 #define	PF_ETH_ANCHOR_RULE(f)	(struct pf_keth_rule *)			\
4121 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
4122 #define	PF_ETH_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 		\
4123 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
4124 } while (0)
4125 
4126 void
4127 pf_step_into_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
4128     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
4129     struct pf_keth_rule **a, int *match)
4130 {
4131 	struct pf_keth_anchor_stackframe	*f;
4132 
4133 	NET_EPOCH_ASSERT();
4134 
4135 	if (match)
4136 		*match = 0;
4137 	if (*depth >= PF_ANCHOR_STACKSIZE) {
4138 		printf("%s: anchor stack overflow on %s\n",
4139 		    __func__, (*r)->anchor->name);
4140 		*r = TAILQ_NEXT(*r, entries);
4141 		return;
4142 	} else if (*depth == 0 && a != NULL)
4143 		*a = *r;
4144 	f = stack + (*depth)++;
4145 	f->rs = *rs;
4146 	f->r = *r;
4147 	if ((*r)->anchor_wildcard) {
4148 		struct pf_keth_anchor_node *parent = &(*r)->anchor->children;
4149 
4150 		if ((f->child = RB_MIN(pf_keth_anchor_node, parent)) == NULL) {
4151 			*r = NULL;
4152 			return;
4153 		}
4154 		*rs = &f->child->ruleset;
4155 	} else {
4156 		f->child = NULL;
4157 		*rs = &(*r)->anchor->ruleset;
4158 	}
4159 	*r = TAILQ_FIRST((*rs)->active.rules);
4160 }
4161 
4162 int
4163 pf_step_out_of_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
4164     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
4165     struct pf_keth_rule **a, int *match)
4166 {
4167 	struct pf_keth_anchor_stackframe	*f;
4168 	struct pf_keth_rule *fr;
4169 	int quick = 0;
4170 
4171 	NET_EPOCH_ASSERT();
4172 
4173 	do {
4174 		if (*depth <= 0)
4175 			break;
4176 		f = stack + *depth - 1;
4177 		fr = PF_ETH_ANCHOR_RULE(f);
4178 		if (f->child != NULL) {
4179 			/*
4180 			 * This block traverses through
4181 			 * a wildcard anchor.
4182 			 */
4183 			if (match != NULL && *match) {
4184 				/*
4185 				 * If any of "*" matched, then
4186 				 * "foo/ *" matched, mark frame
4187 				 * appropriately.
4188 				 */
4189 				PF_ETH_ANCHOR_SET_MATCH(f);
4190 				*match = 0;
4191 			}
4192 			f->child = RB_NEXT(pf_keth_anchor_node,
4193 			    &fr->anchor->children, f->child);
4194 			if (f->child != NULL) {
4195 				*rs = &f->child->ruleset;
4196 				*r = TAILQ_FIRST((*rs)->active.rules);
4197 				if (*r == NULL)
4198 					continue;
4199 				else
4200 					break;
4201 			}
4202 		}
4203 		(*depth)--;
4204 		if (*depth == 0 && a != NULL)
4205 			*a = NULL;
4206 		*rs = f->rs;
4207 		if (PF_ETH_ANCHOR_MATCH(f) || (match != NULL && *match))
4208 			quick = fr->quick;
4209 		*r = TAILQ_NEXT(fr, entries);
4210 	} while (*r == NULL);
4211 
4212 	return (quick);
4213 }
4214 
4215 #ifdef INET6
4216 void
4217 pf_poolmask(struct pf_addr *naddr, struct pf_addr *raddr,
4218     struct pf_addr *rmask, struct pf_addr *saddr, sa_family_t af)
4219 {
4220 	switch (af) {
4221 #ifdef INET
4222 	case AF_INET:
4223 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
4224 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
4225 		break;
4226 #endif /* INET */
4227 	case AF_INET6:
4228 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
4229 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
4230 		naddr->addr32[1] = (raddr->addr32[1] & rmask->addr32[1]) |
4231 		((rmask->addr32[1] ^ 0xffffffff ) & saddr->addr32[1]);
4232 		naddr->addr32[2] = (raddr->addr32[2] & rmask->addr32[2]) |
4233 		((rmask->addr32[2] ^ 0xffffffff ) & saddr->addr32[2]);
4234 		naddr->addr32[3] = (raddr->addr32[3] & rmask->addr32[3]) |
4235 		((rmask->addr32[3] ^ 0xffffffff ) & saddr->addr32[3]);
4236 		break;
4237 	}
4238 }
4239 
4240 void
4241 pf_addr_inc(struct pf_addr *addr, sa_family_t af)
4242 {
4243 	switch (af) {
4244 #ifdef INET
4245 	case AF_INET:
4246 		addr->addr32[0] = htonl(ntohl(addr->addr32[0]) + 1);
4247 		break;
4248 #endif /* INET */
4249 	case AF_INET6:
4250 		if (addr->addr32[3] == 0xffffffff) {
4251 			addr->addr32[3] = 0;
4252 			if (addr->addr32[2] == 0xffffffff) {
4253 				addr->addr32[2] = 0;
4254 				if (addr->addr32[1] == 0xffffffff) {
4255 					addr->addr32[1] = 0;
4256 					addr->addr32[0] =
4257 					    htonl(ntohl(addr->addr32[0]) + 1);
4258 				} else
4259 					addr->addr32[1] =
4260 					    htonl(ntohl(addr->addr32[1]) + 1);
4261 			} else
4262 				addr->addr32[2] =
4263 				    htonl(ntohl(addr->addr32[2]) + 1);
4264 		} else
4265 			addr->addr32[3] =
4266 			    htonl(ntohl(addr->addr32[3]) + 1);
4267 		break;
4268 	}
4269 }
4270 #endif /* INET6 */
4271 
4272 void
4273 pf_rule_to_actions(struct pf_krule *r, struct pf_rule_actions *a)
4274 {
4275 	/*
4276 	 * Modern rules use the same flags in rules as they do in states.
4277 	 */
4278 	a->flags |= (r->scrub_flags & (PFSTATE_NODF|PFSTATE_RANDOMID|
4279 	    PFSTATE_SCRUB_TCP|PFSTATE_SETPRIO));
4280 
4281 	/*
4282 	 * Old-style scrub rules have different flags which need to be translated.
4283 	 */
4284 	if (r->rule_flag & PFRULE_RANDOMID)
4285 		a->flags |= PFSTATE_RANDOMID;
4286 	if (r->scrub_flags & PFSTATE_SETTOS || r->rule_flag & PFRULE_SET_TOS ) {
4287 		a->flags |= PFSTATE_SETTOS;
4288 		a->set_tos = r->set_tos;
4289 	}
4290 
4291 	if (r->qid)
4292 		a->qid = r->qid;
4293 	if (r->pqid)
4294 		a->pqid = r->pqid;
4295 	if (r->rtableid >= 0)
4296 		a->rtableid = r->rtableid;
4297 	a->log |= r->log;
4298 	if (r->min_ttl)
4299 		a->min_ttl = r->min_ttl;
4300 	if (r->max_mss)
4301 		a->max_mss = r->max_mss;
4302 	if (r->dnpipe)
4303 		a->dnpipe = r->dnpipe;
4304 	if (r->dnrpipe)
4305 		a->dnrpipe = r->dnrpipe;
4306 	if (r->dnpipe || r->dnrpipe) {
4307 		if (r->free_flags & PFRULE_DN_IS_PIPE)
4308 			a->flags |= PFSTATE_DN_IS_PIPE;
4309 		else
4310 			a->flags &= ~PFSTATE_DN_IS_PIPE;
4311 	}
4312 	if (r->scrub_flags & PFSTATE_SETPRIO) {
4313 		a->set_prio[0] = r->set_prio[0];
4314 		a->set_prio[1] = r->set_prio[1];
4315 	}
4316 }
4317 
4318 int
4319 pf_socket_lookup(struct pf_pdesc *pd, struct mbuf *m)
4320 {
4321 	struct pf_addr		*saddr, *daddr;
4322 	u_int16_t		 sport, dport;
4323 	struct inpcbinfo	*pi;
4324 	struct inpcb		*inp;
4325 
4326 	pd->lookup.uid = UID_MAX;
4327 	pd->lookup.gid = GID_MAX;
4328 
4329 	switch (pd->proto) {
4330 	case IPPROTO_TCP:
4331 		sport = pd->hdr.tcp.th_sport;
4332 		dport = pd->hdr.tcp.th_dport;
4333 		pi = &V_tcbinfo;
4334 		break;
4335 	case IPPROTO_UDP:
4336 		sport = pd->hdr.udp.uh_sport;
4337 		dport = pd->hdr.udp.uh_dport;
4338 		pi = &V_udbinfo;
4339 		break;
4340 	default:
4341 		return (-1);
4342 	}
4343 	if (pd->dir == PF_IN) {
4344 		saddr = pd->src;
4345 		daddr = pd->dst;
4346 	} else {
4347 		u_int16_t	p;
4348 
4349 		p = sport;
4350 		sport = dport;
4351 		dport = p;
4352 		saddr = pd->dst;
4353 		daddr = pd->src;
4354 	}
4355 	switch (pd->af) {
4356 #ifdef INET
4357 	case AF_INET:
4358 		inp = in_pcblookup_mbuf(pi, saddr->v4, sport, daddr->v4,
4359 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
4360 		if (inp == NULL) {
4361 			inp = in_pcblookup_mbuf(pi, saddr->v4, sport,
4362 			   daddr->v4, dport, INPLOOKUP_WILDCARD |
4363 			   INPLOOKUP_RLOCKPCB, NULL, m);
4364 			if (inp == NULL)
4365 				return (-1);
4366 		}
4367 		break;
4368 #endif /* INET */
4369 #ifdef INET6
4370 	case AF_INET6:
4371 		inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport, &daddr->v6,
4372 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
4373 		if (inp == NULL) {
4374 			inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport,
4375 			    &daddr->v6, dport, INPLOOKUP_WILDCARD |
4376 			    INPLOOKUP_RLOCKPCB, NULL, m);
4377 			if (inp == NULL)
4378 				return (-1);
4379 		}
4380 		break;
4381 #endif /* INET6 */
4382 
4383 	default:
4384 		return (-1);
4385 	}
4386 	INP_RLOCK_ASSERT(inp);
4387 	pd->lookup.uid = inp->inp_cred->cr_uid;
4388 	pd->lookup.gid = inp->inp_cred->cr_groups[0];
4389 	INP_RUNLOCK(inp);
4390 
4391 	return (1);
4392 }
4393 
4394 u_int8_t
4395 pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
4396 {
4397 	int		 hlen;
4398 	u_int8_t	 hdr[60];
4399 	u_int8_t	*opt, optlen;
4400 	u_int8_t	 wscale = 0;
4401 
4402 	hlen = th_off << 2;		/* hlen <= sizeof(hdr) */
4403 	if (hlen <= sizeof(struct tcphdr))
4404 		return (0);
4405 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
4406 		return (0);
4407 	opt = hdr + sizeof(struct tcphdr);
4408 	hlen -= sizeof(struct tcphdr);
4409 	while (hlen >= 3) {
4410 		switch (*opt) {
4411 		case TCPOPT_EOL:
4412 		case TCPOPT_NOP:
4413 			++opt;
4414 			--hlen;
4415 			break;
4416 		case TCPOPT_WINDOW:
4417 			wscale = opt[2];
4418 			if (wscale > TCP_MAX_WINSHIFT)
4419 				wscale = TCP_MAX_WINSHIFT;
4420 			wscale |= PF_WSCALE_FLAG;
4421 			/* FALLTHROUGH */
4422 		default:
4423 			optlen = opt[1];
4424 			if (optlen < 2)
4425 				optlen = 2;
4426 			hlen -= optlen;
4427 			opt += optlen;
4428 			break;
4429 		}
4430 	}
4431 	return (wscale);
4432 }
4433 
4434 u_int16_t
4435 pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
4436 {
4437 	int		 hlen;
4438 	u_int8_t	 hdr[60];
4439 	u_int8_t	*opt, optlen;
4440 	u_int16_t	 mss = V_tcp_mssdflt;
4441 
4442 	hlen = th_off << 2;	/* hlen <= sizeof(hdr) */
4443 	if (hlen <= sizeof(struct tcphdr))
4444 		return (0);
4445 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
4446 		return (0);
4447 	opt = hdr + sizeof(struct tcphdr);
4448 	hlen -= sizeof(struct tcphdr);
4449 	while (hlen >= TCPOLEN_MAXSEG) {
4450 		switch (*opt) {
4451 		case TCPOPT_EOL:
4452 		case TCPOPT_NOP:
4453 			++opt;
4454 			--hlen;
4455 			break;
4456 		case TCPOPT_MAXSEG:
4457 			bcopy((caddr_t)(opt + 2), (caddr_t)&mss, 2);
4458 			NTOHS(mss);
4459 			/* FALLTHROUGH */
4460 		default:
4461 			optlen = opt[1];
4462 			if (optlen < 2)
4463 				optlen = 2;
4464 			hlen -= optlen;
4465 			opt += optlen;
4466 			break;
4467 		}
4468 	}
4469 	return (mss);
4470 }
4471 
4472 static u_int16_t
4473 pf_calc_mss(struct pf_addr *addr, sa_family_t af, int rtableid, u_int16_t offer)
4474 {
4475 	struct nhop_object *nh;
4476 #ifdef INET6
4477 	struct in6_addr		dst6;
4478 	uint32_t		scopeid;
4479 #endif /* INET6 */
4480 	int			 hlen = 0;
4481 	uint16_t		 mss = 0;
4482 
4483 	NET_EPOCH_ASSERT();
4484 
4485 	switch (af) {
4486 #ifdef INET
4487 	case AF_INET:
4488 		hlen = sizeof(struct ip);
4489 		nh = fib4_lookup(rtableid, addr->v4, 0, 0, 0);
4490 		if (nh != NULL)
4491 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
4492 		break;
4493 #endif /* INET */
4494 #ifdef INET6
4495 	case AF_INET6:
4496 		hlen = sizeof(struct ip6_hdr);
4497 		in6_splitscope(&addr->v6, &dst6, &scopeid);
4498 		nh = fib6_lookup(rtableid, &dst6, scopeid, 0, 0);
4499 		if (nh != NULL)
4500 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
4501 		break;
4502 #endif /* INET6 */
4503 	}
4504 
4505 	mss = max(V_tcp_mssdflt, mss);
4506 	mss = min(mss, offer);
4507 	mss = max(mss, 64);		/* sanity - at least max opt space */
4508 	return (mss);
4509 }
4510 
4511 static u_int32_t
4512 pf_tcp_iss(struct pf_pdesc *pd)
4513 {
4514 	MD5_CTX ctx;
4515 	u_int32_t digest[4];
4516 
4517 	if (V_pf_tcp_secret_init == 0) {
4518 		arc4random_buf(&V_pf_tcp_secret, sizeof(V_pf_tcp_secret));
4519 		MD5Init(&V_pf_tcp_secret_ctx);
4520 		MD5Update(&V_pf_tcp_secret_ctx, V_pf_tcp_secret,
4521 		    sizeof(V_pf_tcp_secret));
4522 		V_pf_tcp_secret_init = 1;
4523 	}
4524 
4525 	ctx = V_pf_tcp_secret_ctx;
4526 
4527 	MD5Update(&ctx, (char *)&pd->hdr.tcp.th_sport, sizeof(u_short));
4528 	MD5Update(&ctx, (char *)&pd->hdr.tcp.th_dport, sizeof(u_short));
4529 	if (pd->af == AF_INET6) {
4530 		MD5Update(&ctx, (char *)&pd->src->v6, sizeof(struct in6_addr));
4531 		MD5Update(&ctx, (char *)&pd->dst->v6, sizeof(struct in6_addr));
4532 	} else {
4533 		MD5Update(&ctx, (char *)&pd->src->v4, sizeof(struct in_addr));
4534 		MD5Update(&ctx, (char *)&pd->dst->v4, sizeof(struct in_addr));
4535 	}
4536 	MD5Final((u_char *)digest, &ctx);
4537 	V_pf_tcp_iss_off += 4096;
4538 #define	ISN_RANDOM_INCREMENT (4096 - 1)
4539 	return (digest[0] + (arc4random() & ISN_RANDOM_INCREMENT) +
4540 	    V_pf_tcp_iss_off);
4541 #undef	ISN_RANDOM_INCREMENT
4542 }
4543 
4544 static bool
4545 pf_match_eth_addr(const uint8_t *a, const struct pf_keth_rule_addr *r)
4546 {
4547 	bool match = true;
4548 
4549 	/* Always matches if not set */
4550 	if (! r->isset)
4551 		return (!r->neg);
4552 
4553 	for (int i = 0; i < ETHER_ADDR_LEN; i++) {
4554 		if ((a[i] & r->mask[i]) != (r->addr[i] & r->mask[i])) {
4555 			match = false;
4556 			break;
4557 		}
4558 	}
4559 
4560 	return (match ^ r->neg);
4561 }
4562 
4563 static int
4564 pf_match_eth_tag(struct mbuf *m, struct pf_keth_rule *r, int *tag, int mtag)
4565 {
4566 	if (*tag == -1)
4567 		*tag = mtag;
4568 
4569 	return ((!r->match_tag_not && r->match_tag == *tag) ||
4570 	    (r->match_tag_not && r->match_tag != *tag));
4571 }
4572 
4573 static void
4574 pf_bridge_to(struct ifnet *ifp, struct mbuf *m)
4575 {
4576 	/* If we don't have the interface drop the packet. */
4577 	if (ifp == NULL) {
4578 		m_freem(m);
4579 		return;
4580 	}
4581 
4582 	switch (ifp->if_type) {
4583 	case IFT_ETHER:
4584 	case IFT_XETHER:
4585 	case IFT_L2VLAN:
4586 	case IFT_BRIDGE:
4587 	case IFT_IEEE8023ADLAG:
4588 		break;
4589 	default:
4590 		m_freem(m);
4591 		return;
4592 	}
4593 
4594 	ifp->if_transmit(ifp, m);
4595 }
4596 
4597 static int
4598 pf_test_eth_rule(int dir, struct pfi_kkif *kif, struct mbuf **m0)
4599 {
4600 #ifdef INET
4601 	struct ip ip;
4602 #endif
4603 #ifdef INET6
4604 	struct ip6_hdr ip6;
4605 #endif
4606 	struct mbuf *m = *m0;
4607 	struct ether_header *e;
4608 	struct pf_keth_rule *r, *rm, *a = NULL;
4609 	struct pf_keth_ruleset *ruleset = NULL;
4610 	struct pf_mtag *mtag;
4611 	struct pf_keth_ruleq *rules;
4612 	struct pf_addr *src = NULL, *dst = NULL;
4613 	struct pfi_kkif *bridge_to;
4614 	sa_family_t af = 0;
4615 	uint16_t proto;
4616 	int asd = 0, match = 0;
4617 	int tag = -1;
4618 	uint8_t action;
4619 	struct pf_keth_anchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
4620 
4621 	MPASS(kif->pfik_ifp->if_vnet == curvnet);
4622 	NET_EPOCH_ASSERT();
4623 
4624 	PF_RULES_RLOCK_TRACKER;
4625 
4626 	SDT_PROBE3(pf, eth, test_rule, entry, dir, kif->pfik_ifp, m);
4627 
4628 	mtag = pf_find_mtag(m);
4629 	if (mtag != NULL && mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
4630 		/* Dummynet re-injects packets after they've
4631 		 * completed their delay. We've already
4632 		 * processed them, so pass unconditionally. */
4633 
4634 		/* But only once. We may see the packet multiple times (e.g.
4635 		 * PFIL_IN/PFIL_OUT). */
4636 		pf_dummynet_flag_remove(m, mtag);
4637 
4638 		return (PF_PASS);
4639 	}
4640 
4641 	ruleset = V_pf_keth;
4642 	rules = ck_pr_load_ptr(&ruleset->active.rules);
4643 	r = TAILQ_FIRST(rules);
4644 	rm = NULL;
4645 
4646 	if (__predict_false(m->m_len < sizeof(struct ether_header)) &&
4647 	    (m = *m0 = m_pullup(*m0, sizeof(struct ether_header))) == NULL) {
4648 		DPFPRINTF(PF_DEBUG_URGENT,
4649 		    ("pf_test_eth_rule: m_len < sizeof(struct ether_header)"
4650 		     ", pullup failed\n"));
4651 		return (PF_DROP);
4652 	}
4653 	e = mtod(m, struct ether_header *);
4654 	proto = ntohs(e->ether_type);
4655 
4656 	switch (proto) {
4657 #ifdef INET
4658 	case ETHERTYPE_IP: {
4659 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
4660 		    sizeof(ip)))
4661 			return (PF_DROP);
4662 
4663 		af = AF_INET;
4664 		m_copydata(m, sizeof(struct ether_header), sizeof(ip),
4665 		    (caddr_t)&ip);
4666 		src = (struct pf_addr *)&ip.ip_src;
4667 		dst = (struct pf_addr *)&ip.ip_dst;
4668 		break;
4669 	}
4670 #endif /* INET */
4671 #ifdef INET6
4672 	case ETHERTYPE_IPV6: {
4673 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
4674 		    sizeof(ip6)))
4675 			return (PF_DROP);
4676 
4677 		af = AF_INET6;
4678 		m_copydata(m, sizeof(struct ether_header), sizeof(ip6),
4679 		    (caddr_t)&ip6);
4680 		src = (struct pf_addr *)&ip6.ip6_src;
4681 		dst = (struct pf_addr *)&ip6.ip6_dst;
4682 		break;
4683 	}
4684 #endif /* INET6 */
4685 	}
4686 
4687 	PF_RULES_RLOCK();
4688 
4689 	while (r != NULL) {
4690 		counter_u64_add(r->evaluations, 1);
4691 		SDT_PROBE2(pf, eth, test_rule, test, r->nr, r);
4692 
4693 		if (pfi_kkif_match(r->kif, kif) == r->ifnot) {
4694 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4695 			    "kif");
4696 			r = r->skip[PFE_SKIP_IFP].ptr;
4697 		}
4698 		else if (r->direction && r->direction != dir) {
4699 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4700 			    "dir");
4701 			r = r->skip[PFE_SKIP_DIR].ptr;
4702 		}
4703 		else if (r->proto && r->proto != proto) {
4704 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4705 			    "proto");
4706 			r = r->skip[PFE_SKIP_PROTO].ptr;
4707 		}
4708 		else if (! pf_match_eth_addr(e->ether_shost, &r->src)) {
4709 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4710 			    "src");
4711 			r = r->skip[PFE_SKIP_SRC_ADDR].ptr;
4712 		}
4713 		else if (! pf_match_eth_addr(e->ether_dhost, &r->dst)) {
4714 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4715 			    "dst");
4716 			r = r->skip[PFE_SKIP_DST_ADDR].ptr;
4717 		}
4718 		else if (src != NULL && PF_MISMATCHAW(&r->ipsrc.addr, src, af,
4719 		    r->ipsrc.neg, kif, M_GETFIB(m))) {
4720 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4721 			    "ip_src");
4722 			r = r->skip[PFE_SKIP_SRC_IP_ADDR].ptr;
4723 		}
4724 		else if (dst != NULL && PF_MISMATCHAW(&r->ipdst.addr, dst, af,
4725 		    r->ipdst.neg, kif, M_GETFIB(m))) {
4726 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4727 			    "ip_dst");
4728 			r = r->skip[PFE_SKIP_DST_IP_ADDR].ptr;
4729 		}
4730 		else if (r->match_tag && !pf_match_eth_tag(m, r, &tag,
4731 		    mtag ? mtag->tag : 0)) {
4732 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4733 			    "match_tag");
4734 			r = TAILQ_NEXT(r, entries);
4735 		}
4736 		else {
4737 			if (r->tag)
4738 				tag = r->tag;
4739 			if (r->anchor == NULL) {
4740 				/* Rule matches */
4741 				rm = r;
4742 
4743 				SDT_PROBE2(pf, eth, test_rule, match, r->nr, r);
4744 
4745 				if (r->quick)
4746 					break;
4747 
4748 				r = TAILQ_NEXT(r, entries);
4749 			} else {
4750 				pf_step_into_keth_anchor(anchor_stack, &asd,
4751 				    &ruleset, &r, &a, &match);
4752 			}
4753 		}
4754 		if (r == NULL && pf_step_out_of_keth_anchor(anchor_stack, &asd,
4755 		    &ruleset, &r, &a, &match))
4756 			break;
4757 	}
4758 
4759 	r = rm;
4760 
4761 	SDT_PROBE2(pf, eth, test_rule, final_match, (r != NULL ? r->nr : -1), r);
4762 
4763 	/* Default to pass. */
4764 	if (r == NULL) {
4765 		PF_RULES_RUNLOCK();
4766 		return (PF_PASS);
4767 	}
4768 
4769 	/* Execute action. */
4770 	counter_u64_add(r->packets[dir == PF_OUT], 1);
4771 	counter_u64_add(r->bytes[dir == PF_OUT], m_length(m, NULL));
4772 	pf_update_timestamp(r);
4773 
4774 	/* Shortcut. Don't tag if we're just going to drop anyway. */
4775 	if (r->action == PF_DROP) {
4776 		PF_RULES_RUNLOCK();
4777 		return (PF_DROP);
4778 	}
4779 
4780 	if (tag > 0) {
4781 		if (mtag == NULL)
4782 			mtag = pf_get_mtag(m);
4783 		if (mtag == NULL) {
4784 			PF_RULES_RUNLOCK();
4785 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4786 			return (PF_DROP);
4787 		}
4788 		mtag->tag = tag;
4789 	}
4790 
4791 	if (r->qid != 0) {
4792 		if (mtag == NULL)
4793 			mtag = pf_get_mtag(m);
4794 		if (mtag == NULL) {
4795 			PF_RULES_RUNLOCK();
4796 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4797 			return (PF_DROP);
4798 		}
4799 		mtag->qid = r->qid;
4800 	}
4801 
4802 	action = r->action;
4803 	bridge_to = r->bridge_to;
4804 
4805 	/* Dummynet */
4806 	if (r->dnpipe) {
4807 		struct ip_fw_args dnflow;
4808 
4809 		/* Drop packet if dummynet is not loaded. */
4810 		if (ip_dn_io_ptr == NULL) {
4811 			PF_RULES_RUNLOCK();
4812 			m_freem(m);
4813 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4814 			return (PF_DROP);
4815 		}
4816 		if (mtag == NULL)
4817 			mtag = pf_get_mtag(m);
4818 		if (mtag == NULL) {
4819 			PF_RULES_RUNLOCK();
4820 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4821 			return (PF_DROP);
4822 		}
4823 
4824 		bzero(&dnflow, sizeof(dnflow));
4825 
4826 		/* We don't have port numbers here, so we set 0.  That means
4827 		 * that we'll be somewhat limited in distinguishing flows (i.e.
4828 		 * only based on IP addresses, not based on port numbers), but
4829 		 * it's better than nothing. */
4830 		dnflow.f_id.dst_port = 0;
4831 		dnflow.f_id.src_port = 0;
4832 		dnflow.f_id.proto = 0;
4833 
4834 		dnflow.rule.info = r->dnpipe;
4835 		dnflow.rule.info |= IPFW_IS_DUMMYNET;
4836 		if (r->dnflags & PFRULE_DN_IS_PIPE)
4837 			dnflow.rule.info |= IPFW_IS_PIPE;
4838 
4839 		dnflow.f_id.extra = dnflow.rule.info;
4840 
4841 		dnflow.flags = dir == PF_IN ? IPFW_ARGS_IN : IPFW_ARGS_OUT;
4842 		dnflow.flags |= IPFW_ARGS_ETHER;
4843 		dnflow.ifp = kif->pfik_ifp;
4844 
4845 		switch (af) {
4846 		case AF_INET:
4847 			dnflow.f_id.addr_type = 4;
4848 			dnflow.f_id.src_ip = src->v4.s_addr;
4849 			dnflow.f_id.dst_ip = dst->v4.s_addr;
4850 			break;
4851 		case AF_INET6:
4852 			dnflow.flags |= IPFW_ARGS_IP6;
4853 			dnflow.f_id.addr_type = 6;
4854 			dnflow.f_id.src_ip6 = src->v6;
4855 			dnflow.f_id.dst_ip6 = dst->v6;
4856 			break;
4857 		}
4858 
4859 		PF_RULES_RUNLOCK();
4860 
4861 		mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
4862 		ip_dn_io_ptr(m0, &dnflow);
4863 		if (*m0 != NULL)
4864 			pf_dummynet_flag_remove(m, mtag);
4865 	} else {
4866 		PF_RULES_RUNLOCK();
4867 	}
4868 
4869 	if (action == PF_PASS && bridge_to) {
4870 		pf_bridge_to(bridge_to->pfik_ifp, *m0);
4871 		*m0 = NULL; /* We've eaten the packet. */
4872 	}
4873 
4874 	return (action);
4875 }
4876 
4877 #define PF_TEST_ATTRIB(t, a)\
4878 	do {				\
4879 		if (t) {		\
4880 			r = a;		\
4881 			goto nextrule;	\
4882 		}			\
4883 	} while (0)
4884 
4885 static int
4886 pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, struct pfi_kkif *kif,
4887     struct mbuf *m, int off, struct pf_pdesc *pd, struct pf_krule **am,
4888     struct pf_kruleset **rsm, struct inpcb *inp, int hdrlen)
4889 {
4890 	struct pf_krule		*nr = NULL;
4891 	struct pf_addr		* const saddr = pd->src;
4892 	struct pf_addr		* const daddr = pd->dst;
4893 	sa_family_t		 af = pd->af;
4894 	struct pf_krule		*r, *a = NULL;
4895 	struct pf_kruleset	*ruleset = NULL;
4896 	struct pf_krule_slist	 match_rules;
4897 	struct pf_krule_item	*ri;
4898 	struct pf_ksrc_node	*nsn = NULL;
4899 	struct tcphdr		*th = &pd->hdr.tcp;
4900 	struct pf_state_key	*sk = NULL, *nk = NULL;
4901 	u_short			 reason, transerror;
4902 	int			 rewrite = 0;
4903 	int			 tag = -1;
4904 	int			 asd = 0;
4905 	int			 match = 0;
4906 	int			 state_icmp = 0, icmp_dir, multi;
4907 	u_int16_t		 sport = 0, dport = 0, virtual_type, virtual_id;
4908 	u_int16_t		 bproto_sum = 0, bip_sum = 0;
4909 	u_int8_t		 icmptype = 0, icmpcode = 0;
4910 	struct pf_kanchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
4911 	struct pf_udp_mapping	*udp_mapping = NULL;
4912 
4913 	PF_RULES_RASSERT();
4914 
4915 	SLIST_INIT(&match_rules);
4916 
4917 	if (inp != NULL) {
4918 		INP_LOCK_ASSERT(inp);
4919 		pd->lookup.uid = inp->inp_cred->cr_uid;
4920 		pd->lookup.gid = inp->inp_cred->cr_groups[0];
4921 		pd->lookup.done = 1;
4922 	}
4923 
4924 	switch (pd->virtual_proto) {
4925 	case IPPROTO_TCP:
4926 		sport = th->th_sport;
4927 		dport = th->th_dport;
4928 		break;
4929 	case IPPROTO_UDP:
4930 		sport = pd->hdr.udp.uh_sport;
4931 		dport = pd->hdr.udp.uh_dport;
4932 		break;
4933 	case IPPROTO_SCTP:
4934 		sport = pd->hdr.sctp.src_port;
4935 		dport = pd->hdr.sctp.dest_port;
4936 		break;
4937 #ifdef INET
4938 	case IPPROTO_ICMP:
4939 		MPASS(af == AF_INET);
4940 		icmptype = pd->hdr.icmp.icmp_type;
4941 		icmpcode = pd->hdr.icmp.icmp_code;
4942 		state_icmp = pf_icmp_mapping(pd, icmptype,
4943 		    &icmp_dir, &multi, &virtual_id, &virtual_type);
4944 		if (icmp_dir == PF_IN) {
4945 			sport = virtual_id;
4946 			dport = virtual_type;
4947 		} else {
4948 			sport = virtual_type;
4949 			dport = virtual_id;
4950 		}
4951 		break;
4952 #endif /* INET */
4953 #ifdef INET6
4954 	case IPPROTO_ICMPV6:
4955 		MPASS(af == AF_INET6);
4956 		icmptype = pd->hdr.icmp6.icmp6_type;
4957 		icmpcode = pd->hdr.icmp6.icmp6_code;
4958 		state_icmp = pf_icmp_mapping(pd, icmptype,
4959 		    &icmp_dir, &multi, &virtual_id, &virtual_type);
4960 		if (icmp_dir == PF_IN) {
4961 			sport = virtual_id;
4962 			dport = virtual_type;
4963 		} else {
4964 			sport = virtual_type;
4965 			dport = virtual_id;
4966 		}
4967 
4968 		break;
4969 #endif /* INET6 */
4970 	default:
4971 		sport = dport = 0;
4972 		break;
4973 	}
4974 
4975 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
4976 
4977 	/* check packet for BINAT/NAT/RDR */
4978 	transerror = pf_get_translation(pd, m, off, kif, &nsn, &sk,
4979 	    &nk, saddr, daddr, sport, dport, anchor_stack, &nr, &udp_mapping);
4980 	switch (transerror) {
4981 	default:
4982 		/* A translation error occurred. */
4983 		REASON_SET(&reason, transerror);
4984 		goto cleanup;
4985 	case PFRES_MAX:
4986 		/* No match. */
4987 		break;
4988 	case PFRES_MATCH:
4989 		KASSERT(sk != NULL, ("%s: null sk", __func__));
4990 		KASSERT(nk != NULL, ("%s: null nk", __func__));
4991 
4992 		if (nr->log) {
4993 			PFLOG_PACKET(kif, m, PF_PASS, PFRES_MATCH, nr, a,
4994 			    ruleset, pd, 1);
4995 		}
4996 
4997 		if (pd->ip_sum)
4998 			bip_sum = *pd->ip_sum;
4999 
5000 		switch (pd->proto) {
5001 		case IPPROTO_TCP:
5002 			bproto_sum = th->th_sum;
5003 			pd->proto_sum = &th->th_sum;
5004 
5005 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
5006 			    nk->port[pd->sidx] != sport) {
5007 				pf_change_ap(m, saddr, &th->th_sport, pd->ip_sum,
5008 				    &th->th_sum, &nk->addr[pd->sidx],
5009 				    nk->port[pd->sidx], 0, af);
5010 				pd->sport = &th->th_sport;
5011 				sport = th->th_sport;
5012 			}
5013 
5014 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
5015 			    nk->port[pd->didx] != dport) {
5016 				pf_change_ap(m, daddr, &th->th_dport, pd->ip_sum,
5017 				    &th->th_sum, &nk->addr[pd->didx],
5018 				    nk->port[pd->didx], 0, af);
5019 				dport = th->th_dport;
5020 				pd->dport = &th->th_dport;
5021 			}
5022 			rewrite++;
5023 			break;
5024 		case IPPROTO_UDP:
5025 			bproto_sum = pd->hdr.udp.uh_sum;
5026 			pd->proto_sum = &pd->hdr.udp.uh_sum;
5027 
5028 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
5029 			    nk->port[pd->sidx] != sport) {
5030 				pf_change_ap(m, saddr, &pd->hdr.udp.uh_sport,
5031 				    pd->ip_sum, &pd->hdr.udp.uh_sum,
5032 				    &nk->addr[pd->sidx],
5033 				    nk->port[pd->sidx], 1, af);
5034 				sport = pd->hdr.udp.uh_sport;
5035 				pd->sport = &pd->hdr.udp.uh_sport;
5036 			}
5037 
5038 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
5039 			    nk->port[pd->didx] != dport) {
5040 				pf_change_ap(m, daddr, &pd->hdr.udp.uh_dport,
5041 				    pd->ip_sum, &pd->hdr.udp.uh_sum,
5042 				    &nk->addr[pd->didx],
5043 				    nk->port[pd->didx], 1, af);
5044 				dport = pd->hdr.udp.uh_dport;
5045 				pd->dport = &pd->hdr.udp.uh_dport;
5046 			}
5047 			rewrite++;
5048 			break;
5049 		case IPPROTO_SCTP: {
5050 			uint16_t checksum = 0;
5051 
5052 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
5053 			    nk->port[pd->sidx] != sport) {
5054 				pf_change_ap(m, saddr, &pd->hdr.sctp.src_port,
5055 				    pd->ip_sum, &checksum,
5056 				    &nk->addr[pd->sidx],
5057 				    nk->port[pd->sidx], 1, af);
5058 			}
5059 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
5060 			    nk->port[pd->didx] != dport) {
5061 				pf_change_ap(m, daddr, &pd->hdr.sctp.dest_port,
5062 				    pd->ip_sum, &checksum,
5063 				    &nk->addr[pd->didx],
5064 				    nk->port[pd->didx], 1, af);
5065 			}
5066 			break;
5067 		}
5068 #ifdef INET
5069 		case IPPROTO_ICMP:
5070 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET))
5071 				pf_change_a(&saddr->v4.s_addr, pd->ip_sum,
5072 				    nk->addr[pd->sidx].v4.s_addr, 0);
5073 
5074 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET))
5075 				pf_change_a(&daddr->v4.s_addr, pd->ip_sum,
5076 				    nk->addr[pd->didx].v4.s_addr, 0);
5077 
5078 			if (virtual_type == htons(ICMP_ECHO) &&
5079 			     nk->port[pd->sidx] != pd->hdr.icmp.icmp_id) {
5080 				pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
5081 				    pd->hdr.icmp.icmp_cksum, sport,
5082 				    nk->port[pd->sidx], 0);
5083 				pd->hdr.icmp.icmp_id = nk->port[pd->sidx];
5084 				pd->sport = &pd->hdr.icmp.icmp_id;
5085 			}
5086 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
5087 			break;
5088 #endif /* INET */
5089 #ifdef INET6
5090 		case IPPROTO_ICMPV6:
5091 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET6))
5092 				pf_change_a6(saddr, &pd->hdr.icmp6.icmp6_cksum,
5093 				    &nk->addr[pd->sidx], 0);
5094 
5095 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET6))
5096 				pf_change_a6(daddr, &pd->hdr.icmp6.icmp6_cksum,
5097 				    &nk->addr[pd->didx], 0);
5098 			rewrite++;
5099 			break;
5100 #endif /* INET */
5101 		default:
5102 			switch (af) {
5103 #ifdef INET
5104 			case AF_INET:
5105 				if (PF_ANEQ(saddr,
5106 				    &nk->addr[pd->sidx], AF_INET))
5107 					pf_change_a(&saddr->v4.s_addr,
5108 					    pd->ip_sum,
5109 					    nk->addr[pd->sidx].v4.s_addr, 0);
5110 
5111 				if (PF_ANEQ(daddr,
5112 				    &nk->addr[pd->didx], AF_INET))
5113 					pf_change_a(&daddr->v4.s_addr,
5114 					    pd->ip_sum,
5115 					    nk->addr[pd->didx].v4.s_addr, 0);
5116 				break;
5117 #endif /* INET */
5118 #ifdef INET6
5119 			case AF_INET6:
5120 				if (PF_ANEQ(saddr,
5121 				    &nk->addr[pd->sidx], AF_INET6))
5122 					PF_ACPY(saddr, &nk->addr[pd->sidx], af);
5123 
5124 				if (PF_ANEQ(daddr,
5125 				    &nk->addr[pd->didx], AF_INET6))
5126 					PF_ACPY(daddr, &nk->addr[pd->didx], af);
5127 				break;
5128 #endif /* INET */
5129 			}
5130 			break;
5131 		}
5132 		if (nr->natpass)
5133 			r = NULL;
5134 		pd->nat_rule = nr;
5135 	}
5136 
5137 	while (r != NULL) {
5138 		pf_counter_u64_add(&r->evaluations, 1);
5139 		PF_TEST_ATTRIB(pfi_kkif_match(r->kif, kif) == r->ifnot,
5140 			r->skip[PF_SKIP_IFP].ptr);
5141 		PF_TEST_ATTRIB(r->direction && r->direction != pd->dir,
5142 			r->skip[PF_SKIP_DIR].ptr);
5143 		PF_TEST_ATTRIB(r->af && r->af != af,
5144 			r->skip[PF_SKIP_AF].ptr);
5145 		PF_TEST_ATTRIB(r->proto && r->proto != pd->proto,
5146 			r->skip[PF_SKIP_PROTO].ptr);
5147 		PF_TEST_ATTRIB(PF_MISMATCHAW(&r->src.addr, saddr, af,
5148 		    r->src.neg, kif, M_GETFIB(m)),
5149 			r->skip[PF_SKIP_SRC_ADDR].ptr);
5150 		PF_TEST_ATTRIB(PF_MISMATCHAW(&r->dst.addr, daddr, af,
5151 		    r->dst.neg, NULL, M_GETFIB(m)),
5152 			r->skip[PF_SKIP_DST_ADDR].ptr);
5153 		switch (pd->virtual_proto) {
5154 		case PF_VPROTO_FRAGMENT:
5155 			/* tcp/udp only. port_op always 0 in other cases */
5156 			PF_TEST_ATTRIB((r->src.port_op || r->dst.port_op),
5157 				TAILQ_NEXT(r, entries));
5158 			PF_TEST_ATTRIB((pd->proto == IPPROTO_TCP && r->flagset),
5159 				TAILQ_NEXT(r, entries));
5160 			/* icmp only. type/code always 0 in other cases */
5161 			PF_TEST_ATTRIB((r->type || r->code),
5162 				TAILQ_NEXT(r, entries));
5163 			/* tcp/udp only. {uid|gid}.op always 0 in other cases */
5164 			PF_TEST_ATTRIB((r->gid.op || r->uid.op),
5165 				TAILQ_NEXT(r, entries));
5166 			break;
5167 
5168 		case IPPROTO_TCP:
5169 			PF_TEST_ATTRIB((r->flagset & th->th_flags) != r->flags,
5170 				TAILQ_NEXT(r, entries));
5171 			/* FALLTHROUGH */
5172 		case IPPROTO_SCTP:
5173 		case IPPROTO_UDP:
5174 			/* tcp/udp only. port_op always 0 in other cases */
5175 			PF_TEST_ATTRIB(r->src.port_op && !pf_match_port(r->src.port_op,
5176 			    r->src.port[0], r->src.port[1], sport),
5177 				r->skip[PF_SKIP_SRC_PORT].ptr);
5178 			/* tcp/udp only. port_op always 0 in other cases */
5179 			PF_TEST_ATTRIB(r->dst.port_op && !pf_match_port(r->dst.port_op,
5180 			    r->dst.port[0], r->dst.port[1], dport),
5181 				r->skip[PF_SKIP_DST_PORT].ptr);
5182 			/* tcp/udp only. uid.op always 0 in other cases */
5183 			PF_TEST_ATTRIB(r->uid.op && (pd->lookup.done || (pd->lookup.done =
5184 			    pf_socket_lookup(pd, m), 1)) &&
5185 			    !pf_match_uid(r->uid.op, r->uid.uid[0], r->uid.uid[1],
5186 			    pd->lookup.uid),
5187 				TAILQ_NEXT(r, entries));
5188 			/* tcp/udp only. gid.op always 0 in other cases */
5189 			PF_TEST_ATTRIB(r->gid.op && (pd->lookup.done || (pd->lookup.done =
5190 			    pf_socket_lookup(pd, m), 1)) &&
5191 			    !pf_match_gid(r->gid.op, r->gid.gid[0], r->gid.gid[1],
5192 			    pd->lookup.gid),
5193 				TAILQ_NEXT(r, entries));
5194 			break;
5195 
5196 		case IPPROTO_ICMP:
5197 		case IPPROTO_ICMPV6:
5198 			/* icmp only. type always 0 in other cases */
5199 			PF_TEST_ATTRIB(r->type && r->type != icmptype + 1,
5200 				TAILQ_NEXT(r, entries));
5201 			/* icmp only. type always 0 in other cases */
5202 			PF_TEST_ATTRIB(r->code && r->code != icmpcode + 1,
5203 				TAILQ_NEXT(r, entries));
5204 			break;
5205 
5206 		default:
5207 			break;
5208 		}
5209 		PF_TEST_ATTRIB(r->tos && !(r->tos == pd->tos),
5210 			TAILQ_NEXT(r, entries));
5211 		PF_TEST_ATTRIB(r->prio &&
5212 		    !pf_match_ieee8021q_pcp(r->prio, m),
5213 			TAILQ_NEXT(r, entries));
5214 		PF_TEST_ATTRIB(r->prob &&
5215 		    r->prob <= arc4random(),
5216 			TAILQ_NEXT(r, entries));
5217 		PF_TEST_ATTRIB(r->match_tag && !pf_match_tag(m, r, &tag,
5218 		    pd->pf_mtag ? pd->pf_mtag->tag : 0),
5219 			TAILQ_NEXT(r, entries));
5220 		PF_TEST_ATTRIB(r->rcv_kif && !pf_match_rcvif(m, r),
5221 			TAILQ_NEXT(r, entries));
5222 		PF_TEST_ATTRIB((r->rule_flag & PFRULE_FRAGMENT &&
5223 		    pd->virtual_proto != PF_VPROTO_FRAGMENT),
5224 			TAILQ_NEXT(r, entries));
5225 		PF_TEST_ATTRIB(r->os_fingerprint != PF_OSFP_ANY &&
5226 		    (pd->virtual_proto != IPPROTO_TCP || !pf_osfp_match(
5227 		    pf_osfp_fingerprint(pd, m, off, th),
5228 		    r->os_fingerprint)),
5229 			TAILQ_NEXT(r, entries));
5230 		/* FALLTHROUGH */
5231 		if (r->tag)
5232 			tag = r->tag;
5233 		if (r->anchor == NULL) {
5234 			if (r->action == PF_MATCH) {
5235 				ri = malloc(sizeof(struct pf_krule_item), M_PF_RULE_ITEM, M_NOWAIT | M_ZERO);
5236 				if (ri == NULL) {
5237 					REASON_SET(&reason, PFRES_MEMORY);
5238 					goto cleanup;
5239 				}
5240 				ri->r = r;
5241 				SLIST_INSERT_HEAD(&match_rules, ri, entry);
5242 				pf_counter_u64_critical_enter();
5243 				pf_counter_u64_add_protected(&r->packets[pd->dir == PF_OUT], 1);
5244 				pf_counter_u64_add_protected(&r->bytes[pd->dir == PF_OUT], pd->tot_len);
5245 				pf_counter_u64_critical_exit();
5246 				pf_rule_to_actions(r, &pd->act);
5247 				if (r->log || pd->act.log & PF_LOG_MATCHES)
5248 					PFLOG_PACKET(kif, m,
5249 					    r->action, PFRES_MATCH, r,
5250 					    a, ruleset, pd, 1);
5251 			} else {
5252 				match = 1;
5253 				*rm = r;
5254 				*am = a;
5255 				*rsm = ruleset;
5256 				if (pd->act.log & PF_LOG_MATCHES)
5257 					PFLOG_PACKET(kif, m,
5258 					    r->action, PFRES_MATCH, r,
5259 					    a, ruleset, pd, 1);
5260 			}
5261 			if ((*rm)->quick)
5262 				break;
5263 			r = TAILQ_NEXT(r, entries);
5264 		} else
5265 			pf_step_into_anchor(anchor_stack, &asd,
5266 			    &ruleset, PF_RULESET_FILTER, &r, &a,
5267 			    &match);
5268 nextrule:
5269 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
5270 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
5271 			break;
5272 	}
5273 	r = *rm;
5274 	a = *am;
5275 	ruleset = *rsm;
5276 
5277 	REASON_SET(&reason, PFRES_MATCH);
5278 
5279 	/* apply actions for last matching pass/block rule */
5280 	pf_rule_to_actions(r, &pd->act);
5281 
5282 	if (r->log || pd->act.log & PF_LOG_MATCHES) {
5283 		if (rewrite)
5284 			m_copyback(m, off, hdrlen, pd->hdr.any);
5285 		PFLOG_PACKET(kif, m, r->action, reason, r, a, ruleset, pd, 1);
5286 	}
5287 
5288 	if (pd->virtual_proto != PF_VPROTO_FRAGMENT &&
5289 	   (r->action == PF_DROP) &&
5290 	    ((r->rule_flag & PFRULE_RETURNRST) ||
5291 	    (r->rule_flag & PFRULE_RETURNICMP) ||
5292 	    (r->rule_flag & PFRULE_RETURN))) {
5293 		pf_return(r, nr, pd, sk, off, m, th, kif, bproto_sum,
5294 		    bip_sum, hdrlen, &reason, r->rtableid);
5295 	}
5296 
5297 	if (r->action == PF_DROP)
5298 		goto cleanup;
5299 
5300 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
5301 		REASON_SET(&reason, PFRES_MEMORY);
5302 		goto cleanup;
5303 	}
5304 	if (pd->act.rtableid >= 0)
5305 		M_SETFIB(m, pd->act.rtableid);
5306 
5307 	if (pd->virtual_proto != PF_VPROTO_FRAGMENT &&
5308 	   (!state_icmp && (r->keep_state || nr != NULL ||
5309 	    (pd->flags & PFDESC_TCP_NORM)))) {
5310 		int action;
5311 		action = pf_create_state(r, nr, a, pd, nsn, nk, sk, m, off,
5312 		    sport, dport, &rewrite, kif, sm, tag, bproto_sum, bip_sum,
5313 		    hdrlen, &match_rules, udp_mapping);
5314 		if (action != PF_PASS) {
5315 			pf_udp_mapping_release(udp_mapping);
5316 			if (action == PF_DROP &&
5317 			    (r->rule_flag & PFRULE_RETURN))
5318 				pf_return(r, nr, pd, sk, off, m, th, kif,
5319 				    bproto_sum, bip_sum, hdrlen, &reason,
5320 				    pd->act.rtableid);
5321 			return (action);
5322 		}
5323 	} else {
5324 		while ((ri = SLIST_FIRST(&match_rules))) {
5325 			SLIST_REMOVE_HEAD(&match_rules, entry);
5326 			free(ri, M_PF_RULE_ITEM);
5327 		}
5328 
5329 		uma_zfree(V_pf_state_key_z, sk);
5330 		uma_zfree(V_pf_state_key_z, nk);
5331 		pf_udp_mapping_release(udp_mapping);
5332 	}
5333 
5334 	/* copy back packet headers if we performed NAT operations */
5335 	if (rewrite)
5336 		m_copyback(m, off, hdrlen, pd->hdr.any);
5337 
5338 	if (*sm != NULL && !((*sm)->state_flags & PFSTATE_NOSYNC) &&
5339 	    pd->dir == PF_OUT &&
5340 	    V_pfsync_defer_ptr != NULL && V_pfsync_defer_ptr(*sm, m))
5341 		/*
5342 		 * We want the state created, but we dont
5343 		 * want to send this in case a partner
5344 		 * firewall has to know about it to allow
5345 		 * replies through it.
5346 		 */
5347 		return (PF_DEFER);
5348 
5349 	return (PF_PASS);
5350 
5351 cleanup:
5352 	while ((ri = SLIST_FIRST(&match_rules))) {
5353 		SLIST_REMOVE_HEAD(&match_rules, entry);
5354 		free(ri, M_PF_RULE_ITEM);
5355 	}
5356 
5357 	uma_zfree(V_pf_state_key_z, sk);
5358 	uma_zfree(V_pf_state_key_z, nk);
5359 	pf_udp_mapping_release(udp_mapping);
5360 
5361 	return (PF_DROP);
5362 }
5363 
5364 static int
5365 pf_create_state(struct pf_krule *r, struct pf_krule *nr, struct pf_krule *a,
5366     struct pf_pdesc *pd, struct pf_ksrc_node *nsn, struct pf_state_key *nk,
5367     struct pf_state_key *sk, struct mbuf *m, int off, u_int16_t sport,
5368     u_int16_t dport, int *rewrite, struct pfi_kkif *kif, struct pf_kstate **sm,
5369     int tag, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen,
5370     struct pf_krule_slist *match_rules, struct pf_udp_mapping *udp_mapping)
5371 {
5372 	struct pf_kstate	*s = NULL;
5373 	struct pf_ksrc_node	*sn = NULL;
5374 	struct tcphdr		*th = &pd->hdr.tcp;
5375 	u_int16_t		 mss = V_tcp_mssdflt;
5376 	u_short			 reason, sn_reason;
5377 	struct pf_krule_item	*ri;
5378 
5379 	/* check maximums */
5380 	if (r->max_states &&
5381 	    (counter_u64_fetch(r->states_cur) >= r->max_states)) {
5382 		counter_u64_add(V_pf_status.lcounters[LCNT_STATES], 1);
5383 		REASON_SET(&reason, PFRES_MAXSTATES);
5384 		goto csfailed;
5385 	}
5386 	/* src node for filter rule */
5387 	if ((r->rule_flag & PFRULE_SRCTRACK ||
5388 	    r->rpool.opts & PF_POOL_STICKYADDR) &&
5389 	    (sn_reason = pf_insert_src_node(&sn, r, pd->src, pd->af)) != 0) {
5390 		REASON_SET(&reason, sn_reason);
5391 		goto csfailed;
5392 	}
5393 	/* src node for translation rule */
5394 	if (nr != NULL && (nr->rpool.opts & PF_POOL_STICKYADDR) &&
5395 	    (sn_reason = pf_insert_src_node(&nsn, nr, &sk->addr[pd->sidx],
5396 	    pd->af)) != 0 ) {
5397 		REASON_SET(&reason, sn_reason);
5398 		goto csfailed;
5399 	}
5400 	s = pf_alloc_state(M_NOWAIT);
5401 	if (s == NULL) {
5402 		REASON_SET(&reason, PFRES_MEMORY);
5403 		goto csfailed;
5404 	}
5405 	s->rule.ptr = r;
5406 	s->nat_rule.ptr = nr;
5407 	s->anchor.ptr = a;
5408 	bcopy(match_rules, &s->match_rules, sizeof(s->match_rules));
5409 	memcpy(&s->act, &pd->act, sizeof(struct pf_rule_actions));
5410 
5411 	STATE_INC_COUNTERS(s);
5412 	if (r->allow_opts)
5413 		s->state_flags |= PFSTATE_ALLOWOPTS;
5414 	if (r->rule_flag & PFRULE_STATESLOPPY)
5415 		s->state_flags |= PFSTATE_SLOPPY;
5416 	if (pd->flags & PFDESC_TCP_NORM) /* Set by old-style scrub rules */
5417 		s->state_flags |= PFSTATE_SCRUB_TCP;
5418 	if ((r->rule_flag & PFRULE_PFLOW) ||
5419 	    (nr != NULL && nr->rule_flag & PFRULE_PFLOW))
5420 		s->state_flags |= PFSTATE_PFLOW;
5421 
5422 	s->act.log = pd->act.log & PF_LOG_ALL;
5423 	s->sync_state = PFSYNC_S_NONE;
5424 	s->state_flags |= pd->act.flags; /* Only needed for pfsync and state export */
5425 
5426 	if (nr != NULL)
5427 		s->act.log |= nr->log & PF_LOG_ALL;
5428 	switch (pd->proto) {
5429 	case IPPROTO_TCP:
5430 		s->src.seqlo = ntohl(th->th_seq);
5431 		s->src.seqhi = s->src.seqlo + pd->p_len + 1;
5432 		if ((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN &&
5433 		    r->keep_state == PF_STATE_MODULATE) {
5434 			/* Generate sequence number modulator */
5435 			if ((s->src.seqdiff = pf_tcp_iss(pd) - s->src.seqlo) ==
5436 			    0)
5437 				s->src.seqdiff = 1;
5438 			pf_change_proto_a(m, &th->th_seq, &th->th_sum,
5439 			    htonl(s->src.seqlo + s->src.seqdiff), 0);
5440 			*rewrite = 1;
5441 		} else
5442 			s->src.seqdiff = 0;
5443 		if (th->th_flags & TH_SYN) {
5444 			s->src.seqhi++;
5445 			s->src.wscale = pf_get_wscale(m, off,
5446 			    th->th_off, pd->af);
5447 		}
5448 		s->src.max_win = MAX(ntohs(th->th_win), 1);
5449 		if (s->src.wscale & PF_WSCALE_MASK) {
5450 			/* Remove scale factor from initial window */
5451 			int win = s->src.max_win;
5452 			win += 1 << (s->src.wscale & PF_WSCALE_MASK);
5453 			s->src.max_win = (win - 1) >>
5454 			    (s->src.wscale & PF_WSCALE_MASK);
5455 		}
5456 		if (th->th_flags & TH_FIN)
5457 			s->src.seqhi++;
5458 		s->dst.seqhi = 1;
5459 		s->dst.max_win = 1;
5460 		pf_set_protostate(s, PF_PEER_SRC, TCPS_SYN_SENT);
5461 		pf_set_protostate(s, PF_PEER_DST, TCPS_CLOSED);
5462 		s->timeout = PFTM_TCP_FIRST_PACKET;
5463 		atomic_add_32(&V_pf_status.states_halfopen, 1);
5464 		break;
5465 	case IPPROTO_UDP:
5466 		pf_set_protostate(s, PF_PEER_SRC, PFUDPS_SINGLE);
5467 		pf_set_protostate(s, PF_PEER_DST, PFUDPS_NO_TRAFFIC);
5468 		s->timeout = PFTM_UDP_FIRST_PACKET;
5469 		break;
5470 	case IPPROTO_SCTP:
5471 		pf_set_protostate(s, PF_PEER_SRC, SCTP_COOKIE_WAIT);
5472 		pf_set_protostate(s, PF_PEER_DST, SCTP_CLOSED);
5473 		s->timeout = PFTM_SCTP_FIRST_PACKET;
5474 		break;
5475 	case IPPROTO_ICMP:
5476 #ifdef INET6
5477 	case IPPROTO_ICMPV6:
5478 #endif
5479 		s->timeout = PFTM_ICMP_FIRST_PACKET;
5480 		break;
5481 	default:
5482 		pf_set_protostate(s, PF_PEER_SRC, PFOTHERS_SINGLE);
5483 		pf_set_protostate(s, PF_PEER_DST, PFOTHERS_NO_TRAFFIC);
5484 		s->timeout = PFTM_OTHER_FIRST_PACKET;
5485 	}
5486 
5487 	if (r->rt) {
5488 		/* pf_map_addr increases the reason counters */
5489 		if ((reason = pf_map_addr_sn(pd->af, r, pd->src, &s->rt_addr,
5490 		    &s->rt_kif, NULL, &sn)) != 0)
5491 			goto csfailed;
5492 		s->rt = r->rt;
5493 	}
5494 
5495 	s->creation = s->expire = pf_get_uptime();
5496 
5497 	if (sn != NULL)
5498 		s->src_node = sn;
5499 	if (nsn != NULL) {
5500 		/* XXX We only modify one side for now. */
5501 		PF_ACPY(&nsn->raddr, &nk->addr[1], pd->af);
5502 		s->nat_src_node = nsn;
5503 	}
5504 	if (pd->proto == IPPROTO_TCP) {
5505 		if (s->state_flags & PFSTATE_SCRUB_TCP &&
5506 		    pf_normalize_tcp_init(m, off, pd, th, &s->src, &s->dst)) {
5507 			REASON_SET(&reason, PFRES_MEMORY);
5508 			goto csfailed;
5509 		}
5510 		if (s->state_flags & PFSTATE_SCRUB_TCP && s->src.scrub &&
5511 		    pf_normalize_tcp_stateful(m, off, pd, &reason, th, s,
5512 		    &s->src, &s->dst, rewrite)) {
5513 			/* This really shouldn't happen!!! */
5514 			DPFPRINTF(PF_DEBUG_URGENT,
5515 			    ("pf_normalize_tcp_stateful failed on first "
5516 			     "pkt\n"));
5517 			goto csfailed;
5518 		}
5519 	} else if (pd->proto == IPPROTO_SCTP) {
5520 		if (pf_normalize_sctp_init(m, off, pd, &s->src, &s->dst))
5521 			goto csfailed;
5522 		if (! (pd->sctp_flags & (PFDESC_SCTP_INIT | PFDESC_SCTP_ADD_IP)))
5523 			goto csfailed;
5524 	}
5525 	s->direction = pd->dir;
5526 
5527 	/*
5528 	 * sk/nk could already been setup by pf_get_translation().
5529 	 */
5530 	if (nr == NULL) {
5531 		KASSERT((sk == NULL && nk == NULL), ("%s: nr %p sk %p, nk %p",
5532 		    __func__, nr, sk, nk));
5533 		sk = pf_state_key_setup(pd, m, off, pd->src, pd->dst, sport, dport);
5534 		if (sk == NULL)
5535 			goto csfailed;
5536 		nk = sk;
5537 	} else
5538 		KASSERT((sk != NULL && nk != NULL), ("%s: nr %p sk %p, nk %p",
5539 		    __func__, nr, sk, nk));
5540 
5541 	/* Swap sk/nk for PF_OUT. */
5542 	if (pf_state_insert(BOUND_IFACE(s, kif), kif,
5543 	    (pd->dir == PF_IN) ? sk : nk,
5544 	    (pd->dir == PF_IN) ? nk : sk, s)) {
5545 		REASON_SET(&reason, PFRES_STATEINS);
5546 		goto drop;
5547 	} else
5548 		*sm = s;
5549 
5550 	if (tag > 0)
5551 		s->tag = tag;
5552 	if (pd->proto == IPPROTO_TCP && (th->th_flags & (TH_SYN|TH_ACK)) ==
5553 	    TH_SYN && r->keep_state == PF_STATE_SYNPROXY) {
5554 		pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_SRC);
5555 		/* undo NAT changes, if they have taken place */
5556 		if (nr != NULL) {
5557 			struct pf_state_key *skt = s->key[PF_SK_WIRE];
5558 			if (pd->dir == PF_OUT)
5559 				skt = s->key[PF_SK_STACK];
5560 			PF_ACPY(pd->src, &skt->addr[pd->sidx], pd->af);
5561 			PF_ACPY(pd->dst, &skt->addr[pd->didx], pd->af);
5562 			if (pd->sport)
5563 				*pd->sport = skt->port[pd->sidx];
5564 			if (pd->dport)
5565 				*pd->dport = skt->port[pd->didx];
5566 			if (pd->proto_sum)
5567 				*pd->proto_sum = bproto_sum;
5568 			if (pd->ip_sum)
5569 				*pd->ip_sum = bip_sum;
5570 			m_copyback(m, off, hdrlen, pd->hdr.any);
5571 		}
5572 		s->src.seqhi = htonl(arc4random());
5573 		/* Find mss option */
5574 		int rtid = M_GETFIB(m);
5575 		mss = pf_get_mss(m, off, th->th_off, pd->af);
5576 		mss = pf_calc_mss(pd->src, pd->af, rtid, mss);
5577 		mss = pf_calc_mss(pd->dst, pd->af, rtid, mss);
5578 		s->src.mss = mss;
5579 		pf_send_tcp(r, pd->af, pd->dst, pd->src, th->th_dport,
5580 		    th->th_sport, s->src.seqhi, ntohl(th->th_seq) + 1,
5581 		    TH_SYN|TH_ACK, 0, s->src.mss, 0, true, 0, 0,
5582 		    pd->act.rtableid);
5583 		REASON_SET(&reason, PFRES_SYNPROXY);
5584 		return (PF_SYNPROXY_DROP);
5585 	}
5586 
5587 	s->udp_mapping = udp_mapping;
5588 
5589 	return (PF_PASS);
5590 
5591 csfailed:
5592 	while ((ri = SLIST_FIRST(match_rules))) {
5593 		SLIST_REMOVE_HEAD(match_rules, entry);
5594 		free(ri, M_PF_RULE_ITEM);
5595 	}
5596 
5597 	uma_zfree(V_pf_state_key_z, sk);
5598 	uma_zfree(V_pf_state_key_z, nk);
5599 
5600 	if (sn != NULL) {
5601 		PF_SRC_NODE_LOCK(sn);
5602 		if (--sn->states == 0 && sn->expire == 0) {
5603 			pf_unlink_src_node(sn);
5604 			uma_zfree(V_pf_sources_z, sn);
5605 			counter_u64_add(
5606 			    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
5607 		}
5608 		PF_SRC_NODE_UNLOCK(sn);
5609 	}
5610 
5611 	if (nsn != sn && nsn != NULL) {
5612 		PF_SRC_NODE_LOCK(nsn);
5613 		if (--nsn->states == 0 && nsn->expire == 0) {
5614 			pf_unlink_src_node(nsn);
5615 			uma_zfree(V_pf_sources_z, nsn);
5616 			counter_u64_add(
5617 			    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
5618 		}
5619 		PF_SRC_NODE_UNLOCK(nsn);
5620 	}
5621 
5622 drop:
5623 	if (s != NULL) {
5624 		pf_src_tree_remove_state(s);
5625 		s->timeout = PFTM_UNLINKED;
5626 		STATE_DEC_COUNTERS(s);
5627 		pf_free_state(s);
5628 	}
5629 
5630 	return (PF_DROP);
5631 }
5632 
5633 static int
5634 pf_tcp_track_full(struct pf_kstate **state, struct pfi_kkif *kif,
5635     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason,
5636     int *copyback)
5637 {
5638 	struct tcphdr		*th = &pd->hdr.tcp;
5639 	struct pf_state_peer	*src, *dst;
5640 	u_int16_t		 win = ntohs(th->th_win);
5641 	u_int32_t		 ack, end, data_end, seq, orig_seq;
5642 	u_int8_t		 sws, dws, psrc, pdst;
5643 	int			 ackskew;
5644 
5645 	if (pd->dir == (*state)->direction) {
5646 		src = &(*state)->src;
5647 		dst = &(*state)->dst;
5648 		psrc = PF_PEER_SRC;
5649 		pdst = PF_PEER_DST;
5650 	} else {
5651 		src = &(*state)->dst;
5652 		dst = &(*state)->src;
5653 		psrc = PF_PEER_DST;
5654 		pdst = PF_PEER_SRC;
5655 	}
5656 
5657 	if (src->wscale && dst->wscale && !(th->th_flags & TH_SYN)) {
5658 		sws = src->wscale & PF_WSCALE_MASK;
5659 		dws = dst->wscale & PF_WSCALE_MASK;
5660 	} else
5661 		sws = dws = 0;
5662 
5663 	/*
5664 	 * Sequence tracking algorithm from Guido van Rooij's paper:
5665 	 *   http://www.madison-gurkha.com/publications/tcp_filtering/
5666 	 *	tcp_filtering.ps
5667 	 */
5668 
5669 	orig_seq = seq = ntohl(th->th_seq);
5670 	if (src->seqlo == 0) {
5671 		/* First packet from this end. Set its state */
5672 
5673 		if (((*state)->state_flags & PFSTATE_SCRUB_TCP || dst->scrub) &&
5674 		    src->scrub == NULL) {
5675 			if (pf_normalize_tcp_init(m, off, pd, th, src, dst)) {
5676 				REASON_SET(reason, PFRES_MEMORY);
5677 				return (PF_DROP);
5678 			}
5679 		}
5680 
5681 		/* Deferred generation of sequence number modulator */
5682 		if (dst->seqdiff && !src->seqdiff) {
5683 			/* use random iss for the TCP server */
5684 			while ((src->seqdiff = arc4random() - seq) == 0)
5685 				;
5686 			ack = ntohl(th->th_ack) - dst->seqdiff;
5687 			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
5688 			    src->seqdiff), 0);
5689 			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
5690 			*copyback = 1;
5691 		} else {
5692 			ack = ntohl(th->th_ack);
5693 		}
5694 
5695 		end = seq + pd->p_len;
5696 		if (th->th_flags & TH_SYN) {
5697 			end++;
5698 			if (dst->wscale & PF_WSCALE_FLAG) {
5699 				src->wscale = pf_get_wscale(m, off, th->th_off,
5700 				    pd->af);
5701 				if (src->wscale & PF_WSCALE_FLAG) {
5702 					/* Remove scale factor from initial
5703 					 * window */
5704 					sws = src->wscale & PF_WSCALE_MASK;
5705 					win = ((u_int32_t)win + (1 << sws) - 1)
5706 					    >> sws;
5707 					dws = dst->wscale & PF_WSCALE_MASK;
5708 				} else {
5709 					/* fixup other window */
5710 					dst->max_win = MIN(TCP_MAXWIN,
5711 					    (u_int32_t)dst->max_win <<
5712 					    (dst->wscale & PF_WSCALE_MASK));
5713 					/* in case of a retrans SYN|ACK */
5714 					dst->wscale = 0;
5715 				}
5716 			}
5717 		}
5718 		data_end = end;
5719 		if (th->th_flags & TH_FIN)
5720 			end++;
5721 
5722 		src->seqlo = seq;
5723 		if (src->state < TCPS_SYN_SENT)
5724 			pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5725 
5726 		/*
5727 		 * May need to slide the window (seqhi may have been set by
5728 		 * the crappy stack check or if we picked up the connection
5729 		 * after establishment)
5730 		 */
5731 		if (src->seqhi == 1 ||
5732 		    SEQ_GEQ(end + MAX(1, dst->max_win << dws), src->seqhi))
5733 			src->seqhi = end + MAX(1, dst->max_win << dws);
5734 		if (win > src->max_win)
5735 			src->max_win = win;
5736 
5737 	} else {
5738 		ack = ntohl(th->th_ack) - dst->seqdiff;
5739 		if (src->seqdiff) {
5740 			/* Modulate sequence numbers */
5741 			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
5742 			    src->seqdiff), 0);
5743 			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
5744 			*copyback = 1;
5745 		}
5746 		end = seq + pd->p_len;
5747 		if (th->th_flags & TH_SYN)
5748 			end++;
5749 		data_end = end;
5750 		if (th->th_flags & TH_FIN)
5751 			end++;
5752 	}
5753 
5754 	if ((th->th_flags & TH_ACK) == 0) {
5755 		/* Let it pass through the ack skew check */
5756 		ack = dst->seqlo;
5757 	} else if ((ack == 0 &&
5758 	    (th->th_flags & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) ||
5759 	    /* broken tcp stacks do not set ack */
5760 	    (dst->state < TCPS_SYN_SENT)) {
5761 		/*
5762 		 * Many stacks (ours included) will set the ACK number in an
5763 		 * FIN|ACK if the SYN times out -- no sequence to ACK.
5764 		 */
5765 		ack = dst->seqlo;
5766 	}
5767 
5768 	if (seq == end) {
5769 		/* Ease sequencing restrictions on no data packets */
5770 		seq = src->seqlo;
5771 		data_end = end = seq;
5772 	}
5773 
5774 	ackskew = dst->seqlo - ack;
5775 
5776 	/*
5777 	 * Need to demodulate the sequence numbers in any TCP SACK options
5778 	 * (Selective ACK). We could optionally validate the SACK values
5779 	 * against the current ACK window, either forwards or backwards, but
5780 	 * I'm not confident that SACK has been implemented properly
5781 	 * everywhere. It wouldn't surprise me if several stacks accidentally
5782 	 * SACK too far backwards of previously ACKed data. There really aren't
5783 	 * any security implications of bad SACKing unless the target stack
5784 	 * doesn't validate the option length correctly. Someone trying to
5785 	 * spoof into a TCP connection won't bother blindly sending SACK
5786 	 * options anyway.
5787 	 */
5788 	if (dst->seqdiff && (th->th_off << 2) > sizeof(struct tcphdr)) {
5789 		if (pf_modulate_sack(m, off, pd, th, dst))
5790 			*copyback = 1;
5791 	}
5792 
5793 #define	MAXACKWINDOW (0xffff + 1500)	/* 1500 is an arbitrary fudge factor */
5794 	if (SEQ_GEQ(src->seqhi, data_end) &&
5795 	    /* Last octet inside other's window space */
5796 	    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) &&
5797 	    /* Retrans: not more than one window back */
5798 	    (ackskew >= -MAXACKWINDOW) &&
5799 	    /* Acking not more than one reassembled fragment backwards */
5800 	    (ackskew <= (MAXACKWINDOW << sws)) &&
5801 	    /* Acking not more than one window forward */
5802 	    ((th->th_flags & TH_RST) == 0 || orig_seq == src->seqlo ||
5803 	    (orig_seq == src->seqlo + 1) || (orig_seq + 1 == src->seqlo))) {
5804 	    /* Require an exact/+1 sequence match on resets when possible */
5805 
5806 		if (dst->scrub || src->scrub) {
5807 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
5808 			    *state, src, dst, copyback))
5809 				return (PF_DROP);
5810 		}
5811 
5812 		/* update max window */
5813 		if (src->max_win < win)
5814 			src->max_win = win;
5815 		/* synchronize sequencing */
5816 		if (SEQ_GT(end, src->seqlo))
5817 			src->seqlo = end;
5818 		/* slide the window of what the other end can send */
5819 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
5820 			dst->seqhi = ack + MAX((win << sws), 1);
5821 
5822 		/* update states */
5823 		if (th->th_flags & TH_SYN)
5824 			if (src->state < TCPS_SYN_SENT)
5825 				pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5826 		if (th->th_flags & TH_FIN)
5827 			if (src->state < TCPS_CLOSING)
5828 				pf_set_protostate(*state, psrc, TCPS_CLOSING);
5829 		if (th->th_flags & TH_ACK) {
5830 			if (dst->state == TCPS_SYN_SENT) {
5831 				pf_set_protostate(*state, pdst,
5832 				    TCPS_ESTABLISHED);
5833 				if (src->state == TCPS_ESTABLISHED &&
5834 				    (*state)->src_node != NULL &&
5835 				    pf_src_connlimit(state)) {
5836 					REASON_SET(reason, PFRES_SRCLIMIT);
5837 					return (PF_DROP);
5838 				}
5839 			} else if (dst->state == TCPS_CLOSING)
5840 				pf_set_protostate(*state, pdst,
5841 				    TCPS_FIN_WAIT_2);
5842 		}
5843 		if (th->th_flags & TH_RST)
5844 			pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
5845 
5846 		/* update expire time */
5847 		(*state)->expire = pf_get_uptime();
5848 		if (src->state >= TCPS_FIN_WAIT_2 &&
5849 		    dst->state >= TCPS_FIN_WAIT_2)
5850 			(*state)->timeout = PFTM_TCP_CLOSED;
5851 		else if (src->state >= TCPS_CLOSING &&
5852 		    dst->state >= TCPS_CLOSING)
5853 			(*state)->timeout = PFTM_TCP_FIN_WAIT;
5854 		else if (src->state < TCPS_ESTABLISHED ||
5855 		    dst->state < TCPS_ESTABLISHED)
5856 			(*state)->timeout = PFTM_TCP_OPENING;
5857 		else if (src->state >= TCPS_CLOSING ||
5858 		    dst->state >= TCPS_CLOSING)
5859 			(*state)->timeout = PFTM_TCP_CLOSING;
5860 		else
5861 			(*state)->timeout = PFTM_TCP_ESTABLISHED;
5862 
5863 		/* Fall through to PASS packet */
5864 
5865 	} else if ((dst->state < TCPS_SYN_SENT ||
5866 		dst->state >= TCPS_FIN_WAIT_2 ||
5867 		src->state >= TCPS_FIN_WAIT_2) &&
5868 	    SEQ_GEQ(src->seqhi + MAXACKWINDOW, data_end) &&
5869 	    /* Within a window forward of the originating packet */
5870 	    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW)) {
5871 	    /* Within a window backward of the originating packet */
5872 
5873 		/*
5874 		 * This currently handles three situations:
5875 		 *  1) Stupid stacks will shotgun SYNs before their peer
5876 		 *     replies.
5877 		 *  2) When PF catches an already established stream (the
5878 		 *     firewall rebooted, the state table was flushed, routes
5879 		 *     changed...)
5880 		 *  3) Packets get funky immediately after the connection
5881 		 *     closes (this should catch Solaris spurious ACK|FINs
5882 		 *     that web servers like to spew after a close)
5883 		 *
5884 		 * This must be a little more careful than the above code
5885 		 * since packet floods will also be caught here. We don't
5886 		 * update the TTL here to mitigate the damage of a packet
5887 		 * flood and so the same code can handle awkward establishment
5888 		 * and a loosened connection close.
5889 		 * In the establishment case, a correct peer response will
5890 		 * validate the connection, go through the normal state code
5891 		 * and keep updating the state TTL.
5892 		 */
5893 
5894 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
5895 			printf("pf: loose state match: ");
5896 			pf_print_state(*state);
5897 			pf_print_flags(th->th_flags);
5898 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
5899 			    "pkts=%llu:%llu dir=%s,%s\n", seq, orig_seq, ack,
5900 			    pd->p_len, ackskew, (unsigned long long)(*state)->packets[0],
5901 			    (unsigned long long)(*state)->packets[1],
5902 			    pd->dir == PF_IN ? "in" : "out",
5903 			    pd->dir == (*state)->direction ? "fwd" : "rev");
5904 		}
5905 
5906 		if (dst->scrub || src->scrub) {
5907 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
5908 			    *state, src, dst, copyback))
5909 				return (PF_DROP);
5910 		}
5911 
5912 		/* update max window */
5913 		if (src->max_win < win)
5914 			src->max_win = win;
5915 		/* synchronize sequencing */
5916 		if (SEQ_GT(end, src->seqlo))
5917 			src->seqlo = end;
5918 		/* slide the window of what the other end can send */
5919 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
5920 			dst->seqhi = ack + MAX((win << sws), 1);
5921 
5922 		/*
5923 		 * Cannot set dst->seqhi here since this could be a shotgunned
5924 		 * SYN and not an already established connection.
5925 		 */
5926 
5927 		if (th->th_flags & TH_FIN)
5928 			if (src->state < TCPS_CLOSING)
5929 				pf_set_protostate(*state, psrc, TCPS_CLOSING);
5930 		if (th->th_flags & TH_RST)
5931 			pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
5932 
5933 		/* Fall through to PASS packet */
5934 
5935 	} else {
5936 		if ((*state)->dst.state == TCPS_SYN_SENT &&
5937 		    (*state)->src.state == TCPS_SYN_SENT) {
5938 			/* Send RST for state mismatches during handshake */
5939 			if (!(th->th_flags & TH_RST))
5940 				pf_send_tcp((*state)->rule.ptr, pd->af,
5941 				    pd->dst, pd->src, th->th_dport,
5942 				    th->th_sport, ntohl(th->th_ack), 0,
5943 				    TH_RST, 0, 0,
5944 				    (*state)->rule.ptr->return_ttl, true, 0, 0,
5945 				    (*state)->act.rtableid);
5946 			src->seqlo = 0;
5947 			src->seqhi = 1;
5948 			src->max_win = 1;
5949 		} else if (V_pf_status.debug >= PF_DEBUG_MISC) {
5950 			printf("pf: BAD state: ");
5951 			pf_print_state(*state);
5952 			pf_print_flags(th->th_flags);
5953 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
5954 			    "pkts=%llu:%llu dir=%s,%s\n",
5955 			    seq, orig_seq, ack, pd->p_len, ackskew,
5956 			    (unsigned long long)(*state)->packets[0],
5957 			    (unsigned long long)(*state)->packets[1],
5958 			    pd->dir == PF_IN ? "in" : "out",
5959 			    pd->dir == (*state)->direction ? "fwd" : "rev");
5960 			printf("pf: State failure on: %c %c %c %c | %c %c\n",
5961 			    SEQ_GEQ(src->seqhi, data_end) ? ' ' : '1',
5962 			    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) ?
5963 			    ' ': '2',
5964 			    (ackskew >= -MAXACKWINDOW) ? ' ' : '3',
5965 			    (ackskew <= (MAXACKWINDOW << sws)) ? ' ' : '4',
5966 			    SEQ_GEQ(src->seqhi + MAXACKWINDOW, data_end) ?' ' :'5',
5967 			    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW) ?' ' :'6');
5968 		}
5969 		REASON_SET(reason, PFRES_BADSTATE);
5970 		return (PF_DROP);
5971 	}
5972 
5973 	return (PF_PASS);
5974 }
5975 
5976 static int
5977 pf_tcp_track_sloppy(struct pf_kstate **state, struct pf_pdesc *pd, u_short *reason)
5978 {
5979 	struct tcphdr		*th = &pd->hdr.tcp;
5980 	struct pf_state_peer	*src, *dst;
5981 	u_int8_t		 psrc, pdst;
5982 
5983 	if (pd->dir == (*state)->direction) {
5984 		src = &(*state)->src;
5985 		dst = &(*state)->dst;
5986 		psrc = PF_PEER_SRC;
5987 		pdst = PF_PEER_DST;
5988 	} else {
5989 		src = &(*state)->dst;
5990 		dst = &(*state)->src;
5991 		psrc = PF_PEER_DST;
5992 		pdst = PF_PEER_SRC;
5993 	}
5994 
5995 	if (th->th_flags & TH_SYN)
5996 		if (src->state < TCPS_SYN_SENT)
5997 			pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5998 	if (th->th_flags & TH_FIN)
5999 		if (src->state < TCPS_CLOSING)
6000 			pf_set_protostate(*state, psrc, TCPS_CLOSING);
6001 	if (th->th_flags & TH_ACK) {
6002 		if (dst->state == TCPS_SYN_SENT) {
6003 			pf_set_protostate(*state, pdst, TCPS_ESTABLISHED);
6004 			if (src->state == TCPS_ESTABLISHED &&
6005 			    (*state)->src_node != NULL &&
6006 			    pf_src_connlimit(state)) {
6007 				REASON_SET(reason, PFRES_SRCLIMIT);
6008 				return (PF_DROP);
6009 			}
6010 		} else if (dst->state == TCPS_CLOSING) {
6011 			pf_set_protostate(*state, pdst, TCPS_FIN_WAIT_2);
6012 		} else if (src->state == TCPS_SYN_SENT &&
6013 		    dst->state < TCPS_SYN_SENT) {
6014 			/*
6015 			 * Handle a special sloppy case where we only see one
6016 			 * half of the connection. If there is a ACK after
6017 			 * the initial SYN without ever seeing a packet from
6018 			 * the destination, set the connection to established.
6019 			 */
6020 			pf_set_protostate(*state, PF_PEER_BOTH,
6021 			    TCPS_ESTABLISHED);
6022 			dst->state = src->state = TCPS_ESTABLISHED;
6023 			if ((*state)->src_node != NULL &&
6024 			    pf_src_connlimit(state)) {
6025 				REASON_SET(reason, PFRES_SRCLIMIT);
6026 				return (PF_DROP);
6027 			}
6028 		} else if (src->state == TCPS_CLOSING &&
6029 		    dst->state == TCPS_ESTABLISHED &&
6030 		    dst->seqlo == 0) {
6031 			/*
6032 			 * Handle the closing of half connections where we
6033 			 * don't see the full bidirectional FIN/ACK+ACK
6034 			 * handshake.
6035 			 */
6036 			pf_set_protostate(*state, pdst, TCPS_CLOSING);
6037 		}
6038 	}
6039 	if (th->th_flags & TH_RST)
6040 		pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
6041 
6042 	/* update expire time */
6043 	(*state)->expire = pf_get_uptime();
6044 	if (src->state >= TCPS_FIN_WAIT_2 &&
6045 	    dst->state >= TCPS_FIN_WAIT_2)
6046 		(*state)->timeout = PFTM_TCP_CLOSED;
6047 	else if (src->state >= TCPS_CLOSING &&
6048 	    dst->state >= TCPS_CLOSING)
6049 		(*state)->timeout = PFTM_TCP_FIN_WAIT;
6050 	else if (src->state < TCPS_ESTABLISHED ||
6051 	    dst->state < TCPS_ESTABLISHED)
6052 		(*state)->timeout = PFTM_TCP_OPENING;
6053 	else if (src->state >= TCPS_CLOSING ||
6054 	    dst->state >= TCPS_CLOSING)
6055 		(*state)->timeout = PFTM_TCP_CLOSING;
6056 	else
6057 		(*state)->timeout = PFTM_TCP_ESTABLISHED;
6058 
6059 	return (PF_PASS);
6060 }
6061 
6062 static int
6063 pf_synproxy(struct pf_pdesc *pd, struct pf_kstate **state, u_short *reason)
6064 {
6065 	struct pf_state_key	*sk = (*state)->key[pd->didx];
6066 	struct tcphdr		*th = &pd->hdr.tcp;
6067 
6068 	if ((*state)->src.state == PF_TCPS_PROXY_SRC) {
6069 		if (pd->dir != (*state)->direction) {
6070 			REASON_SET(reason, PFRES_SYNPROXY);
6071 			return (PF_SYNPROXY_DROP);
6072 		}
6073 		if (th->th_flags & TH_SYN) {
6074 			if (ntohl(th->th_seq) != (*state)->src.seqlo) {
6075 				REASON_SET(reason, PFRES_SYNPROXY);
6076 				return (PF_DROP);
6077 			}
6078 			pf_send_tcp((*state)->rule.ptr, pd->af, pd->dst,
6079 			    pd->src, th->th_dport, th->th_sport,
6080 			    (*state)->src.seqhi, ntohl(th->th_seq) + 1,
6081 			    TH_SYN|TH_ACK, 0, (*state)->src.mss, 0, true, 0, 0,
6082 			    (*state)->act.rtableid);
6083 			REASON_SET(reason, PFRES_SYNPROXY);
6084 			return (PF_SYNPROXY_DROP);
6085 		} else if ((th->th_flags & (TH_ACK|TH_RST|TH_FIN)) != TH_ACK ||
6086 		    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
6087 		    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
6088 			REASON_SET(reason, PFRES_SYNPROXY);
6089 			return (PF_DROP);
6090 		} else if ((*state)->src_node != NULL &&
6091 		    pf_src_connlimit(state)) {
6092 			REASON_SET(reason, PFRES_SRCLIMIT);
6093 			return (PF_DROP);
6094 		} else
6095 			pf_set_protostate(*state, PF_PEER_SRC,
6096 			    PF_TCPS_PROXY_DST);
6097 	}
6098 	if ((*state)->src.state == PF_TCPS_PROXY_DST) {
6099 		if (pd->dir == (*state)->direction) {
6100 			if (((th->th_flags & (TH_SYN|TH_ACK)) != TH_ACK) ||
6101 			    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
6102 			    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
6103 				REASON_SET(reason, PFRES_SYNPROXY);
6104 				return (PF_DROP);
6105 			}
6106 			(*state)->src.max_win = MAX(ntohs(th->th_win), 1);
6107 			if ((*state)->dst.seqhi == 1)
6108 				(*state)->dst.seqhi = htonl(arc4random());
6109 			pf_send_tcp((*state)->rule.ptr, pd->af,
6110 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
6111 			    sk->port[pd->sidx], sk->port[pd->didx],
6112 			    (*state)->dst.seqhi, 0, TH_SYN, 0,
6113 			    (*state)->src.mss, 0, false, (*state)->tag, 0,
6114 			    (*state)->act.rtableid);
6115 			REASON_SET(reason, PFRES_SYNPROXY);
6116 			return (PF_SYNPROXY_DROP);
6117 		} else if (((th->th_flags & (TH_SYN|TH_ACK)) !=
6118 		    (TH_SYN|TH_ACK)) ||
6119 		    (ntohl(th->th_ack) != (*state)->dst.seqhi + 1)) {
6120 			REASON_SET(reason, PFRES_SYNPROXY);
6121 			return (PF_DROP);
6122 		} else {
6123 			(*state)->dst.max_win = MAX(ntohs(th->th_win), 1);
6124 			(*state)->dst.seqlo = ntohl(th->th_seq);
6125 			pf_send_tcp((*state)->rule.ptr, pd->af, pd->dst,
6126 			    pd->src, th->th_dport, th->th_sport,
6127 			    ntohl(th->th_ack), ntohl(th->th_seq) + 1,
6128 			    TH_ACK, (*state)->src.max_win, 0, 0, false,
6129 			    (*state)->tag, 0, (*state)->act.rtableid);
6130 			pf_send_tcp((*state)->rule.ptr, pd->af,
6131 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
6132 			    sk->port[pd->sidx], sk->port[pd->didx],
6133 			    (*state)->src.seqhi + 1, (*state)->src.seqlo + 1,
6134 			    TH_ACK, (*state)->dst.max_win, 0, 0, true, 0, 0,
6135 			    (*state)->act.rtableid);
6136 			(*state)->src.seqdiff = (*state)->dst.seqhi -
6137 			    (*state)->src.seqlo;
6138 			(*state)->dst.seqdiff = (*state)->src.seqhi -
6139 			    (*state)->dst.seqlo;
6140 			(*state)->src.seqhi = (*state)->src.seqlo +
6141 			    (*state)->dst.max_win;
6142 			(*state)->dst.seqhi = (*state)->dst.seqlo +
6143 			    (*state)->src.max_win;
6144 			(*state)->src.wscale = (*state)->dst.wscale = 0;
6145 			pf_set_protostate(*state, PF_PEER_BOTH,
6146 			    TCPS_ESTABLISHED);
6147 			REASON_SET(reason, PFRES_SYNPROXY);
6148 			return (PF_SYNPROXY_DROP);
6149 		}
6150 	}
6151 
6152 	return (PF_PASS);
6153 }
6154 
6155 static int
6156 pf_test_state_tcp(struct pf_kstate **state, struct pfi_kkif *kif,
6157     struct mbuf *m, int off, struct pf_pdesc *pd,
6158     u_short *reason)
6159 {
6160 	struct pf_state_key_cmp	 key;
6161 	struct tcphdr		*th = &pd->hdr.tcp;
6162 	int			 copyback = 0;
6163 	int			 action;
6164 	struct pf_state_peer	*src, *dst;
6165 
6166 	bzero(&key, sizeof(key));
6167 	key.af = pd->af;
6168 	key.proto = IPPROTO_TCP;
6169 	if (pd->dir == PF_IN)	{	/* wire side, straight */
6170 		PF_ACPY(&key.addr[0], pd->src, key.af);
6171 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6172 		key.port[0] = th->th_sport;
6173 		key.port[1] = th->th_dport;
6174 	} else {			/* stack side, reverse */
6175 		PF_ACPY(&key.addr[1], pd->src, key.af);
6176 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6177 		key.port[1] = th->th_sport;
6178 		key.port[0] = th->th_dport;
6179 	}
6180 
6181 	STATE_LOOKUP(kif, &key, *state, pd);
6182 
6183 	if (pd->dir == (*state)->direction) {
6184 		src = &(*state)->src;
6185 		dst = &(*state)->dst;
6186 	} else {
6187 		src = &(*state)->dst;
6188 		dst = &(*state)->src;
6189 	}
6190 
6191 	if ((action = pf_synproxy(pd, state, reason)) != PF_PASS)
6192 		return (action);
6193 
6194 	if (dst->state >= TCPS_FIN_WAIT_2 &&
6195 	    src->state >= TCPS_FIN_WAIT_2 &&
6196 	    (((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN) ||
6197 	    ((th->th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_ACK &&
6198 	    pf_syncookie_check(pd) && pd->dir == PF_IN))) {
6199 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
6200 			printf("pf: state reuse ");
6201 			pf_print_state(*state);
6202 			pf_print_flags(th->th_flags);
6203 			printf("\n");
6204 		}
6205 		/* XXX make sure it's the same direction ?? */
6206 		pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
6207 		pf_unlink_state(*state);
6208 		*state = NULL;
6209 		return (PF_DROP);
6210 	}
6211 
6212 	if ((*state)->state_flags & PFSTATE_SLOPPY) {
6213 		if (pf_tcp_track_sloppy(state, pd, reason) == PF_DROP)
6214 			return (PF_DROP);
6215 	} else {
6216 		if (pf_tcp_track_full(state, kif, m, off, pd, reason,
6217 		    &copyback) == PF_DROP)
6218 			return (PF_DROP);
6219 	}
6220 
6221 	/* translate source/destination address, if necessary */
6222 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6223 		struct pf_state_key *nk = (*state)->key[pd->didx];
6224 
6225 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
6226 		    nk->port[pd->sidx] != th->th_sport)
6227 			pf_change_ap(m, pd->src, &th->th_sport,
6228 			    pd->ip_sum, &th->th_sum, &nk->addr[pd->sidx],
6229 			    nk->port[pd->sidx], 0, pd->af);
6230 
6231 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
6232 		    nk->port[pd->didx] != th->th_dport)
6233 			pf_change_ap(m, pd->dst, &th->th_dport,
6234 			    pd->ip_sum, &th->th_sum, &nk->addr[pd->didx],
6235 			    nk->port[pd->didx], 0, pd->af);
6236 		copyback = 1;
6237 	}
6238 
6239 	/* Copyback sequence modulation or stateful scrub changes if needed */
6240 	if (copyback)
6241 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
6242 
6243 	return (PF_PASS);
6244 }
6245 
6246 static int
6247 pf_test_state_udp(struct pf_kstate **state, struct pfi_kkif *kif,
6248     struct mbuf *m, int off, struct pf_pdesc *pd)
6249 {
6250 	struct pf_state_peer	*src, *dst;
6251 	struct pf_state_key_cmp	 key;
6252 	struct udphdr		*uh = &pd->hdr.udp;
6253 	uint8_t			 psrc, pdst;
6254 
6255 	bzero(&key, sizeof(key));
6256 	key.af = pd->af;
6257 	key.proto = IPPROTO_UDP;
6258 	if (pd->dir == PF_IN)	{	/* wire side, straight */
6259 		PF_ACPY(&key.addr[0], pd->src, key.af);
6260 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6261 		key.port[0] = uh->uh_sport;
6262 		key.port[1] = uh->uh_dport;
6263 	} else {			/* stack side, reverse */
6264 		PF_ACPY(&key.addr[1], pd->src, key.af);
6265 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6266 		key.port[1] = uh->uh_sport;
6267 		key.port[0] = uh->uh_dport;
6268 	}
6269 
6270 	STATE_LOOKUP(kif, &key, *state, pd);
6271 
6272 	if (pd->dir == (*state)->direction) {
6273 		src = &(*state)->src;
6274 		dst = &(*state)->dst;
6275 		psrc = PF_PEER_SRC;
6276 		pdst = PF_PEER_DST;
6277 	} else {
6278 		src = &(*state)->dst;
6279 		dst = &(*state)->src;
6280 		psrc = PF_PEER_DST;
6281 		pdst = PF_PEER_SRC;
6282 	}
6283 
6284 	/* update states */
6285 	if (src->state < PFUDPS_SINGLE)
6286 		pf_set_protostate(*state, psrc, PFUDPS_SINGLE);
6287 	if (dst->state == PFUDPS_SINGLE)
6288 		pf_set_protostate(*state, pdst, PFUDPS_MULTIPLE);
6289 
6290 	/* update expire time */
6291 	(*state)->expire = pf_get_uptime();
6292 	if (src->state == PFUDPS_MULTIPLE && dst->state == PFUDPS_MULTIPLE)
6293 		(*state)->timeout = PFTM_UDP_MULTIPLE;
6294 	else
6295 		(*state)->timeout = PFTM_UDP_SINGLE;
6296 
6297 	/* translate source/destination address, if necessary */
6298 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6299 		struct pf_state_key *nk = (*state)->key[pd->didx];
6300 
6301 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
6302 		    nk->port[pd->sidx] != uh->uh_sport)
6303 			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
6304 			    &uh->uh_sum, &nk->addr[pd->sidx],
6305 			    nk->port[pd->sidx], 1, pd->af);
6306 
6307 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
6308 		    nk->port[pd->didx] != uh->uh_dport)
6309 			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
6310 			    &uh->uh_sum, &nk->addr[pd->didx],
6311 			    nk->port[pd->didx], 1, pd->af);
6312 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
6313 	}
6314 
6315 	return (PF_PASS);
6316 }
6317 
6318 static int
6319 pf_test_state_sctp(struct pf_kstate **state, struct pfi_kkif *kif,
6320     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason)
6321 {
6322 	struct pf_state_key_cmp	 key;
6323 	struct pf_state_peer	*src, *dst;
6324 	struct sctphdr		*sh = &pd->hdr.sctp;
6325 	u_int8_t		 psrc; //, pdst;
6326 
6327 	bzero(&key, sizeof(key));
6328 	key.af = pd->af;
6329 	key.proto = IPPROTO_SCTP;
6330 	if (pd->dir == PF_IN)	{	/* wire side, straight */
6331 		PF_ACPY(&key.addr[0], pd->src, key.af);
6332 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6333 		key.port[0] = sh->src_port;
6334 		key.port[1] = sh->dest_port;
6335 	} else {			/* stack side, reverse */
6336 		PF_ACPY(&key.addr[1], pd->src, key.af);
6337 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6338 		key.port[1] = sh->src_port;
6339 		key.port[0] = sh->dest_port;
6340 	}
6341 
6342 	STATE_LOOKUP(kif, &key, *state, pd);
6343 
6344 	if (pd->dir == (*state)->direction) {
6345 		src = &(*state)->src;
6346 		dst = &(*state)->dst;
6347 		psrc = PF_PEER_SRC;
6348 	} else {
6349 		src = &(*state)->dst;
6350 		dst = &(*state)->src;
6351 		psrc = PF_PEER_DST;
6352 	}
6353 
6354 	if ((src->state >= SCTP_SHUTDOWN_SENT || src->state == SCTP_CLOSED) &&
6355 	    (dst->state >= SCTP_SHUTDOWN_SENT || dst->state == SCTP_CLOSED) &&
6356 	    pd->sctp_flags & PFDESC_SCTP_INIT) {
6357 		pf_set_protostate(*state, PF_PEER_BOTH, SCTP_CLOSED);
6358 		pf_unlink_state(*state);
6359 		*state = NULL;
6360 		return (PF_DROP);
6361 	}
6362 
6363 	/* Track state. */
6364 	if (pd->sctp_flags & PFDESC_SCTP_INIT) {
6365 		if (src->state < SCTP_COOKIE_WAIT) {
6366 			pf_set_protostate(*state, psrc, SCTP_COOKIE_WAIT);
6367 			(*state)->timeout = PFTM_SCTP_OPENING;
6368 		}
6369 	}
6370 	if (pd->sctp_flags & PFDESC_SCTP_INIT_ACK) {
6371 		MPASS(dst->scrub != NULL);
6372 		if (dst->scrub->pfss_v_tag == 0)
6373 			dst->scrub->pfss_v_tag = pd->sctp_initiate_tag;
6374 	}
6375 
6376 	if (pd->sctp_flags & (PFDESC_SCTP_COOKIE | PFDESC_SCTP_HEARTBEAT_ACK)) {
6377 		if (src->state < SCTP_ESTABLISHED) {
6378 			pf_set_protostate(*state, psrc, SCTP_ESTABLISHED);
6379 			(*state)->timeout = PFTM_SCTP_ESTABLISHED;
6380 		}
6381 	}
6382 	if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN | PFDESC_SCTP_ABORT |
6383 	    PFDESC_SCTP_SHUTDOWN_COMPLETE)) {
6384 		if (src->state < SCTP_SHUTDOWN_PENDING) {
6385 			pf_set_protostate(*state, psrc, SCTP_SHUTDOWN_PENDING);
6386 			(*state)->timeout = PFTM_SCTP_CLOSING;
6387 		}
6388 	}
6389 	if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN_COMPLETE)) {
6390 		pf_set_protostate(*state, psrc, SCTP_CLOSED);
6391 		(*state)->timeout = PFTM_SCTP_CLOSED;
6392 	}
6393 
6394 	if (src->scrub != NULL) {
6395 		if (src->scrub->pfss_v_tag == 0) {
6396 			src->scrub->pfss_v_tag = pd->hdr.sctp.v_tag;
6397 		} else  if (src->scrub->pfss_v_tag != pd->hdr.sctp.v_tag)
6398 			return (PF_DROP);
6399 	}
6400 
6401 	(*state)->expire = pf_get_uptime();
6402 
6403 	/* translate source/destination address, if necessary */
6404 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6405 		uint16_t checksum = 0;
6406 		struct pf_state_key *nk = (*state)->key[pd->didx];
6407 
6408 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
6409 		    nk->port[pd->sidx] != pd->hdr.sctp.src_port) {
6410 			pf_change_ap(m, pd->src, &pd->hdr.sctp.src_port,
6411 			    pd->ip_sum, &checksum, &nk->addr[pd->sidx],
6412 			    nk->port[pd->sidx], 1, pd->af);
6413 		}
6414 
6415 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
6416 		    nk->port[pd->didx] != pd->hdr.sctp.dest_port) {
6417 			pf_change_ap(m, pd->dst, &pd->hdr.sctp.dest_port,
6418 			    pd->ip_sum, &checksum, &nk->addr[pd->didx],
6419 			    nk->port[pd->didx], 1, pd->af);
6420 		}
6421 	}
6422 
6423 	return (PF_PASS);
6424 }
6425 
6426 static void
6427 pf_sctp_multihome_detach_addr(const struct pf_kstate *s)
6428 {
6429 	struct pf_sctp_endpoint key;
6430 	struct pf_sctp_endpoint *ep;
6431 	struct pf_state_key *sks = s->key[PF_SK_STACK];
6432 	struct pf_sctp_source *i, *tmp;
6433 
6434 	if (sks == NULL || sks->proto != IPPROTO_SCTP || s->dst.scrub == NULL)
6435 		return;
6436 
6437 	PF_SCTP_ENDPOINTS_LOCK();
6438 
6439 	key.v_tag = s->dst.scrub->pfss_v_tag;
6440 	ep  = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6441 	if (ep != NULL) {
6442 		TAILQ_FOREACH_SAFE(i, &ep->sources, entry, tmp) {
6443 			if (pf_addr_cmp(&i->addr,
6444 			    &s->key[PF_SK_WIRE]->addr[s->direction == PF_OUT],
6445 			    s->key[PF_SK_WIRE]->af) == 0) {
6446 				SDT_PROBE3(pf, sctp, multihome, remove,
6447 				    key.v_tag, s, i);
6448 				TAILQ_REMOVE(&ep->sources, i, entry);
6449 				free(i, M_PFTEMP);
6450 				break;
6451 			}
6452 		}
6453 
6454 		if (TAILQ_EMPTY(&ep->sources)) {
6455 			RB_REMOVE(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
6456 			free(ep, M_PFTEMP);
6457 		}
6458 	}
6459 
6460 	/* Other direction. */
6461 	key.v_tag = s->src.scrub->pfss_v_tag;
6462 	ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6463 	if (ep != NULL) {
6464 		TAILQ_FOREACH_SAFE(i, &ep->sources, entry, tmp) {
6465 			if (pf_addr_cmp(&i->addr,
6466 			    &s->key[PF_SK_WIRE]->addr[s->direction == PF_IN],
6467 			    s->key[PF_SK_WIRE]->af) == 0) {
6468 				SDT_PROBE3(pf, sctp, multihome, remove,
6469 				    key.v_tag, s, i);
6470 				TAILQ_REMOVE(&ep->sources, i, entry);
6471 				free(i, M_PFTEMP);
6472 				break;
6473 			}
6474 		}
6475 
6476 		if (TAILQ_EMPTY(&ep->sources)) {
6477 			RB_REMOVE(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
6478 			free(ep, M_PFTEMP);
6479 		}
6480 	}
6481 
6482 	PF_SCTP_ENDPOINTS_UNLOCK();
6483 }
6484 
6485 static void
6486 pf_sctp_multihome_add_addr(struct pf_pdesc *pd, struct pf_addr *a, uint32_t v_tag)
6487 {
6488 	struct pf_sctp_endpoint key = {
6489 		.v_tag = v_tag,
6490 	};
6491 	struct pf_sctp_source *i;
6492 	struct pf_sctp_endpoint *ep;
6493 
6494 	PF_SCTP_ENDPOINTS_LOCK();
6495 
6496 	ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6497 	if (ep == NULL) {
6498 		ep = malloc(sizeof(struct pf_sctp_endpoint),
6499 		    M_PFTEMP, M_NOWAIT);
6500 		if (ep == NULL) {
6501 			PF_SCTP_ENDPOINTS_UNLOCK();
6502 			return;
6503 		}
6504 
6505 		ep->v_tag = v_tag;
6506 		TAILQ_INIT(&ep->sources);
6507 		RB_INSERT(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
6508 	}
6509 
6510 	/* Avoid inserting duplicates. */
6511 	TAILQ_FOREACH(i, &ep->sources, entry) {
6512 		if (pf_addr_cmp(&i->addr, a, pd->af) == 0) {
6513 			PF_SCTP_ENDPOINTS_UNLOCK();
6514 			return;
6515 		}
6516 	}
6517 
6518 	i = malloc(sizeof(*i), M_PFTEMP, M_NOWAIT);
6519 	if (i == NULL) {
6520 		PF_SCTP_ENDPOINTS_UNLOCK();
6521 		return;
6522 	}
6523 
6524 	i->af = pd->af;
6525 	memcpy(&i->addr, a, sizeof(*a));
6526 	TAILQ_INSERT_TAIL(&ep->sources, i, entry);
6527 	SDT_PROBE2(pf, sctp, multihome, add, v_tag, i);
6528 
6529 	PF_SCTP_ENDPOINTS_UNLOCK();
6530 }
6531 
6532 static void
6533 pf_sctp_multihome_delayed(struct pf_pdesc *pd, int off, struct pfi_kkif *kif,
6534     struct pf_kstate *s, int action)
6535 {
6536 	struct pf_sctp_multihome_job	*j, *tmp;
6537 	struct pf_sctp_source		*i;
6538 	int			 ret __unused;
6539 	struct pf_kstate	*sm = NULL;
6540 	struct pf_krule		*ra = NULL;
6541 	struct pf_krule		*r = &V_pf_default_rule;
6542 	struct pf_kruleset	*rs = NULL;
6543 	bool do_extra = true;
6544 
6545 	PF_RULES_RLOCK_TRACKER;
6546 
6547 again:
6548 	TAILQ_FOREACH_SAFE(j, &pd->sctp_multihome_jobs, next, tmp) {
6549 		if (s == NULL || action != PF_PASS)
6550 			goto free;
6551 
6552 		/* Confirm we don't recurse here. */
6553 		MPASS(! (pd->sctp_flags & PFDESC_SCTP_ADD_IP));
6554 
6555 		switch (j->op) {
6556 		case  SCTP_ADD_IP_ADDRESS: {
6557 			uint32_t v_tag = pd->sctp_initiate_tag;
6558 
6559 			if (v_tag == 0) {
6560 				if (s->direction == pd->dir)
6561 					v_tag = s->src.scrub->pfss_v_tag;
6562 				else
6563 					v_tag = s->dst.scrub->pfss_v_tag;
6564 			}
6565 
6566 			/*
6567 			 * Avoid duplicating states. We'll already have
6568 			 * created a state based on the source address of
6569 			 * the packet, but SCTP endpoints may also list this
6570 			 * address again in the INIT(_ACK) parameters.
6571 			 */
6572 			if (pf_addr_cmp(&j->src, pd->src, pd->af) == 0) {
6573 				break;
6574 			}
6575 
6576 			j->pd.sctp_flags |= PFDESC_SCTP_ADD_IP;
6577 			PF_RULES_RLOCK();
6578 			sm = NULL;
6579 			/*
6580 			 * New connections need to be floating, because
6581 			 * we cannot know what interfaces it will use.
6582 			 * That's why we pass V_pfi_all rather than kif.
6583 			 */
6584 			ret = pf_test_rule(&r, &sm, V_pfi_all,
6585 			    j->m, off, &j->pd, &ra, &rs, NULL,
6586 			    sizeof(j->pd.hdr.sctp));
6587 			PF_RULES_RUNLOCK();
6588 			SDT_PROBE4(pf, sctp, multihome, test, kif, r, j->m, ret);
6589 			if (ret != PF_DROP && sm != NULL) {
6590 				/* Inherit v_tag values. */
6591 				if (sm->direction == s->direction) {
6592 					sm->src.scrub->pfss_v_tag = s->src.scrub->pfss_v_tag;
6593 					sm->dst.scrub->pfss_v_tag = s->dst.scrub->pfss_v_tag;
6594 				} else {
6595 					sm->src.scrub->pfss_v_tag = s->dst.scrub->pfss_v_tag;
6596 					sm->dst.scrub->pfss_v_tag = s->src.scrub->pfss_v_tag;
6597 				}
6598 				PF_STATE_UNLOCK(sm);
6599 			} else {
6600 				/* If we try duplicate inserts? */
6601 				break;
6602 			}
6603 
6604 			/* Only add the address if we've actually allowed the state. */
6605 			pf_sctp_multihome_add_addr(pd, &j->src, v_tag);
6606 
6607 			if (! do_extra) {
6608 				break;
6609 			}
6610 			/*
6611 			 * We need to do this for each of our source addresses.
6612 			 * Find those based on the verification tag.
6613 			 */
6614 			struct pf_sctp_endpoint key = {
6615 				.v_tag = pd->hdr.sctp.v_tag,
6616 			};
6617 			struct pf_sctp_endpoint *ep;
6618 
6619 			PF_SCTP_ENDPOINTS_LOCK();
6620 			ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6621 			if (ep == NULL) {
6622 				PF_SCTP_ENDPOINTS_UNLOCK();
6623 				break;
6624 			}
6625 			MPASS(ep != NULL);
6626 
6627 			TAILQ_FOREACH(i, &ep->sources, entry) {
6628 				struct pf_sctp_multihome_job *nj;
6629 
6630 				/* SCTP can intermingle IPv4 and IPv6. */
6631 				if (i->af != pd->af)
6632 					continue;
6633 
6634 				nj = malloc(sizeof(*nj), M_PFTEMP, M_NOWAIT | M_ZERO);
6635 				if (! nj) {
6636 					continue;
6637 				}
6638 				memcpy(&nj->pd, &j->pd, sizeof(j->pd));
6639 				memcpy(&nj->src, &j->src, sizeof(nj->src));
6640 				nj->pd.src = &nj->src;
6641 				// New destination address!
6642 				memcpy(&nj->dst, &i->addr, sizeof(nj->dst));
6643 				nj->pd.dst = &nj->dst;
6644 				nj->m = j->m;
6645 				nj->op = j->op;
6646 
6647 				TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, nj, next);
6648 			}
6649 			PF_SCTP_ENDPOINTS_UNLOCK();
6650 
6651 			break;
6652 		}
6653 		case SCTP_DEL_IP_ADDRESS: {
6654 			struct pf_state_key_cmp key;
6655 			uint8_t psrc;
6656 
6657 			bzero(&key, sizeof(key));
6658 			key.af = j->pd.af;
6659 			key.proto = IPPROTO_SCTP;
6660 			if (j->pd.dir == PF_IN)	{	/* wire side, straight */
6661 				PF_ACPY(&key.addr[0], j->pd.src, key.af);
6662 				PF_ACPY(&key.addr[1], j->pd.dst, key.af);
6663 				key.port[0] = j->pd.hdr.sctp.src_port;
6664 				key.port[1] = j->pd.hdr.sctp.dest_port;
6665 			} else {			/* stack side, reverse */
6666 				PF_ACPY(&key.addr[1], j->pd.src, key.af);
6667 				PF_ACPY(&key.addr[0], j->pd.dst, key.af);
6668 				key.port[1] = j->pd.hdr.sctp.src_port;
6669 				key.port[0] = j->pd.hdr.sctp.dest_port;
6670 			}
6671 
6672 			sm = pf_find_state(kif, &key, j->pd.dir);
6673 			if (sm != NULL) {
6674 				PF_STATE_LOCK_ASSERT(sm);
6675 				if (j->pd.dir == sm->direction) {
6676 					psrc = PF_PEER_SRC;
6677 				} else {
6678 					psrc = PF_PEER_DST;
6679 				}
6680 				pf_set_protostate(sm, psrc, SCTP_SHUTDOWN_PENDING);
6681 				sm->timeout = PFTM_SCTP_CLOSING;
6682 				PF_STATE_UNLOCK(sm);
6683 			}
6684 			break;
6685 		default:
6686 			panic("Unknown op %#x", j->op);
6687 		}
6688 	}
6689 
6690 	free:
6691 		TAILQ_REMOVE(&pd->sctp_multihome_jobs, j, next);
6692 		free(j, M_PFTEMP);
6693 	}
6694 
6695 	/* We may have inserted extra work while processing the list. */
6696 	if (! TAILQ_EMPTY(&pd->sctp_multihome_jobs)) {
6697 		do_extra = false;
6698 		goto again;
6699 	}
6700 }
6701 
6702 static int
6703 pf_multihome_scan(struct mbuf *m, int start, int len, struct pf_pdesc *pd,
6704     struct pfi_kkif *kif, int op)
6705 {
6706 	int			 off = 0;
6707 	struct pf_sctp_multihome_job	*job;
6708 
6709 	while (off < len) {
6710 		struct sctp_paramhdr h;
6711 
6712 		if (!pf_pull_hdr(m, start + off, &h, sizeof(h), NULL, NULL,
6713 		    pd->af))
6714 			return (PF_DROP);
6715 
6716 		/* Parameters are at least 4 bytes. */
6717 		if (ntohs(h.param_length) < 4)
6718 			return (PF_DROP);
6719 
6720 		switch (ntohs(h.param_type)) {
6721 		case  SCTP_IPV4_ADDRESS: {
6722 			struct in_addr t;
6723 
6724 			if (ntohs(h.param_length) !=
6725 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
6726 				return (PF_DROP);
6727 
6728 			if (!pf_pull_hdr(m, start + off + sizeof(h), &t, sizeof(t),
6729 			    NULL, NULL, pd->af))
6730 				return (PF_DROP);
6731 
6732 			if (in_nullhost(t))
6733 				t.s_addr = pd->src->v4.s_addr;
6734 
6735 			/*
6736 			 * We hold the state lock (idhash) here, which means
6737 			 * that we can't acquire the keyhash, or we'll get a
6738 			 * LOR (and potentially double-lock things too). We also
6739 			 * can't release the state lock here, so instead we'll
6740 			 * enqueue this for async handling.
6741 			 * There's a relatively small race here, in that a
6742 			 * packet using the new addresses could arrive already,
6743 			 * but that's just though luck for it.
6744 			 */
6745 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
6746 			if (! job)
6747 				return (PF_DROP);
6748 
6749 			memcpy(&job->pd, pd, sizeof(*pd));
6750 
6751 			// New source address!
6752 			memcpy(&job->src, &t, sizeof(t));
6753 			job->pd.src = &job->src;
6754 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
6755 			job->pd.dst = &job->dst;
6756 			job->m = m;
6757 			job->op = op;
6758 
6759 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
6760 			break;
6761 		}
6762 #ifdef INET6
6763 		case SCTP_IPV6_ADDRESS: {
6764 			struct in6_addr t;
6765 
6766 			if (ntohs(h.param_length) !=
6767 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
6768 				return (PF_DROP);
6769 
6770 			if (!pf_pull_hdr(m, start + off + sizeof(h), &t, sizeof(t),
6771 			    NULL, NULL, pd->af))
6772 				return (PF_DROP);
6773 			if (memcmp(&t, &pd->src->v6, sizeof(t)) == 0)
6774 				break;
6775 			if (memcmp(&t, &in6addr_any, sizeof(t)) == 0)
6776 				memcpy(&t, &pd->src->v6, sizeof(t));
6777 
6778 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
6779 			if (! job)
6780 				return (PF_DROP);
6781 
6782 			memcpy(&job->pd, pd, sizeof(*pd));
6783 			memcpy(&job->src, &t, sizeof(t));
6784 			job->pd.src = &job->src;
6785 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
6786 			job->pd.dst = &job->dst;
6787 			job->m = m;
6788 			job->op = op;
6789 
6790 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
6791 			break;
6792 		}
6793 #endif
6794 		case SCTP_ADD_IP_ADDRESS: {
6795 			int ret;
6796 			struct sctp_asconf_paramhdr ah;
6797 
6798 			if (!pf_pull_hdr(m, start + off, &ah, sizeof(ah),
6799 			    NULL, NULL, pd->af))
6800 				return (PF_DROP);
6801 
6802 			ret = pf_multihome_scan(m, start + off + sizeof(ah),
6803 			    ntohs(ah.ph.param_length) - sizeof(ah), pd, kif,
6804 			    SCTP_ADD_IP_ADDRESS);
6805 			if (ret != PF_PASS)
6806 				return (ret);
6807 			break;
6808 		}
6809 		case SCTP_DEL_IP_ADDRESS: {
6810 			int ret;
6811 			struct sctp_asconf_paramhdr ah;
6812 
6813 			if (!pf_pull_hdr(m, start + off, &ah, sizeof(ah),
6814 			    NULL, NULL, pd->af))
6815 				return (PF_DROP);
6816 			ret = pf_multihome_scan(m, start + off + sizeof(ah),
6817 			    ntohs(ah.ph.param_length) - sizeof(ah), pd, kif,
6818 			    SCTP_DEL_IP_ADDRESS);
6819 			if (ret != PF_PASS)
6820 				return (ret);
6821 			break;
6822 		}
6823 		default:
6824 			break;
6825 		}
6826 
6827 		off += roundup(ntohs(h.param_length), 4);
6828 	}
6829 
6830 	return (PF_PASS);
6831 }
6832 int
6833 pf_multihome_scan_init(struct mbuf *m, int start, int len, struct pf_pdesc *pd,
6834     struct pfi_kkif *kif)
6835 {
6836 	start += sizeof(struct sctp_init_chunk);
6837 	len -= sizeof(struct sctp_init_chunk);
6838 
6839 	return (pf_multihome_scan(m, start, len, pd, kif, SCTP_ADD_IP_ADDRESS));
6840 }
6841 
6842 int
6843 pf_multihome_scan_asconf(struct mbuf *m, int start, int len,
6844     struct pf_pdesc *pd, struct pfi_kkif *kif)
6845 {
6846 	start += sizeof(struct sctp_asconf_chunk);
6847 	len -= sizeof(struct sctp_asconf_chunk);
6848 
6849 	return (pf_multihome_scan(m, start, len, pd, kif, SCTP_ADD_IP_ADDRESS));
6850 }
6851 
6852 int
6853 pf_icmp_state_lookup(struct pf_state_key_cmp *key, struct pf_pdesc *pd,
6854     struct pf_kstate **state, struct mbuf *m, int off, int direction,
6855     struct pfi_kkif *kif, u_int16_t icmpid, u_int16_t type, int icmp_dir,
6856     int *iidx, int multi, int inner)
6857 {
6858 	key->af = pd->af;
6859 	key->proto = pd->proto;
6860 	if (icmp_dir == PF_IN) {
6861 		*iidx = pd->sidx;
6862 		key->port[pd->sidx] = icmpid;
6863 		key->port[pd->didx] = type;
6864 	} else {
6865 		*iidx = pd->didx;
6866 		key->port[pd->sidx] = type;
6867 		key->port[pd->didx] = icmpid;
6868 	}
6869 	if (pf_state_key_addr_setup(pd, m, off, key, pd->sidx, pd->src,
6870 	    pd->didx, pd->dst, multi))
6871 		return (PF_DROP);
6872 
6873 	STATE_LOOKUP(kif, key, *state, pd);
6874 
6875 	if ((*state)->state_flags & PFSTATE_SLOPPY)
6876 		return (-1);
6877 
6878 	/* Is this ICMP message flowing in right direction? */
6879 	if ((*state)->rule.ptr->type &&
6880 	    (((!inner && (*state)->direction == direction) ||
6881 	    (inner && (*state)->direction != direction)) ?
6882 	    PF_IN : PF_OUT) != icmp_dir) {
6883 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
6884 			printf("pf: icmp type %d in wrong direction (%d): ",
6885 			    ntohs(type), icmp_dir);
6886 			pf_print_state(*state);
6887 			printf("\n");
6888 		}
6889 		PF_STATE_UNLOCK(*state);
6890 		*state = NULL;
6891 		return (PF_DROP);
6892 	}
6893 	return (-1);
6894 }
6895 
6896 static int
6897 pf_test_state_icmp(struct pf_kstate **state, struct pfi_kkif *kif,
6898     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason)
6899 {
6900 	struct pf_addr  *saddr = pd->src, *daddr = pd->dst;
6901 	u_int16_t	*icmpsum, virtual_id, virtual_type;
6902 	u_int8_t	 icmptype, icmpcode;
6903 	int		 icmp_dir, iidx, ret, multi;
6904 	struct pf_state_key_cmp key;
6905 #ifdef INET
6906 	u_int16_t	 icmpid;
6907 #endif
6908 
6909 	MPASS(*state == NULL);
6910 
6911 	bzero(&key, sizeof(key));
6912 	switch (pd->proto) {
6913 #ifdef INET
6914 	case IPPROTO_ICMP:
6915 		icmptype = pd->hdr.icmp.icmp_type;
6916 		icmpcode = pd->hdr.icmp.icmp_code;
6917 		icmpid = pd->hdr.icmp.icmp_id;
6918 		icmpsum = &pd->hdr.icmp.icmp_cksum;
6919 		break;
6920 #endif /* INET */
6921 #ifdef INET6
6922 	case IPPROTO_ICMPV6:
6923 		icmptype = pd->hdr.icmp6.icmp6_type;
6924 		icmpcode = pd->hdr.icmp6.icmp6_code;
6925 #ifdef INET
6926 		icmpid = pd->hdr.icmp6.icmp6_id;
6927 #endif
6928 		icmpsum = &pd->hdr.icmp6.icmp6_cksum;
6929 		break;
6930 #endif /* INET6 */
6931 	}
6932 
6933 	if (pf_icmp_mapping(pd, icmptype, &icmp_dir, &multi,
6934 	    &virtual_id, &virtual_type) == 0) {
6935 		/*
6936 		 * ICMP query/reply message not related to a TCP/UDP packet.
6937 		 * Search for an ICMP state.
6938 		 */
6939 		ret = pf_icmp_state_lookup(&key, pd, state, m, off, pd->dir,
6940 		    kif, virtual_id, virtual_type, icmp_dir, &iidx,
6941 		    PF_ICMP_MULTI_NONE, 0);
6942 		if (ret >= 0) {
6943 			MPASS(*state == NULL);
6944 			if (ret == PF_DROP && pd->af == AF_INET6 &&
6945 			    icmp_dir == PF_OUT) {
6946 				ret = pf_icmp_state_lookup(&key, pd, state, m, off,
6947 				    pd->dir, kif, virtual_id, virtual_type,
6948 				    icmp_dir, &iidx, multi, 0);
6949 				if (ret >= 0) {
6950 					MPASS(*state == NULL);
6951 					return (ret);
6952 				}
6953 			} else
6954 				return (ret);
6955 		}
6956 
6957 		(*state)->expire = pf_get_uptime();
6958 		(*state)->timeout = PFTM_ICMP_ERROR_REPLY;
6959 
6960 		/* translate source/destination address, if necessary */
6961 		if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6962 			struct pf_state_key *nk = (*state)->key[pd->didx];
6963 
6964 			switch (pd->af) {
6965 #ifdef INET
6966 			case AF_INET:
6967 				if (PF_ANEQ(pd->src,
6968 				    &nk->addr[pd->sidx], AF_INET))
6969 					pf_change_a(&saddr->v4.s_addr,
6970 					    pd->ip_sum,
6971 					    nk->addr[pd->sidx].v4.s_addr, 0);
6972 
6973 				if (PF_ANEQ(pd->dst, &nk->addr[pd->didx],
6974 				    AF_INET))
6975 					pf_change_a(&daddr->v4.s_addr,
6976 					    pd->ip_sum,
6977 					    nk->addr[pd->didx].v4.s_addr, 0);
6978 
6979 				if (nk->port[iidx] !=
6980 				    pd->hdr.icmp.icmp_id) {
6981 					pd->hdr.icmp.icmp_cksum =
6982 					    pf_cksum_fixup(
6983 					    pd->hdr.icmp.icmp_cksum, icmpid,
6984 					    nk->port[iidx], 0);
6985 					pd->hdr.icmp.icmp_id =
6986 					    nk->port[iidx];
6987 				}
6988 
6989 				m_copyback(m, off, ICMP_MINLEN,
6990 				    (caddr_t )&pd->hdr.icmp);
6991 				break;
6992 #endif /* INET */
6993 #ifdef INET6
6994 			case AF_INET6:
6995 				if (PF_ANEQ(pd->src,
6996 				    &nk->addr[pd->sidx], AF_INET6))
6997 					pf_change_a6(saddr,
6998 					    &pd->hdr.icmp6.icmp6_cksum,
6999 					    &nk->addr[pd->sidx], 0);
7000 
7001 				if (PF_ANEQ(pd->dst,
7002 				    &nk->addr[pd->didx], AF_INET6))
7003 					pf_change_a6(daddr,
7004 					    &pd->hdr.icmp6.icmp6_cksum,
7005 					    &nk->addr[pd->didx], 0);
7006 
7007 				m_copyback(m, off, sizeof(struct icmp6_hdr),
7008 				    (caddr_t )&pd->hdr.icmp6);
7009 				break;
7010 #endif /* INET6 */
7011 			}
7012 		}
7013 		return (PF_PASS);
7014 
7015 	} else {
7016 		/*
7017 		 * ICMP error message in response to a TCP/UDP packet.
7018 		 * Extract the inner TCP/UDP header and search for that state.
7019 		 */
7020 
7021 		struct pf_pdesc	pd2;
7022 		bzero(&pd2, sizeof pd2);
7023 #ifdef INET
7024 		struct ip	h2;
7025 #endif /* INET */
7026 #ifdef INET6
7027 		struct ip6_hdr	h2_6;
7028 		int		terminal = 0;
7029 #endif /* INET6 */
7030 		int		ipoff2 = 0;
7031 		int		off2 = 0;
7032 
7033 		pd2.af = pd->af;
7034 		pd2.dir = pd->dir;
7035 		/* Payload packet is from the opposite direction. */
7036 		pd2.sidx = (pd->dir == PF_IN) ? 1 : 0;
7037 		pd2.didx = (pd->dir == PF_IN) ? 0 : 1;
7038 		switch (pd->af) {
7039 #ifdef INET
7040 		case AF_INET:
7041 			/* offset of h2 in mbuf chain */
7042 			ipoff2 = off + ICMP_MINLEN;
7043 
7044 			if (!pf_pull_hdr(m, ipoff2, &h2, sizeof(h2),
7045 			    NULL, reason, pd2.af)) {
7046 				DPFPRINTF(PF_DEBUG_MISC,
7047 				    ("pf: ICMP error message too short "
7048 				    "(ip)\n"));
7049 				return (PF_DROP);
7050 			}
7051 			/*
7052 			 * ICMP error messages don't refer to non-first
7053 			 * fragments
7054 			 */
7055 			if (h2.ip_off & htons(IP_OFFMASK)) {
7056 				REASON_SET(reason, PFRES_FRAG);
7057 				return (PF_DROP);
7058 			}
7059 
7060 			/* offset of protocol header that follows h2 */
7061 			off2 = ipoff2 + (h2.ip_hl << 2);
7062 
7063 			pd2.proto = h2.ip_p;
7064 			pd2.src = (struct pf_addr *)&h2.ip_src;
7065 			pd2.dst = (struct pf_addr *)&h2.ip_dst;
7066 			pd2.ip_sum = &h2.ip_sum;
7067 			break;
7068 #endif /* INET */
7069 #ifdef INET6
7070 		case AF_INET6:
7071 			ipoff2 = off + sizeof(struct icmp6_hdr);
7072 
7073 			if (!pf_pull_hdr(m, ipoff2, &h2_6, sizeof(h2_6),
7074 			    NULL, reason, pd2.af)) {
7075 				DPFPRINTF(PF_DEBUG_MISC,
7076 				    ("pf: ICMP error message too short "
7077 				    "(ip6)\n"));
7078 				return (PF_DROP);
7079 			}
7080 			pd2.proto = h2_6.ip6_nxt;
7081 			pd2.src = (struct pf_addr *)&h2_6.ip6_src;
7082 			pd2.dst = (struct pf_addr *)&h2_6.ip6_dst;
7083 			pd2.ip_sum = NULL;
7084 			off2 = ipoff2 + sizeof(h2_6);
7085 			do {
7086 				switch (pd2.proto) {
7087 				case IPPROTO_FRAGMENT:
7088 					/*
7089 					 * ICMPv6 error messages for
7090 					 * non-first fragments
7091 					 */
7092 					REASON_SET(reason, PFRES_FRAG);
7093 					return (PF_DROP);
7094 				case IPPROTO_AH:
7095 				case IPPROTO_HOPOPTS:
7096 				case IPPROTO_ROUTING:
7097 				case IPPROTO_DSTOPTS: {
7098 					/* get next header and header length */
7099 					struct ip6_ext opt6;
7100 
7101 					if (!pf_pull_hdr(m, off2, &opt6,
7102 					    sizeof(opt6), NULL, reason,
7103 					    pd2.af)) {
7104 						DPFPRINTF(PF_DEBUG_MISC,
7105 						    ("pf: ICMPv6 short opt\n"));
7106 						return (PF_DROP);
7107 					}
7108 					if (pd2.proto == IPPROTO_AH)
7109 						off2 += (opt6.ip6e_len + 2) * 4;
7110 					else
7111 						off2 += (opt6.ip6e_len + 1) * 8;
7112 					pd2.proto = opt6.ip6e_nxt;
7113 					/* goto the next header */
7114 					break;
7115 				}
7116 				default:
7117 					terminal++;
7118 					break;
7119 				}
7120 			} while (!terminal);
7121 			break;
7122 #endif /* INET6 */
7123 		}
7124 
7125 		if (PF_ANEQ(pd->dst, pd2.src, pd->af)) {
7126 			if (V_pf_status.debug >= PF_DEBUG_MISC) {
7127 				printf("pf: BAD ICMP %d:%d outer dst: ",
7128 				    icmptype, icmpcode);
7129 				pf_print_host(pd->src, 0, pd->af);
7130 				printf(" -> ");
7131 				pf_print_host(pd->dst, 0, pd->af);
7132 				printf(" inner src: ");
7133 				pf_print_host(pd2.src, 0, pd2.af);
7134 				printf(" -> ");
7135 				pf_print_host(pd2.dst, 0, pd2.af);
7136 				printf("\n");
7137 			}
7138 			REASON_SET(reason, PFRES_BADSTATE);
7139 			return (PF_DROP);
7140 		}
7141 
7142 		switch (pd2.proto) {
7143 		case IPPROTO_TCP: {
7144 			struct tcphdr		 th;
7145 			u_int32_t		 seq;
7146 			struct pf_state_peer	*src, *dst;
7147 			u_int8_t		 dws;
7148 			int			 copyback = 0;
7149 
7150 			/*
7151 			 * Only the first 8 bytes of the TCP header can be
7152 			 * expected. Don't access any TCP header fields after
7153 			 * th_seq, an ackskew test is not possible.
7154 			 */
7155 			if (!pf_pull_hdr(m, off2, &th, 8, NULL, reason,
7156 			    pd2.af)) {
7157 				DPFPRINTF(PF_DEBUG_MISC,
7158 				    ("pf: ICMP error message too short "
7159 				    "(tcp)\n"));
7160 				return (PF_DROP);
7161 			}
7162 
7163 			key.af = pd2.af;
7164 			key.proto = IPPROTO_TCP;
7165 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
7166 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
7167 			key.port[pd2.sidx] = th.th_sport;
7168 			key.port[pd2.didx] = th.th_dport;
7169 
7170 			STATE_LOOKUP(kif, &key, *state, pd);
7171 
7172 			if (pd->dir == (*state)->direction) {
7173 				src = &(*state)->dst;
7174 				dst = &(*state)->src;
7175 			} else {
7176 				src = &(*state)->src;
7177 				dst = &(*state)->dst;
7178 			}
7179 
7180 			if (src->wscale && dst->wscale)
7181 				dws = dst->wscale & PF_WSCALE_MASK;
7182 			else
7183 				dws = 0;
7184 
7185 			/* Demodulate sequence number */
7186 			seq = ntohl(th.th_seq) - src->seqdiff;
7187 			if (src->seqdiff) {
7188 				pf_change_a(&th.th_seq, icmpsum,
7189 				    htonl(seq), 0);
7190 				copyback = 1;
7191 			}
7192 
7193 			if (!((*state)->state_flags & PFSTATE_SLOPPY) &&
7194 			    (!SEQ_GEQ(src->seqhi, seq) ||
7195 			    !SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)))) {
7196 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
7197 					printf("pf: BAD ICMP %d:%d ",
7198 					    icmptype, icmpcode);
7199 					pf_print_host(pd->src, 0, pd->af);
7200 					printf(" -> ");
7201 					pf_print_host(pd->dst, 0, pd->af);
7202 					printf(" state: ");
7203 					pf_print_state(*state);
7204 					printf(" seq=%u\n", seq);
7205 				}
7206 				REASON_SET(reason, PFRES_BADSTATE);
7207 				return (PF_DROP);
7208 			} else {
7209 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
7210 					printf("pf: OK ICMP %d:%d ",
7211 					    icmptype, icmpcode);
7212 					pf_print_host(pd->src, 0, pd->af);
7213 					printf(" -> ");
7214 					pf_print_host(pd->dst, 0, pd->af);
7215 					printf(" state: ");
7216 					pf_print_state(*state);
7217 					printf(" seq=%u\n", seq);
7218 				}
7219 			}
7220 
7221 			/* translate source/destination address, if necessary */
7222 			if ((*state)->key[PF_SK_WIRE] !=
7223 			    (*state)->key[PF_SK_STACK]) {
7224 				struct pf_state_key *nk =
7225 				    (*state)->key[pd->didx];
7226 
7227 				if (PF_ANEQ(pd2.src,
7228 				    &nk->addr[pd2.sidx], pd2.af) ||
7229 				    nk->port[pd2.sidx] != th.th_sport)
7230 					pf_change_icmp(pd2.src, &th.th_sport,
7231 					    daddr, &nk->addr[pd2.sidx],
7232 					    nk->port[pd2.sidx], NULL,
7233 					    pd2.ip_sum, icmpsum,
7234 					    pd->ip_sum, 0, pd2.af);
7235 
7236 				if (PF_ANEQ(pd2.dst,
7237 				    &nk->addr[pd2.didx], pd2.af) ||
7238 				    nk->port[pd2.didx] != th.th_dport)
7239 					pf_change_icmp(pd2.dst, &th.th_dport,
7240 					    saddr, &nk->addr[pd2.didx],
7241 					    nk->port[pd2.didx], NULL,
7242 					    pd2.ip_sum, icmpsum,
7243 					    pd->ip_sum, 0, pd2.af);
7244 				copyback = 1;
7245 			}
7246 
7247 			if (copyback) {
7248 				switch (pd2.af) {
7249 #ifdef INET
7250 				case AF_INET:
7251 					m_copyback(m, off, ICMP_MINLEN,
7252 					    (caddr_t )&pd->hdr.icmp);
7253 					m_copyback(m, ipoff2, sizeof(h2),
7254 					    (caddr_t )&h2);
7255 					break;
7256 #endif /* INET */
7257 #ifdef INET6
7258 				case AF_INET6:
7259 					m_copyback(m, off,
7260 					    sizeof(struct icmp6_hdr),
7261 					    (caddr_t )&pd->hdr.icmp6);
7262 					m_copyback(m, ipoff2, sizeof(h2_6),
7263 					    (caddr_t )&h2_6);
7264 					break;
7265 #endif /* INET6 */
7266 				}
7267 				m_copyback(m, off2, 8, (caddr_t)&th);
7268 			}
7269 
7270 			return (PF_PASS);
7271 			break;
7272 		}
7273 		case IPPROTO_UDP: {
7274 			struct udphdr		uh;
7275 
7276 			if (!pf_pull_hdr(m, off2, &uh, sizeof(uh),
7277 			    NULL, reason, pd2.af)) {
7278 				DPFPRINTF(PF_DEBUG_MISC,
7279 				    ("pf: ICMP error message too short "
7280 				    "(udp)\n"));
7281 				return (PF_DROP);
7282 			}
7283 
7284 			key.af = pd2.af;
7285 			key.proto = IPPROTO_UDP;
7286 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
7287 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
7288 			key.port[pd2.sidx] = uh.uh_sport;
7289 			key.port[pd2.didx] = uh.uh_dport;
7290 
7291 			STATE_LOOKUP(kif, &key, *state, pd);
7292 
7293 			/* translate source/destination address, if necessary */
7294 			if ((*state)->key[PF_SK_WIRE] !=
7295 			    (*state)->key[PF_SK_STACK]) {
7296 				struct pf_state_key *nk =
7297 				    (*state)->key[pd->didx];
7298 
7299 				if (PF_ANEQ(pd2.src,
7300 				    &nk->addr[pd2.sidx], pd2.af) ||
7301 				    nk->port[pd2.sidx] != uh.uh_sport)
7302 					pf_change_icmp(pd2.src, &uh.uh_sport,
7303 					    daddr, &nk->addr[pd2.sidx],
7304 					    nk->port[pd2.sidx], &uh.uh_sum,
7305 					    pd2.ip_sum, icmpsum,
7306 					    pd->ip_sum, 1, pd2.af);
7307 
7308 				if (PF_ANEQ(pd2.dst,
7309 				    &nk->addr[pd2.didx], pd2.af) ||
7310 				    nk->port[pd2.didx] != uh.uh_dport)
7311 					pf_change_icmp(pd2.dst, &uh.uh_dport,
7312 					    saddr, &nk->addr[pd2.didx],
7313 					    nk->port[pd2.didx], &uh.uh_sum,
7314 					    pd2.ip_sum, icmpsum,
7315 					    pd->ip_sum, 1, pd2.af);
7316 
7317 				switch (pd2.af) {
7318 #ifdef INET
7319 				case AF_INET:
7320 					m_copyback(m, off, ICMP_MINLEN,
7321 					    (caddr_t )&pd->hdr.icmp);
7322 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
7323 					break;
7324 #endif /* INET */
7325 #ifdef INET6
7326 				case AF_INET6:
7327 					m_copyback(m, off,
7328 					    sizeof(struct icmp6_hdr),
7329 					    (caddr_t )&pd->hdr.icmp6);
7330 					m_copyback(m, ipoff2, sizeof(h2_6),
7331 					    (caddr_t )&h2_6);
7332 					break;
7333 #endif /* INET6 */
7334 				}
7335 				m_copyback(m, off2, sizeof(uh), (caddr_t)&uh);
7336 			}
7337 			return (PF_PASS);
7338 			break;
7339 		}
7340 #ifdef INET
7341 		case IPPROTO_ICMP: {
7342 			struct icmp	*iih = &pd2.hdr.icmp;
7343 
7344 			if (!pf_pull_hdr(m, off2, iih, ICMP_MINLEN,
7345 			    NULL, reason, pd2.af)) {
7346 				DPFPRINTF(PF_DEBUG_MISC,
7347 				    ("pf: ICMP error message too short i"
7348 				    "(icmp)\n"));
7349 				return (PF_DROP);
7350 			}
7351 
7352 			icmpid = iih->icmp_id;
7353 			pf_icmp_mapping(&pd2, iih->icmp_type,
7354 			    &icmp_dir, &multi, &virtual_id, &virtual_type);
7355 
7356 			ret = pf_icmp_state_lookup(&key, &pd2, state, m, off,
7357 			    pd2.dir, kif, virtual_id, virtual_type,
7358 			    icmp_dir, &iidx, PF_ICMP_MULTI_NONE, 1);
7359 			if (ret >= 0) {
7360 				MPASS(*state == NULL);
7361 				return (ret);
7362 			}
7363 
7364 			/* translate source/destination address, if necessary */
7365 			if ((*state)->key[PF_SK_WIRE] !=
7366 			    (*state)->key[PF_SK_STACK]) {
7367 				struct pf_state_key *nk =
7368 				    (*state)->key[pd->didx];
7369 
7370 				if (PF_ANEQ(pd2.src,
7371 				    &nk->addr[pd2.sidx], pd2.af) ||
7372 				    (virtual_type == htons(ICMP_ECHO) &&
7373 				    nk->port[iidx] != iih->icmp_id))
7374 					pf_change_icmp(pd2.src,
7375 					    (virtual_type == htons(ICMP_ECHO)) ?
7376 					    &iih->icmp_id : NULL,
7377 					    daddr, &nk->addr[pd2.sidx],
7378 					    (virtual_type == htons(ICMP_ECHO)) ?
7379 					    nk->port[iidx] : 0, NULL,
7380 					    pd2.ip_sum, icmpsum,
7381 					    pd->ip_sum, 0, AF_INET);
7382 
7383 				if (PF_ANEQ(pd2.dst,
7384 				    &nk->addr[pd2.didx], pd2.af))
7385 					pf_change_icmp(pd2.dst, NULL, NULL,
7386 					    &nk->addr[pd2.didx], 0, NULL,
7387 					    pd2.ip_sum, icmpsum, pd->ip_sum, 0,
7388 					    AF_INET);
7389 
7390 				m_copyback(m, off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
7391 				m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
7392 				m_copyback(m, off2, ICMP_MINLEN, (caddr_t)iih);
7393 			}
7394 			return (PF_PASS);
7395 			break;
7396 		}
7397 #endif /* INET */
7398 #ifdef INET6
7399 		case IPPROTO_ICMPV6: {
7400 			struct icmp6_hdr	*iih = &pd2.hdr.icmp6;
7401 
7402 			if (!pf_pull_hdr(m, off2, iih,
7403 			    sizeof(struct icmp6_hdr), NULL, reason, pd2.af)) {
7404 				DPFPRINTF(PF_DEBUG_MISC,
7405 				    ("pf: ICMP error message too short "
7406 				    "(icmp6)\n"));
7407 				return (PF_DROP);
7408 			}
7409 
7410 			pf_icmp_mapping(&pd2, iih->icmp6_type,
7411 			    &icmp_dir, &multi, &virtual_id, &virtual_type);
7412 
7413 			ret = pf_icmp_state_lookup(&key, &pd2, state, m, off,
7414 			    pd->dir, kif, virtual_id, virtual_type,
7415 			    icmp_dir, &iidx, PF_ICMP_MULTI_NONE, 1);
7416 			if (ret >= 0) {
7417 				MPASS(*state == NULL);
7418 				if (ret == PF_DROP && pd2.af == AF_INET6 &&
7419 				    icmp_dir == PF_OUT) {
7420 					ret = pf_icmp_state_lookup(&key, &pd2,
7421 					    state, m, off, pd->dir, kif,
7422 					    virtual_id, virtual_type,
7423 					    icmp_dir, &iidx, multi, 1);
7424 					if (ret >= 0) {
7425 						MPASS(*state == NULL);
7426 						return (ret);
7427 					}
7428 				} else
7429 					return (ret);
7430 			}
7431 
7432 			/* translate source/destination address, if necessary */
7433 			if ((*state)->key[PF_SK_WIRE] !=
7434 			    (*state)->key[PF_SK_STACK]) {
7435 				struct pf_state_key *nk =
7436 				    (*state)->key[pd->didx];
7437 
7438 				if (PF_ANEQ(pd2.src,
7439 				    &nk->addr[pd2.sidx], pd2.af) ||
7440 				    ((virtual_type == htons(ICMP6_ECHO_REQUEST)) &&
7441 				    nk->port[pd2.sidx] != iih->icmp6_id))
7442 					pf_change_icmp(pd2.src,
7443 					    (virtual_type == htons(ICMP6_ECHO_REQUEST))
7444 					    ? &iih->icmp6_id : NULL,
7445 					    daddr, &nk->addr[pd2.sidx],
7446 					    (virtual_type == htons(ICMP6_ECHO_REQUEST))
7447 					    ? nk->port[iidx] : 0, NULL,
7448 					    pd2.ip_sum, icmpsum,
7449 					    pd->ip_sum, 0, AF_INET6);
7450 
7451 				if (PF_ANEQ(pd2.dst,
7452 				    &nk->addr[pd2.didx], pd2.af))
7453 					pf_change_icmp(pd2.dst, NULL, NULL,
7454 					    &nk->addr[pd2.didx], 0, NULL,
7455 					    pd2.ip_sum, icmpsum,
7456 					    pd->ip_sum, 0, AF_INET6);
7457 
7458 				m_copyback(m, off, sizeof(struct icmp6_hdr),
7459 				    (caddr_t)&pd->hdr.icmp6);
7460 				m_copyback(m, ipoff2, sizeof(h2_6), (caddr_t)&h2_6);
7461 				m_copyback(m, off2, sizeof(struct icmp6_hdr),
7462 				    (caddr_t)iih);
7463 			}
7464 			return (PF_PASS);
7465 			break;
7466 		}
7467 #endif /* INET6 */
7468 		default: {
7469 			key.af = pd2.af;
7470 			key.proto = pd2.proto;
7471 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
7472 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
7473 			key.port[0] = key.port[1] = 0;
7474 
7475 			STATE_LOOKUP(kif, &key, *state, pd);
7476 
7477 			/* translate source/destination address, if necessary */
7478 			if ((*state)->key[PF_SK_WIRE] !=
7479 			    (*state)->key[PF_SK_STACK]) {
7480 				struct pf_state_key *nk =
7481 				    (*state)->key[pd->didx];
7482 
7483 				if (PF_ANEQ(pd2.src,
7484 				    &nk->addr[pd2.sidx], pd2.af))
7485 					pf_change_icmp(pd2.src, NULL, daddr,
7486 					    &nk->addr[pd2.sidx], 0, NULL,
7487 					    pd2.ip_sum, icmpsum,
7488 					    pd->ip_sum, 0, pd2.af);
7489 
7490 				if (PF_ANEQ(pd2.dst,
7491 				    &nk->addr[pd2.didx], pd2.af))
7492 					pf_change_icmp(pd2.dst, NULL, saddr,
7493 					    &nk->addr[pd2.didx], 0, NULL,
7494 					    pd2.ip_sum, icmpsum,
7495 					    pd->ip_sum, 0, pd2.af);
7496 
7497 				switch (pd2.af) {
7498 #ifdef INET
7499 				case AF_INET:
7500 					m_copyback(m, off, ICMP_MINLEN,
7501 					    (caddr_t)&pd->hdr.icmp);
7502 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
7503 					break;
7504 #endif /* INET */
7505 #ifdef INET6
7506 				case AF_INET6:
7507 					m_copyback(m, off,
7508 					    sizeof(struct icmp6_hdr),
7509 					    (caddr_t )&pd->hdr.icmp6);
7510 					m_copyback(m, ipoff2, sizeof(h2_6),
7511 					    (caddr_t )&h2_6);
7512 					break;
7513 #endif /* INET6 */
7514 				}
7515 			}
7516 			return (PF_PASS);
7517 			break;
7518 		}
7519 		}
7520 	}
7521 }
7522 
7523 static int
7524 pf_test_state_other(struct pf_kstate **state, struct pfi_kkif *kif,
7525     struct mbuf *m, struct pf_pdesc *pd)
7526 {
7527 	struct pf_state_peer	*src, *dst;
7528 	struct pf_state_key_cmp	 key;
7529 	uint8_t			 psrc, pdst;
7530 
7531 	bzero(&key, sizeof(key));
7532 	key.af = pd->af;
7533 	key.proto = pd->proto;
7534 	if (pd->dir == PF_IN)	{
7535 		PF_ACPY(&key.addr[0], pd->src, key.af);
7536 		PF_ACPY(&key.addr[1], pd->dst, key.af);
7537 		key.port[0] = key.port[1] = 0;
7538 	} else {
7539 		PF_ACPY(&key.addr[1], pd->src, key.af);
7540 		PF_ACPY(&key.addr[0], pd->dst, key.af);
7541 		key.port[1] = key.port[0] = 0;
7542 	}
7543 
7544 	STATE_LOOKUP(kif, &key, *state, pd);
7545 
7546 	if (pd->dir == (*state)->direction) {
7547 		src = &(*state)->src;
7548 		dst = &(*state)->dst;
7549 		psrc = PF_PEER_SRC;
7550 		pdst = PF_PEER_DST;
7551 	} else {
7552 		src = &(*state)->dst;
7553 		dst = &(*state)->src;
7554 		psrc = PF_PEER_DST;
7555 		pdst = PF_PEER_SRC;
7556 	}
7557 
7558 	/* update states */
7559 	if (src->state < PFOTHERS_SINGLE)
7560 		pf_set_protostate(*state, psrc, PFOTHERS_SINGLE);
7561 	if (dst->state == PFOTHERS_SINGLE)
7562 		pf_set_protostate(*state, pdst, PFOTHERS_MULTIPLE);
7563 
7564 	/* update expire time */
7565 	(*state)->expire = pf_get_uptime();
7566 	if (src->state == PFOTHERS_MULTIPLE && dst->state == PFOTHERS_MULTIPLE)
7567 		(*state)->timeout = PFTM_OTHER_MULTIPLE;
7568 	else
7569 		(*state)->timeout = PFTM_OTHER_SINGLE;
7570 
7571 	/* translate source/destination address, if necessary */
7572 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
7573 		struct pf_state_key *nk = (*state)->key[pd->didx];
7574 
7575 		KASSERT(nk, ("%s: nk is null", __func__));
7576 		KASSERT(pd, ("%s: pd is null", __func__));
7577 		KASSERT(pd->src, ("%s: pd->src is null", __func__));
7578 		KASSERT(pd->dst, ("%s: pd->dst is null", __func__));
7579 		switch (pd->af) {
7580 #ifdef INET
7581 		case AF_INET:
7582 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET))
7583 				pf_change_a(&pd->src->v4.s_addr,
7584 				    pd->ip_sum,
7585 				    nk->addr[pd->sidx].v4.s_addr,
7586 				    0);
7587 
7588 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET))
7589 				pf_change_a(&pd->dst->v4.s_addr,
7590 				    pd->ip_sum,
7591 				    nk->addr[pd->didx].v4.s_addr,
7592 				    0);
7593 
7594 			break;
7595 #endif /* INET */
7596 #ifdef INET6
7597 		case AF_INET6:
7598 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET6))
7599 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
7600 
7601 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET6))
7602 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
7603 #endif /* INET6 */
7604 		}
7605 	}
7606 	return (PF_PASS);
7607 }
7608 
7609 /*
7610  * ipoff and off are measured from the start of the mbuf chain.
7611  * h must be at "ipoff" on the mbuf chain.
7612  */
7613 void *
7614 pf_pull_hdr(const struct mbuf *m, int off, void *p, int len,
7615     u_short *actionp, u_short *reasonp, sa_family_t af)
7616 {
7617 	switch (af) {
7618 #ifdef INET
7619 	case AF_INET: {
7620 		const struct ip	*h = mtod(m, struct ip *);
7621 		u_int16_t	 fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3;
7622 
7623 		if (fragoff) {
7624 			if (fragoff >= len)
7625 				ACTION_SET(actionp, PF_PASS);
7626 			else {
7627 				ACTION_SET(actionp, PF_DROP);
7628 				REASON_SET(reasonp, PFRES_FRAG);
7629 			}
7630 			return (NULL);
7631 		}
7632 		if (m->m_pkthdr.len < off + len ||
7633 		    ntohs(h->ip_len) < off + len) {
7634 			ACTION_SET(actionp, PF_DROP);
7635 			REASON_SET(reasonp, PFRES_SHORT);
7636 			return (NULL);
7637 		}
7638 		break;
7639 	}
7640 #endif /* INET */
7641 #ifdef INET6
7642 	case AF_INET6: {
7643 		const struct ip6_hdr	*h = mtod(m, struct ip6_hdr *);
7644 
7645 		if (m->m_pkthdr.len < off + len ||
7646 		    (ntohs(h->ip6_plen) + sizeof(struct ip6_hdr)) <
7647 		    (unsigned)(off + len)) {
7648 			ACTION_SET(actionp, PF_DROP);
7649 			REASON_SET(reasonp, PFRES_SHORT);
7650 			return (NULL);
7651 		}
7652 		break;
7653 	}
7654 #endif /* INET6 */
7655 	}
7656 	m_copydata(m, off, len, p);
7657 	return (p);
7658 }
7659 
7660 int
7661 pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kkif *kif,
7662     int rtableid)
7663 {
7664 	struct ifnet		*ifp;
7665 
7666 	/*
7667 	 * Skip check for addresses with embedded interface scope,
7668 	 * as they would always match anyway.
7669 	 */
7670 	if (af == AF_INET6 && IN6_IS_SCOPE_EMBED(&addr->v6))
7671 		return (1);
7672 
7673 	if (af != AF_INET && af != AF_INET6)
7674 		return (0);
7675 
7676 	if (kif == V_pfi_all)
7677 		return (1);
7678 
7679 	/* Skip checks for ipsec interfaces */
7680 	if (kif != NULL && kif->pfik_ifp->if_type == IFT_ENC)
7681 		return (1);
7682 
7683 	ifp = (kif != NULL) ? kif->pfik_ifp : NULL;
7684 
7685 	switch (af) {
7686 #ifdef INET6
7687 	case AF_INET6:
7688 		return (fib6_check_urpf(rtableid, &addr->v6, 0, NHR_NONE,
7689 		    ifp));
7690 #endif
7691 #ifdef INET
7692 	case AF_INET:
7693 		return (fib4_check_urpf(rtableid, addr->v4, 0, NHR_NONE,
7694 		    ifp));
7695 #endif
7696 	}
7697 
7698 	return (0);
7699 }
7700 
7701 #ifdef INET
7702 static void
7703 pf_route(struct mbuf **m, struct pf_krule *r, struct ifnet *oifp,
7704     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
7705 {
7706 	struct mbuf		*m0, *m1, *md;
7707 	struct sockaddr_in	dst;
7708 	struct ip		*ip;
7709 	struct pfi_kkif		*nkif = NULL;
7710 	struct ifnet		*ifp = NULL;
7711 	struct pf_addr		 naddr;
7712 	int			 error = 0;
7713 	uint16_t		 ip_len, ip_off;
7714 	uint16_t		 tmp;
7715 	int			 r_rt, r_dir;
7716 
7717 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
7718 
7719 	if (s) {
7720 		r_rt = s->rt;
7721 		r_dir = s->direction;
7722 	} else {
7723 		r_rt = r->rt;
7724 		r_dir = r->direction;
7725 	}
7726 
7727 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
7728 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
7729 	    __func__));
7730 
7731 	if ((pd->pf_mtag == NULL &&
7732 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
7733 	    pd->pf_mtag->routed++ > 3) {
7734 		m0 = *m;
7735 		*m = NULL;
7736 		goto bad_locked;
7737 	}
7738 
7739 	if (r_rt == PF_DUPTO) {
7740 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
7741 			if (s == NULL) {
7742 				ifp = r->rpool.cur->kif ?
7743 				    r->rpool.cur->kif->pfik_ifp : NULL;
7744 			} else {
7745 				ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7746 				/* If pfsync'd */
7747 				if (ifp == NULL && r->rpool.cur != NULL)
7748 					ifp = r->rpool.cur->kif ?
7749 					    r->rpool.cur->kif->pfik_ifp : NULL;
7750 				PF_STATE_UNLOCK(s);
7751 			}
7752 			if (ifp == oifp) {
7753 				/* When the 2nd interface is not skipped */
7754 				return;
7755 			} else {
7756 				m0 = *m;
7757 				*m = NULL;
7758 				goto bad;
7759 			}
7760 		} else {
7761 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
7762 			if (((m0 = m_dup(*m, M_NOWAIT)) == NULL)) {
7763 				if (s)
7764 					PF_STATE_UNLOCK(s);
7765 				return;
7766 			}
7767 		}
7768 	} else {
7769 		if ((r_rt == PF_REPLYTO) == (r_dir == pd->dir)) {
7770 			pf_dummynet(pd, s, r, m);
7771 			if (s)
7772 				PF_STATE_UNLOCK(s);
7773 			return;
7774 		}
7775 		m0 = *m;
7776 	}
7777 
7778 	ip = mtod(m0, struct ip *);
7779 
7780 	bzero(&dst, sizeof(dst));
7781 	dst.sin_family = AF_INET;
7782 	dst.sin_len = sizeof(dst);
7783 	dst.sin_addr = ip->ip_dst;
7784 
7785 	bzero(&naddr, sizeof(naddr));
7786 
7787 	if (s == NULL) {
7788 		if (TAILQ_EMPTY(&r->rpool.list)) {
7789 			DPFPRINTF(PF_DEBUG_URGENT,
7790 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
7791 			goto bad_locked;
7792 		}
7793 		pf_map_addr(AF_INET, r, (struct pf_addr *)&ip->ip_src,
7794 		    &naddr, &nkif, NULL);
7795 		if (!PF_AZERO(&naddr, AF_INET))
7796 			dst.sin_addr.s_addr = naddr.v4.s_addr;
7797 		ifp = nkif ? nkif->pfik_ifp : NULL;
7798 	} else {
7799 		struct pfi_kkif *kif;
7800 
7801 		if (!PF_AZERO(&s->rt_addr, AF_INET))
7802 			dst.sin_addr.s_addr =
7803 			    s->rt_addr.v4.s_addr;
7804 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7805 		kif = s->rt_kif;
7806 		/* If pfsync'd */
7807 		if (ifp == NULL && r->rpool.cur != NULL) {
7808 			ifp = r->rpool.cur->kif ?
7809 			    r->rpool.cur->kif->pfik_ifp : NULL;
7810 			kif = r->rpool.cur->kif;
7811 		}
7812 		if (ifp != NULL && kif != NULL &&
7813 		    r->rule_flag & PFRULE_IFBOUND &&
7814 		    r->rt == PF_REPLYTO &&
7815 		    s->kif == V_pfi_all) {
7816 			s->kif = kif;
7817 			s->orig_kif = oifp->if_pf_kif;
7818 		}
7819 
7820 		PF_STATE_UNLOCK(s);
7821 	}
7822 
7823 	if (ifp == NULL)
7824 		goto bad;
7825 
7826 	if (pd->dir == PF_IN) {
7827 		if (pf_test(AF_INET, PF_OUT, PFIL_FWD, ifp, &m0, inp,
7828 		    &pd->act) != PF_PASS)
7829 			goto bad;
7830 		else if (m0 == NULL)
7831 			goto done;
7832 		if (m0->m_len < sizeof(struct ip)) {
7833 			DPFPRINTF(PF_DEBUG_URGENT,
7834 			    ("%s: m0->m_len < sizeof(struct ip)\n", __func__));
7835 			goto bad;
7836 		}
7837 		ip = mtod(m0, struct ip *);
7838 	}
7839 
7840 	if (ifp->if_flags & IFF_LOOPBACK)
7841 		m0->m_flags |= M_SKIP_FIREWALL;
7842 
7843 	ip_len = ntohs(ip->ip_len);
7844 	ip_off = ntohs(ip->ip_off);
7845 
7846 	/* Copied from FreeBSD 10.0-CURRENT ip_output. */
7847 	m0->m_pkthdr.csum_flags |= CSUM_IP;
7848 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~ifp->if_hwassist) {
7849 		in_delayed_cksum(m0);
7850 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
7851 	}
7852 	if (m0->m_pkthdr.csum_flags & CSUM_SCTP & ~ifp->if_hwassist) {
7853 		pf_sctp_checksum(m0, (uint32_t)(ip->ip_hl << 2));
7854 		m0->m_pkthdr.csum_flags &= ~CSUM_SCTP;
7855 	}
7856 
7857 	if (pd->dir == PF_IN) {
7858 		/*
7859 		 * Make sure dummynet gets the correct direction, in case it needs to
7860 		 * re-inject later.
7861 		 */
7862 		pd->dir = PF_OUT;
7863 
7864 		/*
7865 		 * The following processing is actually the rest of the inbound processing, even
7866 		 * though we've marked it as outbound (so we don't look through dummynet) and it
7867 		 * happens after the outbound processing (pf_test(PF_OUT) above).
7868 		 * Swap the dummynet pipe numbers, because it's going to come to the wrong
7869 		 * conclusion about what direction it's processing, and we can't fix it or it
7870 		 * will re-inject incorrectly. Swapping the pipe numbers means that its incorrect
7871 		 * decision will pick the right pipe, and everything will mostly work as expected.
7872 		 */
7873 		tmp = pd->act.dnrpipe;
7874 		pd->act.dnrpipe = pd->act.dnpipe;
7875 		pd->act.dnpipe = tmp;
7876 	}
7877 
7878 	/*
7879 	 * If small enough for interface, or the interface will take
7880 	 * care of the fragmentation for us, we can just send directly.
7881 	 */
7882 	if (ip_len <= ifp->if_mtu ||
7883 	    (m0->m_pkthdr.csum_flags & ifp->if_hwassist & CSUM_TSO) != 0) {
7884 		ip->ip_sum = 0;
7885 		if (m0->m_pkthdr.csum_flags & CSUM_IP & ~ifp->if_hwassist) {
7886 			ip->ip_sum = in_cksum(m0, ip->ip_hl << 2);
7887 			m0->m_pkthdr.csum_flags &= ~CSUM_IP;
7888 		}
7889 		m_clrprotoflags(m0);	/* Avoid confusing lower layers. */
7890 
7891 		md = m0;
7892 		error = pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
7893 		if (md != NULL)
7894 			error = (*ifp->if_output)(ifp, md, sintosa(&dst), NULL);
7895 		goto done;
7896 	}
7897 
7898 	/* Balk when DF bit is set or the interface didn't support TSO. */
7899 	if ((ip_off & IP_DF) || (m0->m_pkthdr.csum_flags & CSUM_TSO)) {
7900 		error = EMSGSIZE;
7901 		KMOD_IPSTAT_INC(ips_cantfrag);
7902 		if (r_rt != PF_DUPTO) {
7903 			if (s && pd->nat_rule != NULL)
7904 				PACKET_UNDO_NAT(m0, pd,
7905 				    (ip->ip_hl << 2) + (ip_off & IP_OFFMASK),
7906 				    s);
7907 
7908 			icmp_error(m0, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG, 0,
7909 			    ifp->if_mtu);
7910 			goto done;
7911 		} else
7912 			goto bad;
7913 	}
7914 
7915 	error = ip_fragment(ip, &m0, ifp->if_mtu, ifp->if_hwassist);
7916 	if (error)
7917 		goto bad;
7918 
7919 	for (; m0; m0 = m1) {
7920 		m1 = m0->m_nextpkt;
7921 		m0->m_nextpkt = NULL;
7922 		if (error == 0) {
7923 			m_clrprotoflags(m0);
7924 			md = m0;
7925 			pd->pf_mtag = pf_find_mtag(md);
7926 			error = pf_dummynet_route(pd, s, r, ifp,
7927 			    sintosa(&dst), &md);
7928 			if (md != NULL)
7929 				error = (*ifp->if_output)(ifp, md,
7930 				    sintosa(&dst), NULL);
7931 		} else
7932 			m_freem(m0);
7933 	}
7934 
7935 	if (error == 0)
7936 		KMOD_IPSTAT_INC(ips_fragmented);
7937 
7938 done:
7939 	if (r_rt != PF_DUPTO)
7940 		*m = NULL;
7941 	return;
7942 
7943 bad_locked:
7944 	if (s)
7945 		PF_STATE_UNLOCK(s);
7946 bad:
7947 	m_freem(m0);
7948 	goto done;
7949 }
7950 #endif /* INET */
7951 
7952 #ifdef INET6
7953 static void
7954 pf_route6(struct mbuf **m, struct pf_krule *r, struct ifnet *oifp,
7955     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
7956 {
7957 	struct mbuf		*m0, *md;
7958 	struct sockaddr_in6	dst;
7959 	struct ip6_hdr		*ip6;
7960 	struct pfi_kkif		*nkif = NULL;
7961 	struct ifnet		*ifp = NULL;
7962 	struct pf_addr		 naddr;
7963 	int			 r_rt, r_dir;
7964 
7965 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
7966 
7967 	if (s) {
7968 		r_rt = s->rt;
7969 		r_dir = s->direction;
7970 	} else {
7971 		r_rt = r->rt;
7972 		r_dir = r->direction;
7973 	}
7974 
7975 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
7976 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
7977 	    __func__));
7978 
7979 	if ((pd->pf_mtag == NULL &&
7980 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
7981 	    pd->pf_mtag->routed++ > 3) {
7982 		m0 = *m;
7983 		*m = NULL;
7984 		goto bad_locked;
7985 	}
7986 
7987 	if (r_rt == PF_DUPTO) {
7988 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
7989 			if (s == NULL) {
7990 				ifp = r->rpool.cur->kif ?
7991 				    r->rpool.cur->kif->pfik_ifp : NULL;
7992 			} else {
7993 				ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7994 				/* If pfsync'd */
7995 				if (ifp == NULL && r->rpool.cur != NULL)
7996 					ifp = r->rpool.cur->kif ?
7997 					    r->rpool.cur->kif->pfik_ifp : NULL;
7998 				PF_STATE_UNLOCK(s);
7999 			}
8000 			if (ifp == oifp) {
8001 				/* When the 2nd interface is not skipped */
8002 				return;
8003 			} else {
8004 				m0 = *m;
8005 				*m = NULL;
8006 				goto bad;
8007 			}
8008 		} else {
8009 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
8010 			if (((m0 = m_dup(*m, M_NOWAIT)) == NULL)) {
8011 				if (s)
8012 					PF_STATE_UNLOCK(s);
8013 				return;
8014 			}
8015 		}
8016 	} else {
8017 		if ((r_rt == PF_REPLYTO) == (r_dir == pd->dir)) {
8018 			pf_dummynet(pd, s, r, m);
8019 			if (s)
8020 				PF_STATE_UNLOCK(s);
8021 			return;
8022 		}
8023 		m0 = *m;
8024 	}
8025 
8026 	ip6 = mtod(m0, struct ip6_hdr *);
8027 
8028 	bzero(&dst, sizeof(dst));
8029 	dst.sin6_family = AF_INET6;
8030 	dst.sin6_len = sizeof(dst);
8031 	dst.sin6_addr = ip6->ip6_dst;
8032 
8033 	bzero(&naddr, sizeof(naddr));
8034 
8035 	if (s == NULL) {
8036 		if (TAILQ_EMPTY(&r->rpool.list)) {
8037 			DPFPRINTF(PF_DEBUG_URGENT,
8038 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
8039 			goto bad_locked;
8040 		}
8041 		pf_map_addr(AF_INET6, r, (struct pf_addr *)&ip6->ip6_src,
8042 		    &naddr, &nkif, NULL);
8043 		if (!PF_AZERO(&naddr, AF_INET6))
8044 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
8045 			    &naddr, AF_INET6);
8046 		ifp = nkif ? nkif->pfik_ifp : NULL;
8047 	} else {
8048 		struct pfi_kkif *kif;
8049 
8050 		if (!PF_AZERO(&s->rt_addr, AF_INET6))
8051 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
8052 			    &s->rt_addr, AF_INET6);
8053 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
8054 		kif = s->rt_kif;
8055 		/* If pfsync'd */
8056 		if (ifp == NULL && r->rpool.cur != NULL) {
8057 			ifp = r->rpool.cur->kif ?
8058 			    r->rpool.cur->kif->pfik_ifp : NULL;
8059 			kif = r->rpool.cur->kif;
8060 		}
8061 		if (ifp != NULL && kif != NULL &&
8062 		    r->rule_flag & PFRULE_IFBOUND &&
8063 		    r->rt == PF_REPLYTO &&
8064 		    s->kif == V_pfi_all) {
8065 			s->kif = kif;
8066 			s->orig_kif = oifp->if_pf_kif;
8067 		}
8068 	}
8069 
8070 	if (s)
8071 		PF_STATE_UNLOCK(s);
8072 
8073 	if (ifp == NULL)
8074 		goto bad;
8075 
8076 	if (pd->dir == PF_IN) {
8077 		if (pf_test(AF_INET6, PF_OUT, PFIL_FWD, ifp, &m0, inp,
8078 		    &pd->act) != PF_PASS)
8079 			goto bad;
8080 		else if (m0 == NULL)
8081 			goto done;
8082 		if (m0->m_len < sizeof(struct ip6_hdr)) {
8083 			DPFPRINTF(PF_DEBUG_URGENT,
8084 			    ("%s: m0->m_len < sizeof(struct ip6_hdr)\n",
8085 			    __func__));
8086 			goto bad;
8087 		}
8088 		ip6 = mtod(m0, struct ip6_hdr *);
8089 	}
8090 
8091 	if (ifp->if_flags & IFF_LOOPBACK)
8092 		m0->m_flags |= M_SKIP_FIREWALL;
8093 
8094 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 &
8095 	    ~ifp->if_hwassist) {
8096 		uint32_t plen = m0->m_pkthdr.len - sizeof(*ip6);
8097 		in6_delayed_cksum(m0, plen, sizeof(struct ip6_hdr));
8098 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
8099 	}
8100 
8101 	/*
8102 	 * If the packet is too large for the outgoing interface,
8103 	 * send back an icmp6 error.
8104 	 */
8105 	if (IN6_IS_SCOPE_EMBED(&dst.sin6_addr))
8106 		dst.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
8107 	if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu) {
8108 		md = m0;
8109 		pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
8110 		if (md != NULL)
8111 			nd6_output_ifp(ifp, ifp, md, &dst, NULL);
8112 	}
8113 	else {
8114 		in6_ifstat_inc(ifp, ifs6_in_toobig);
8115 		if (r_rt != PF_DUPTO) {
8116 			if (s && pd->nat_rule != NULL)
8117 				PACKET_UNDO_NAT(m0, pd,
8118 				    ((caddr_t)ip6 - m0->m_data) +
8119 				    sizeof(struct ip6_hdr), s);
8120 
8121 			icmp6_error(m0, ICMP6_PACKET_TOO_BIG, 0, ifp->if_mtu);
8122 		} else
8123 			goto bad;
8124 	}
8125 
8126 done:
8127 	if (r_rt != PF_DUPTO)
8128 		*m = NULL;
8129 	return;
8130 
8131 bad_locked:
8132 	if (s)
8133 		PF_STATE_UNLOCK(s);
8134 bad:
8135 	m_freem(m0);
8136 	goto done;
8137 }
8138 #endif /* INET6 */
8139 
8140 /*
8141  * FreeBSD supports cksum offloads for the following drivers.
8142  *  em(4), fxp(4), lge(4), nge(4), re(4), ti(4), txp(4), xl(4)
8143  *
8144  * CSUM_DATA_VALID | CSUM_PSEUDO_HDR :
8145  *  network driver performed cksum including pseudo header, need to verify
8146  *   csum_data
8147  * CSUM_DATA_VALID :
8148  *  network driver performed cksum, needs to additional pseudo header
8149  *  cksum computation with partial csum_data(i.e. lack of H/W support for
8150  *  pseudo header, for instance sk(4) and possibly gem(4))
8151  *
8152  * After validating the cksum of packet, set both flag CSUM_DATA_VALID and
8153  * CSUM_PSEUDO_HDR in order to avoid recomputation of the cksum in upper
8154  * TCP/UDP layer.
8155  * Also, set csum_data to 0xffff to force cksum validation.
8156  */
8157 static int
8158 pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t af)
8159 {
8160 	u_int16_t sum = 0;
8161 	int hw_assist = 0;
8162 	struct ip *ip;
8163 
8164 	if (off < sizeof(struct ip) || len < sizeof(struct udphdr))
8165 		return (1);
8166 	if (m->m_pkthdr.len < off + len)
8167 		return (1);
8168 
8169 	switch (p) {
8170 	case IPPROTO_TCP:
8171 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
8172 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
8173 				sum = m->m_pkthdr.csum_data;
8174 			} else {
8175 				ip = mtod(m, struct ip *);
8176 				sum = in_pseudo(ip->ip_src.s_addr,
8177 				ip->ip_dst.s_addr, htonl((u_short)len +
8178 				m->m_pkthdr.csum_data + IPPROTO_TCP));
8179 			}
8180 			sum ^= 0xffff;
8181 			++hw_assist;
8182 		}
8183 		break;
8184 	case IPPROTO_UDP:
8185 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
8186 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
8187 				sum = m->m_pkthdr.csum_data;
8188 			} else {
8189 				ip = mtod(m, struct ip *);
8190 				sum = in_pseudo(ip->ip_src.s_addr,
8191 				ip->ip_dst.s_addr, htonl((u_short)len +
8192 				m->m_pkthdr.csum_data + IPPROTO_UDP));
8193 			}
8194 			sum ^= 0xffff;
8195 			++hw_assist;
8196 		}
8197 		break;
8198 	case IPPROTO_ICMP:
8199 #ifdef INET6
8200 	case IPPROTO_ICMPV6:
8201 #endif /* INET6 */
8202 		break;
8203 	default:
8204 		return (1);
8205 	}
8206 
8207 	if (!hw_assist) {
8208 		switch (af) {
8209 		case AF_INET:
8210 			if (p == IPPROTO_ICMP) {
8211 				if (m->m_len < off)
8212 					return (1);
8213 				m->m_data += off;
8214 				m->m_len -= off;
8215 				sum = in_cksum(m, len);
8216 				m->m_data -= off;
8217 				m->m_len += off;
8218 			} else {
8219 				if (m->m_len < sizeof(struct ip))
8220 					return (1);
8221 				sum = in4_cksum(m, p, off, len);
8222 			}
8223 			break;
8224 #ifdef INET6
8225 		case AF_INET6:
8226 			if (m->m_len < sizeof(struct ip6_hdr))
8227 				return (1);
8228 			sum = in6_cksum(m, p, off, len);
8229 			break;
8230 #endif /* INET6 */
8231 		default:
8232 			return (1);
8233 		}
8234 	}
8235 	if (sum) {
8236 		switch (p) {
8237 		case IPPROTO_TCP:
8238 		    {
8239 			KMOD_TCPSTAT_INC(tcps_rcvbadsum);
8240 			break;
8241 		    }
8242 		case IPPROTO_UDP:
8243 		    {
8244 			KMOD_UDPSTAT_INC(udps_badsum);
8245 			break;
8246 		    }
8247 #ifdef INET
8248 		case IPPROTO_ICMP:
8249 		    {
8250 			KMOD_ICMPSTAT_INC(icps_checksum);
8251 			break;
8252 		    }
8253 #endif
8254 #ifdef INET6
8255 		case IPPROTO_ICMPV6:
8256 		    {
8257 			KMOD_ICMP6STAT_INC(icp6s_checksum);
8258 			break;
8259 		    }
8260 #endif /* INET6 */
8261 		}
8262 		return (1);
8263 	} else {
8264 		if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
8265 			m->m_pkthdr.csum_flags |=
8266 			    (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
8267 			m->m_pkthdr.csum_data = 0xffff;
8268 		}
8269 	}
8270 	return (0);
8271 }
8272 
8273 static bool
8274 pf_pdesc_to_dnflow(const struct pf_pdesc *pd, const struct pf_krule *r,
8275     const struct pf_kstate *s, struct ip_fw_args *dnflow)
8276 {
8277 	int dndir = r->direction;
8278 
8279 	if (s && dndir == PF_INOUT) {
8280 		dndir = s->direction;
8281 	} else if (dndir == PF_INOUT) {
8282 		/* Assume primary direction. Happens when we've set dnpipe in
8283 		 * the ethernet level code. */
8284 		dndir = pd->dir;
8285 	}
8286 
8287 	if (pd->pf_mtag->flags & PF_MTAG_FLAG_DUMMYNETED)
8288 		return (false);
8289 
8290 	memset(dnflow, 0, sizeof(*dnflow));
8291 
8292 	if (pd->dport != NULL)
8293 		dnflow->f_id.dst_port = ntohs(*pd->dport);
8294 	if (pd->sport != NULL)
8295 		dnflow->f_id.src_port = ntohs(*pd->sport);
8296 
8297 	if (pd->dir == PF_IN)
8298 		dnflow->flags |= IPFW_ARGS_IN;
8299 	else
8300 		dnflow->flags |= IPFW_ARGS_OUT;
8301 
8302 	if (pd->dir != dndir && pd->act.dnrpipe) {
8303 		dnflow->rule.info = pd->act.dnrpipe;
8304 	}
8305 	else if (pd->dir == dndir && pd->act.dnpipe) {
8306 		dnflow->rule.info = pd->act.dnpipe;
8307 	}
8308 	else {
8309 		return (false);
8310 	}
8311 
8312 	dnflow->rule.info |= IPFW_IS_DUMMYNET;
8313 	if (r->free_flags & PFRULE_DN_IS_PIPE || pd->act.flags & PFSTATE_DN_IS_PIPE)
8314 		dnflow->rule.info |= IPFW_IS_PIPE;
8315 
8316 	dnflow->f_id.proto = pd->proto;
8317 	dnflow->f_id.extra = dnflow->rule.info;
8318 	switch (pd->af) {
8319 	case AF_INET:
8320 		dnflow->f_id.addr_type = 4;
8321 		dnflow->f_id.src_ip = ntohl(pd->src->v4.s_addr);
8322 		dnflow->f_id.dst_ip = ntohl(pd->dst->v4.s_addr);
8323 		break;
8324 	case AF_INET6:
8325 		dnflow->flags |= IPFW_ARGS_IP6;
8326 		dnflow->f_id.addr_type = 6;
8327 		dnflow->f_id.src_ip6 = pd->src->v6;
8328 		dnflow->f_id.dst_ip6 = pd->dst->v6;
8329 		break;
8330 	default:
8331 		panic("Invalid AF");
8332 		break;
8333 	}
8334 
8335 	return (true);
8336 }
8337 
8338 int
8339 pf_test_eth(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
8340     struct inpcb *inp)
8341 {
8342 	struct pfi_kkif		*kif;
8343 	struct mbuf		*m = *m0;
8344 
8345 	M_ASSERTPKTHDR(m);
8346 	MPASS(ifp->if_vnet == curvnet);
8347 	NET_EPOCH_ASSERT();
8348 
8349 	if (!V_pf_status.running)
8350 		return (PF_PASS);
8351 
8352 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
8353 
8354 	if (kif == NULL) {
8355 		DPFPRINTF(PF_DEBUG_URGENT,
8356 		    ("%s: kif == NULL, if_xname %s\n", __func__, ifp->if_xname));
8357 		return (PF_DROP);
8358 	}
8359 	if (kif->pfik_flags & PFI_IFLAG_SKIP)
8360 		return (PF_PASS);
8361 
8362 	if (m->m_flags & M_SKIP_FIREWALL)
8363 		return (PF_PASS);
8364 
8365 	if (__predict_false(! M_WRITABLE(*m0))) {
8366 		m = *m0 = m_unshare(*m0, M_NOWAIT);
8367 		if (*m0 == NULL)
8368 			return (PF_DROP);
8369 	}
8370 
8371 	/* Stateless! */
8372 	return (pf_test_eth_rule(dir, kif, m0));
8373 }
8374 
8375 static __inline void
8376 pf_dummynet_flag_remove(struct mbuf *m, struct pf_mtag *pf_mtag)
8377 {
8378 	struct m_tag *mtag;
8379 
8380 	pf_mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
8381 
8382 	/* dummynet adds this tag, but pf does not need it,
8383 	 * and keeping it creates unexpected behavior,
8384 	 * e.g. in case of divert(4) usage right after dummynet. */
8385 	mtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL);
8386 	if (mtag != NULL)
8387 		m_tag_delete(m, mtag);
8388 }
8389 
8390 static int
8391 pf_dummynet(struct pf_pdesc *pd, struct pf_kstate *s,
8392     struct pf_krule *r, struct mbuf **m0)
8393 {
8394 	return (pf_dummynet_route(pd, s, r, NULL, NULL, m0));
8395 }
8396 
8397 static int
8398 pf_dummynet_route(struct pf_pdesc *pd, struct pf_kstate *s,
8399     struct pf_krule *r, struct ifnet *ifp, struct sockaddr *sa,
8400     struct mbuf **m0)
8401 {
8402 	NET_EPOCH_ASSERT();
8403 
8404 	if (pd->act.dnpipe || pd->act.dnrpipe) {
8405 		struct ip_fw_args dnflow;
8406 		if (ip_dn_io_ptr == NULL) {
8407 			m_freem(*m0);
8408 			*m0 = NULL;
8409 			return (ENOMEM);
8410 		}
8411 
8412 		if (pd->pf_mtag == NULL &&
8413 		    ((pd->pf_mtag = pf_get_mtag(*m0)) == NULL)) {
8414 			m_freem(*m0);
8415 			*m0 = NULL;
8416 			return (ENOMEM);
8417 		}
8418 
8419 		if (ifp != NULL) {
8420 			pd->pf_mtag->flags |= PF_MTAG_FLAG_ROUTE_TO;
8421 
8422 			pd->pf_mtag->if_index = ifp->if_index;
8423 			pd->pf_mtag->if_idxgen = ifp->if_idxgen;
8424 
8425 			MPASS(sa != NULL);
8426 
8427 			if (pd->af == AF_INET)
8428 				memcpy(&pd->pf_mtag->dst, sa,
8429 				    sizeof(struct sockaddr_in));
8430 			else
8431 				memcpy(&pd->pf_mtag->dst, sa,
8432 				    sizeof(struct sockaddr_in6));
8433 		}
8434 
8435 		if (s != NULL && s->nat_rule.ptr != NULL &&
8436 		    s->nat_rule.ptr->action == PF_RDR &&
8437 		    (
8438 #ifdef INET
8439 		    (pd->af == AF_INET && IN_LOOPBACK(ntohl(pd->dst->v4.s_addr))) ||
8440 #endif
8441 		    (pd->af == AF_INET6 && IN6_IS_ADDR_LOOPBACK(&pd->dst->v6)))) {
8442 			/*
8443 			 * If we're redirecting to loopback mark this packet
8444 			 * as being local. Otherwise it might get dropped
8445 			 * if dummynet re-injects.
8446 			 */
8447 			(*m0)->m_pkthdr.rcvif = V_loif;
8448 		}
8449 
8450 		if (pf_pdesc_to_dnflow(pd, r, s, &dnflow)) {
8451 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
8452 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNETED;
8453 			ip_dn_io_ptr(m0, &dnflow);
8454 			if (*m0 != NULL) {
8455 				pd->pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
8456 				pf_dummynet_flag_remove(*m0, pd->pf_mtag);
8457 			}
8458 		}
8459 	}
8460 
8461 	return (0);
8462 }
8463 
8464 static int
8465 pf_setup_pdesc(sa_family_t af, int dir, struct pf_pdesc *pd, struct mbuf **m0,
8466     u_short *action, u_short *reason, struct pfi_kkif *kif, struct pf_krule **a,
8467     struct pf_krule **r, struct pf_kstate **s, struct pf_kruleset **ruleset,
8468     int *off, int *hdrlen, struct inpcb *inp,
8469     struct pf_rule_actions *default_actions)
8470 {
8471 	struct mbuf *m = *m0;
8472 
8473 	memset(pd, 0, sizeof(*pd));
8474 	pd->dir = dir;
8475 
8476 	TAILQ_INIT(&pd->sctp_multihome_jobs);
8477 	if (default_actions != NULL)
8478 		memcpy(&pd->act, default_actions, sizeof(pd->act));
8479 	pd->pf_mtag = pf_find_mtag(m);
8480 
8481 	if (pd->pf_mtag && pd->pf_mtag->dnpipe) {
8482 		pd->act.dnpipe = pd->pf_mtag->dnpipe;
8483 		pd->act.flags = pd->pf_mtag->dnflags;
8484 	}
8485 
8486 	pd->af = af;
8487 
8488 	switch (af) {
8489 #ifdef INET
8490 	case AF_INET: {
8491 		struct ip *h;
8492 
8493 		if (__predict_false(m->m_len < sizeof(struct ip)) &&
8494 		    (m = *m0 = m_pullup(*m0, sizeof(struct ip))) == NULL) {
8495 			DPFPRINTF(PF_DEBUG_URGENT,
8496 			    ("pf_test: m_len < sizeof(struct ip), pullup failed\n"));
8497 			*action = PF_DROP;
8498 			REASON_SET(reason, PFRES_SHORT);
8499 			return (-1);
8500 		}
8501 
8502 		if (pf_normalize_ip(m0, kif, reason, pd) != PF_PASS) {
8503 			/* We do IP header normalization and packet reassembly here */
8504 			*action = PF_DROP;
8505 			return (-1);
8506 		}
8507 		m = *m0;
8508 
8509 		h = mtod(m, struct ip *);
8510 		*off = h->ip_hl << 2;
8511 		if (*off < (int)sizeof(*h)) {
8512 			*action = PF_DROP;
8513 			REASON_SET(reason, PFRES_SHORT);
8514 			return (-1);
8515 		}
8516 		pd->src = (struct pf_addr *)&h->ip_src;
8517 		pd->dst = (struct pf_addr *)&h->ip_dst;
8518 		pd->sport = pd->dport = NULL;
8519 		pd->ip_sum = &h->ip_sum;
8520 		pd->proto_sum = NULL;
8521 		pd->virtual_proto = pd->proto = h->ip_p;
8522 		pd->dir = dir;
8523 		pd->sidx = (dir == PF_IN) ? 0 : 1;
8524 		pd->didx = (dir == PF_IN) ? 1 : 0;
8525 		pd->tos = h->ip_tos;
8526 		pd->tot_len = ntohs(h->ip_len);
8527 		pd->act.rtableid = -1;
8528 
8529 		if (h->ip_hl > 5)	/* has options */
8530 			pd->badopts++;
8531 
8532 		if (h->ip_off & htons(IP_MF | IP_OFFMASK)) {
8533 			/*
8534 			 * handle fragments that aren't reassembled by
8535 			 * normalization
8536 			 */
8537 			pd->virtual_proto = PF_VPROTO_FRAGMENT;
8538 			if (kif == NULL || r == NULL)   /* pflog */
8539 				*action = PF_DROP;
8540 			else
8541 				*action = pf_test_rule(r, s, kif, m, *off,
8542 				    pd, a, ruleset, inp, *hdrlen);
8543 			if (*action != PF_PASS)
8544 				REASON_SET(reason, PFRES_FRAG);
8545 			return (-1);
8546 		}
8547 
8548 		break;
8549 	}
8550 #endif
8551 #ifdef INET6
8552 	case AF_INET6: {
8553 		struct ip6_hdr *h;
8554 		int terminal = 0;
8555 
8556 		if (__predict_false(m->m_len < sizeof(struct ip6_hdr)) &&
8557 		    (m = *m0 = m_pullup(*m0, sizeof(struct ip6_hdr))) == NULL) {
8558 			DPFPRINTF(PF_DEBUG_URGENT,
8559 			    ("pf_test6: m_len < sizeof(struct ip6_hdr)"
8560 			     ", pullup failed\n"));
8561 			*action = PF_DROP;
8562 			REASON_SET(reason, PFRES_SHORT);
8563 			return (-1);
8564 		}
8565 
8566 		/* We do IP header normalization and packet reassembly here */
8567 		if (pf_normalize_ip6(m0, kif, reason, pd) != PF_PASS) {
8568 			*action = PF_DROP;
8569 			return (-1);
8570 		}
8571 		m = *m0;
8572 
8573 		h = mtod(m, struct ip6_hdr *);
8574 		pd->src = (struct pf_addr *)&h->ip6_src;
8575 		pd->dst = (struct pf_addr *)&h->ip6_dst;
8576 		pd->sport = pd->dport = NULL;
8577 		pd->ip_sum = NULL;
8578 		pd->proto_sum = NULL;
8579 		pd->dir = dir;
8580 		pd->sidx = (dir == PF_IN) ? 0 : 1;
8581 		pd->didx = (dir == PF_IN) ? 1 : 0;
8582 		pd->tos = IPV6_DSCP(h);
8583 		pd->tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr);
8584 		*off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr);
8585 		pd->virtual_proto = pd->proto = h->ip6_nxt;
8586 		pd->act.rtableid = -1;
8587 
8588 		do {
8589 			switch (pd->proto) {
8590 			case IPPROTO_FRAGMENT:
8591 				pd->virtual_proto = PF_VPROTO_FRAGMENT;
8592 				if (kif == NULL || r == NULL) /* pflog */
8593 					*action = PF_DROP;
8594 				else
8595 					*action = pf_test_rule(r, s, kif, m, *off,
8596 					    pd, a, ruleset, inp,
8597 					    *hdrlen);
8598 				if (*action == PF_DROP)
8599 					REASON_SET(reason, PFRES_FRAG);
8600 				return (-1);
8601 			case IPPROTO_ROUTING: {
8602 				struct ip6_rthdr rthdr;
8603 
8604 				if (pd->badopts++) {
8605 					DPFPRINTF(PF_DEBUG_MISC,
8606 					    ("pf: IPv6 more than one rthdr"));
8607 					*action = PF_DROP;
8608 					REASON_SET(reason, PFRES_IPOPTIONS);
8609 					return (-1);
8610 				}
8611 				if (!pf_pull_hdr(m, *off, &rthdr, sizeof(rthdr),
8612 					NULL, reason, pd->af)) {
8613 					DPFPRINTF(PF_DEBUG_MISC,
8614 					    ("pf: IPv6 short rthdr"));
8615 					*action = PF_DROP;
8616 					REASON_SET(reason, PFRES_SHORT);
8617 					return (-1);
8618 				}
8619 				if (rthdr.ip6r_type == IPV6_RTHDR_TYPE_0) {
8620 					DPFPRINTF(PF_DEBUG_MISC,
8621 					    ("pf: IPv6 rthdr0"));
8622 					*action = PF_DROP;
8623 					REASON_SET(reason, PFRES_IPOPTIONS);
8624 					return (-1);
8625 				}
8626 				/* FALLTHROUGH */
8627 			}
8628 			case IPPROTO_AH:
8629 			case IPPROTO_HOPOPTS:
8630 			case IPPROTO_DSTOPTS: {
8631 				/* get next header and header length */
8632 				struct ip6_ext opt6;
8633 
8634 				if (!pf_pull_hdr(m, *off, &opt6, sizeof(opt6),
8635 					NULL, reason, pd->af)) {
8636 					DPFPRINTF(PF_DEBUG_MISC,
8637 					    ("pf: IPv6 short opt"));
8638 					*action = PF_DROP;
8639 					return (-1);
8640 				}
8641 				if (pd->proto == IPPROTO_AH)
8642 					*off += (opt6.ip6e_len + 2) * 4;
8643 				else
8644 					*off += (opt6.ip6e_len + 1) * 8;
8645 				pd->virtual_proto = pd->proto = opt6.ip6e_nxt;
8646 				/* goto the next header */
8647 				break;
8648 			}
8649 			default:
8650 				terminal++;
8651 				break;
8652 			}
8653 		} while (!terminal);
8654 
8655 		break;
8656 	}
8657 #endif
8658 	default:
8659 		panic("pf_setup_pdesc called with illegal af %u", af);
8660 	}
8661 
8662 	switch (pd->proto) {
8663 	case IPPROTO_TCP: {
8664 		struct tcphdr *th = &pd->hdr.tcp;
8665 
8666 		if (!pf_pull_hdr(m, *off, th, sizeof(*th), action,
8667 			reason, af)) {
8668 			*action = PF_DROP;
8669 			REASON_SET(reason, PFRES_SHORT);
8670 			return (-1);
8671 		}
8672 		*hdrlen = sizeof(*th);
8673 		pd->p_len = pd->tot_len - *off - (th->th_off << 2);
8674 		pd->sport = &th->th_sport;
8675 		pd->dport = &th->th_dport;
8676 		break;
8677 	}
8678 	case IPPROTO_UDP: {
8679 		struct udphdr *uh = &pd->hdr.udp;
8680 
8681 		if (!pf_pull_hdr(m, *off, uh, sizeof(*uh), action,
8682 			reason, af)) {
8683 			*action = PF_DROP;
8684 			REASON_SET(reason, PFRES_SHORT);
8685 			return (-1);
8686 		}
8687 		*hdrlen = sizeof(*uh);
8688 		if (uh->uh_dport == 0 ||
8689 		    ntohs(uh->uh_ulen) > m->m_pkthdr.len - *off ||
8690 		    ntohs(uh->uh_ulen) < sizeof(struct udphdr)) {
8691 			*action = PF_DROP;
8692 			REASON_SET(reason, PFRES_SHORT);
8693 			return (-1);
8694 		}
8695 		pd->sport = &uh->uh_sport;
8696 		pd->dport = &uh->uh_dport;
8697 		break;
8698 	}
8699 	case IPPROTO_SCTP: {
8700 		if (!pf_pull_hdr(m, *off, &pd->hdr.sctp, sizeof(pd->hdr.sctp),
8701 		    action, reason, af)) {
8702 			*action = PF_DROP;
8703 			REASON_SET(reason, PFRES_SHORT);
8704 			return (-1);
8705 		}
8706 		*hdrlen = sizeof(pd->hdr.sctp);
8707 		pd->p_len = pd->tot_len - *off;
8708 
8709 		pd->sport = &pd->hdr.sctp.src_port;
8710 		pd->dport = &pd->hdr.sctp.dest_port;
8711 		if (pd->hdr.sctp.src_port == 0 || pd->hdr.sctp.dest_port == 0) {
8712 			*action = PF_DROP;
8713 			REASON_SET(reason, PFRES_SHORT);
8714 			return (-1);
8715 		}
8716 		if (pf_scan_sctp(m, *off, pd, kif) != PF_PASS) {
8717 			*action = PF_DROP;
8718 			REASON_SET(reason, PFRES_SHORT);
8719 			return (-1);
8720 		}
8721 		break;
8722 	}
8723 	case IPPROTO_ICMP: {
8724 		if (!pf_pull_hdr(m, *off, &pd->hdr.icmp, ICMP_MINLEN,
8725 			action, reason, af)) {
8726 			*action = PF_DROP;
8727 			REASON_SET(reason, PFRES_SHORT);
8728 			return (-1);
8729 		}
8730 		*hdrlen = ICMP_MINLEN;
8731 		break;
8732 	}
8733 #ifdef INET6
8734 	case IPPROTO_ICMPV6: {
8735 		size_t icmp_hlen = sizeof(struct icmp6_hdr);
8736 
8737 		if (!pf_pull_hdr(m, *off, &pd->hdr.icmp6, icmp_hlen,
8738 			action, reason, af)) {
8739 			*action = PF_DROP;
8740 			REASON_SET(reason, PFRES_SHORT);
8741 			return (-1);
8742 		}
8743 		/* ICMP headers we look further into to match state */
8744 		switch (pd->hdr.icmp6.icmp6_type) {
8745 		case MLD_LISTENER_QUERY:
8746 		case MLD_LISTENER_REPORT:
8747 			icmp_hlen = sizeof(struct mld_hdr);
8748 			break;
8749 		case ND_NEIGHBOR_SOLICIT:
8750 		case ND_NEIGHBOR_ADVERT:
8751 			icmp_hlen = sizeof(struct nd_neighbor_solicit);
8752 			break;
8753 		}
8754 		if (icmp_hlen > sizeof(struct icmp6_hdr) &&
8755 		    !pf_pull_hdr(m, *off, &pd->hdr.icmp6, icmp_hlen,
8756 			action, reason, af)) {
8757 			*action = PF_DROP;
8758 			REASON_SET(reason, PFRES_SHORT);
8759 			return (-1);
8760 		}
8761 		*hdrlen = icmp_hlen;
8762 		break;
8763 	}
8764 #endif
8765 	}
8766 	return (0);
8767 }
8768 
8769 static void
8770 pf_counters_inc(int action, struct pf_pdesc *pd,
8771     struct pfi_kkif *kif, struct pf_kstate *s,
8772     struct pf_krule *r, struct pf_krule *a)
8773 {
8774 	struct pf_krule		*tr, *nr;
8775 	int			 dir = pd->dir;
8776 	int			 dirndx;
8777 
8778 	pf_counter_u64_critical_enter();
8779 	pf_counter_u64_add_protected(
8780 	    &kif->pfik_bytes[pd->af == AF_INET6][dir == PF_OUT][action != PF_PASS],
8781 	    pd->tot_len);
8782 	pf_counter_u64_add_protected(
8783 	    &kif->pfik_packets[pd->af == AF_INET6][dir == PF_OUT][action != PF_PASS],
8784 	    1);
8785 
8786 	if (action == PF_PASS || r->action == PF_DROP) {
8787 		dirndx = (dir == PF_OUT);
8788 		pf_counter_u64_add_protected(&r->packets[dirndx], 1);
8789 		pf_counter_u64_add_protected(&r->bytes[dirndx], pd->tot_len);
8790 		pf_update_timestamp(r);
8791 
8792 		if (a != NULL) {
8793 			pf_counter_u64_add_protected(&a->packets[dirndx], 1);
8794 			pf_counter_u64_add_protected(&a->bytes[dirndx], pd->tot_len);
8795 		}
8796 		if (s != NULL) {
8797 			struct pf_krule_item	*ri;
8798 
8799 			if (s->nat_rule.ptr != NULL) {
8800 				pf_counter_u64_add_protected(&s->nat_rule.ptr->packets[dirndx],
8801 				    1);
8802 				pf_counter_u64_add_protected(&s->nat_rule.ptr->bytes[dirndx],
8803 				    pd->tot_len);
8804 			}
8805 			if (s->src_node != NULL) {
8806 				counter_u64_add(s->src_node->packets[dirndx],
8807 				    1);
8808 				counter_u64_add(s->src_node->bytes[dirndx],
8809 				    pd->tot_len);
8810 			}
8811 			if (s->nat_src_node != NULL) {
8812 				counter_u64_add(s->nat_src_node->packets[dirndx],
8813 				    1);
8814 				counter_u64_add(s->nat_src_node->bytes[dirndx],
8815 				    pd->tot_len);
8816 			}
8817 			dirndx = (dir == s->direction) ? 0 : 1;
8818 			s->packets[dirndx]++;
8819 			s->bytes[dirndx] += pd->tot_len;
8820 
8821 			SLIST_FOREACH(ri, &s->match_rules, entry) {
8822 				pf_counter_u64_add_protected(&ri->r->packets[dirndx], 1);
8823 				pf_counter_u64_add_protected(&ri->r->bytes[dirndx], pd->tot_len);
8824 			}
8825 		}
8826 		tr = r;
8827 		nr = (s != NULL) ? s->nat_rule.ptr : pd->nat_rule;
8828 		if (nr != NULL && r == &V_pf_default_rule)
8829 			tr = nr;
8830 		if (tr->src.addr.type == PF_ADDR_TABLE)
8831 			pfr_update_stats(tr->src.addr.p.tbl,
8832 			    (s == NULL) ? pd->src :
8833 			    &s->key[(s->direction == PF_IN)]->
8834 				addr[(s->direction == PF_OUT)],
8835 			    pd->af, pd->tot_len, dir == PF_OUT,
8836 			    r->action == PF_PASS, tr->src.neg);
8837 		if (tr->dst.addr.type == PF_ADDR_TABLE)
8838 			pfr_update_stats(tr->dst.addr.p.tbl,
8839 			    (s == NULL) ? pd->dst :
8840 			    &s->key[(s->direction == PF_IN)]->
8841 				addr[(s->direction == PF_IN)],
8842 			    pd->af, pd->tot_len, dir == PF_OUT,
8843 			    r->action == PF_PASS, tr->dst.neg);
8844 	}
8845 	pf_counter_u64_critical_exit();
8846 }
8847 
8848 #if defined(INET) || defined(INET6)
8849 int
8850 pf_test(sa_family_t af, int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
8851     struct inpcb *inp, struct pf_rule_actions *default_actions)
8852 {
8853 	struct pfi_kkif		*kif;
8854 	u_short			 action, reason = 0;
8855 	struct mbuf		*m = *m0;
8856 	struct ip		*h = NULL;
8857 	struct ip6_hdr		*h6 = NULL;
8858 	struct m_tag		*mtag;
8859 	struct pf_krule		*a = NULL, *r = &V_pf_default_rule;
8860 	struct pf_kstate	*s = NULL;
8861 	struct pf_kruleset	*ruleset = NULL;
8862 	struct pf_pdesc		 pd;
8863 	int			 off, hdrlen, use_2nd_queue = 0;
8864 	uint16_t		 tag;
8865 	uint8_t			 rt;
8866 	uint8_t			 ttl;
8867 
8868 	PF_RULES_RLOCK_TRACKER;
8869 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: bad direction %d\n", __func__, dir));
8870 	M_ASSERTPKTHDR(m);
8871 
8872 	if (!V_pf_status.running)
8873 		return (PF_PASS);
8874 
8875 	PF_RULES_RLOCK();
8876 
8877 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
8878 
8879 	if (__predict_false(kif == NULL)) {
8880 		DPFPRINTF(PF_DEBUG_URGENT,
8881 		    ("pf_test: kif == NULL, if_xname %s\n", ifp->if_xname));
8882 		PF_RULES_RUNLOCK();
8883 		return (PF_DROP);
8884 	}
8885 	if (kif->pfik_flags & PFI_IFLAG_SKIP) {
8886 		PF_RULES_RUNLOCK();
8887 		return (PF_PASS);
8888 	}
8889 
8890 	if (m->m_flags & M_SKIP_FIREWALL) {
8891 		PF_RULES_RUNLOCK();
8892 		return (PF_PASS);
8893 	}
8894 
8895 #ifdef INET6
8896 	/*
8897 	 * If we end up changing IP addresses (e.g. binat) the stack may get
8898 	 * confused and fail to send the icmp6 packet too big error. Just send
8899 	 * it here, before we do any NAT.
8900 	 */
8901 	if (af == AF_INET6 && dir == PF_OUT && pflags & PFIL_FWD &&
8902 	    IN6_LINKMTU(ifp) < pf_max_frag_size(m)) {
8903 		PF_RULES_RUNLOCK();
8904 		*m0 = NULL;
8905 		icmp6_error(m, ICMP6_PACKET_TOO_BIG, 0, IN6_LINKMTU(ifp));
8906 		return (PF_DROP);
8907 	}
8908 #endif
8909 
8910 	if (__predict_false(! M_WRITABLE(*m0))) {
8911 		m = *m0 = m_unshare(*m0, M_NOWAIT);
8912 		if (*m0 == NULL)
8913 			return (PF_DROP);
8914 	}
8915 
8916 	if (pf_setup_pdesc(af, dir, &pd, m0, &action, &reason, kif, &a, &r,
8917 		&s, &ruleset, &off, &hdrlen, inp, default_actions) == -1) {
8918 		if (action != PF_PASS)
8919 			pd.act.log |= PF_LOG_FORCE;
8920 		goto done;
8921 	}
8922 	m = *m0;
8923 
8924 	switch (af) {
8925 #ifdef INET
8926 	case AF_INET:
8927 		h = mtod(m, struct ip *);
8928 		ttl = h->ip_ttl;
8929 		break;
8930 #endif
8931 #ifdef INET6
8932 	case AF_INET6:
8933 		h6 = mtod(m, struct ip6_hdr *);
8934 		ttl = h6->ip6_hlim;
8935 		break;
8936 #endif
8937 	default:
8938 		panic("Unknown af %d", af);
8939 	}
8940 
8941 	if (pd.pf_mtag != NULL && (pd.pf_mtag->flags & PF_MTAG_FLAG_ROUTE_TO)) {
8942 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
8943 
8944 		ifp = ifnet_byindexgen(pd.pf_mtag->if_index,
8945 		    pd.pf_mtag->if_idxgen);
8946 		if (ifp == NULL || ifp->if_flags & IFF_DYING) {
8947 			PF_RULES_RUNLOCK();
8948 			m_freem(*m0);
8949 			*m0 = NULL;
8950 			return (PF_PASS);
8951 		}
8952 		PF_RULES_RUNLOCK();
8953 		(ifp->if_output)(ifp, m, sintosa(&pd.pf_mtag->dst), NULL);
8954 		*m0 = NULL;
8955 		return (PF_PASS);
8956 	}
8957 
8958 	if (ip_dn_io_ptr != NULL && pd.pf_mtag != NULL &&
8959 	    pd.pf_mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
8960 		/* Dummynet re-injects packets after they've
8961 		 * completed their delay. We've already
8962 		 * processed them, so pass unconditionally. */
8963 
8964 		/* But only once. We may see the packet multiple times (e.g.
8965 		 * PFIL_IN/PFIL_OUT). */
8966 		pf_dummynet_flag_remove(m, pd.pf_mtag);
8967 		PF_RULES_RUNLOCK();
8968 
8969 		return (PF_PASS);
8970 	}
8971 
8972 	if (__predict_false(ip_divert_ptr != NULL) &&
8973 	    ((mtag = m_tag_locate(m, MTAG_PF_DIVERT, 0, NULL)) != NULL)) {
8974 		struct pf_divert_mtag *dt = (struct pf_divert_mtag *)(mtag+1);
8975 		if ((dt->idir == PF_DIVERT_MTAG_DIR_IN && dir == PF_IN) ||
8976 		    (dt->idir == PF_DIVERT_MTAG_DIR_OUT && dir == PF_OUT)) {
8977 			if (pd.pf_mtag == NULL &&
8978 			    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
8979 				action = PF_DROP;
8980 				goto done;
8981 			}
8982 			pd.pf_mtag->flags |= PF_MTAG_FLAG_PACKET_LOOPED;
8983 		}
8984 		if (pd.pf_mtag && pd.pf_mtag->flags & PF_MTAG_FLAG_FASTFWD_OURS_PRESENT) {
8985 			m->m_flags |= M_FASTFWD_OURS;
8986 			pd.pf_mtag->flags &= ~PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
8987 		}
8988 		m_tag_delete(m, mtag);
8989 
8990 		mtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL);
8991 		if (mtag != NULL)
8992 			m_tag_delete(m, mtag);
8993 	}
8994 
8995 #ifdef INET6
8996 	/*
8997 	 * we do not support jumbogram.  if we keep going, zero ip6_plen
8998 	 * will do something bad, so drop the packet for now.
8999 	 */
9000 	if (af == AF_INET6 && htons(h6->ip6_plen) == 0) {
9001 		action = PF_DROP;
9002 		REASON_SET(&reason, PFRES_NORM);	/*XXX*/
9003 		goto done;
9004 	}
9005 #endif
9006 
9007 	switch (pd.proto) {
9008 	case IPPROTO_TCP: {
9009 		/* Respond to SYN with a syncookie. */
9010 		if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_SYN &&
9011 		    pd.dir == PF_IN && pf_synflood_check(&pd)) {
9012 			pf_syncookie_send(m, off, &pd);
9013 			action = PF_DROP;
9014 			break;
9015 		}
9016 
9017 		if ((pd.hdr.tcp.th_flags & TH_ACK) && pd.p_len == 0)
9018 			use_2nd_queue = 1;
9019 		action = pf_normalize_tcp(kif, m, 0, off, &pd);
9020 		if (action == PF_DROP)
9021 			goto done;
9022 		action = pf_test_state_tcp(&s, kif, m, off, &pd, &reason);
9023 		if (action == PF_PASS) {
9024 			if (V_pfsync_update_state_ptr != NULL)
9025 				V_pfsync_update_state_ptr(s);
9026 			r = s->rule.ptr;
9027 			a = s->anchor.ptr;
9028 		} else if (s == NULL) {
9029 			/* Validate remote SYN|ACK, re-create original SYN if
9030 			 * valid. */
9031 			if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) ==
9032 			    TH_ACK && pf_syncookie_validate(&pd) &&
9033 			    pd.dir == PF_IN) {
9034 				struct mbuf *msyn;
9035 
9036 				msyn = pf_syncookie_recreate_syn(ttl, off,
9037 				    &pd);
9038 				if (msyn == NULL) {
9039 					action = PF_DROP;
9040 					break;
9041 				}
9042 
9043 				action = pf_test(af, dir, pflags, ifp, &msyn, inp,
9044 				    &pd.act);
9045 				m_freem(msyn);
9046 				if (action != PF_PASS)
9047 					break;
9048 
9049 				action = pf_test_state_tcp(&s, kif, m, off,
9050 				    &pd, &reason);
9051 				if (action != PF_PASS || s == NULL) {
9052 					action = PF_DROP;
9053 					break;
9054 				}
9055 
9056 				s->src.seqhi = ntohl(pd.hdr.tcp.th_ack) - 1;
9057 				s->src.seqlo = ntohl(pd.hdr.tcp.th_seq) - 1;
9058 				pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_DST);
9059 				action = pf_synproxy(&pd, &s, &reason);
9060 				break;
9061 			} else {
9062 				action = pf_test_rule(&r, &s, kif, m, off, &pd,
9063 				    &a, &ruleset, inp, hdrlen);
9064 			}
9065 		}
9066 		break;
9067 	}
9068 
9069 	case IPPROTO_UDP: {
9070 		action = pf_test_state_udp(&s, kif, m, off, &pd);
9071 		if (action == PF_PASS) {
9072 			if (V_pfsync_update_state_ptr != NULL)
9073 				V_pfsync_update_state_ptr(s);
9074 			r = s->rule.ptr;
9075 			a = s->anchor.ptr;
9076 		} else if (s == NULL)
9077 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9078 			    &a, &ruleset, inp, hdrlen);
9079 		break;
9080 	}
9081 
9082 	case IPPROTO_SCTP: {
9083 		action = pf_normalize_sctp(dir, kif, m, 0, off, &pd);
9084 		if (action == PF_DROP)
9085 			goto done;
9086 		action = pf_test_state_sctp(&s, kif, m, off, &pd,
9087 		    &reason);
9088 		if (action == PF_PASS) {
9089 			if (V_pfsync_update_state_ptr != NULL)
9090 				V_pfsync_update_state_ptr(s);
9091 			r = s->rule.ptr;
9092 			a = s->anchor.ptr;
9093 		} else if (s == NULL) {
9094 			action = pf_test_rule(&r, &s, kif, m, off,
9095 			    &pd, &a, &ruleset, inp, hdrlen);
9096 		}
9097 		break;
9098 	}
9099 
9100 	case IPPROTO_ICMP: {
9101 		if (af != AF_INET) {
9102 			action = PF_DROP;
9103 			DPFPRINTF(PF_DEBUG_MISC,
9104 			    ("dropping IPv6 packet with ICMPv4 payload"));
9105 			goto done;
9106 		}
9107 		action = pf_test_state_icmp(&s, kif, m, off, &pd, &reason);
9108 		if (action == PF_PASS) {
9109 			if (V_pfsync_update_state_ptr != NULL)
9110 				V_pfsync_update_state_ptr(s);
9111 			r = s->rule.ptr;
9112 			a = s->anchor.ptr;
9113 		} else if (s == NULL)
9114 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9115 			    &a, &ruleset, inp, hdrlen);
9116 		break;
9117 	}
9118 
9119 	case IPPROTO_ICMPV6: {
9120 		if (af != AF_INET6) {
9121 			action = PF_DROP;
9122 			DPFPRINTF(PF_DEBUG_MISC,
9123 			    ("pf: dropping IPv4 packet with ICMPv6 payload\n"));
9124 			goto done;
9125 		}
9126 		action = pf_test_state_icmp(&s, kif, m, off, &pd, &reason);
9127 		if (action == PF_PASS) {
9128 			if (V_pfsync_update_state_ptr != NULL)
9129 				V_pfsync_update_state_ptr(s);
9130 			r = s->rule.ptr;
9131 			a = s->anchor.ptr;
9132 		} else if (s == NULL)
9133 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9134 			    &a, &ruleset, inp, hdrlen);
9135 		break;
9136 	}
9137 
9138 	default:
9139 		action = pf_test_state_other(&s, kif, m, &pd);
9140 		if (action == PF_PASS) {
9141 			if (V_pfsync_update_state_ptr != NULL)
9142 				V_pfsync_update_state_ptr(s);
9143 			r = s->rule.ptr;
9144 			a = s->anchor.ptr;
9145 		} else if (s == NULL)
9146 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9147 			    &a, &ruleset, inp, hdrlen);
9148 		break;
9149 	}
9150 
9151 done:
9152 	PF_RULES_RUNLOCK();
9153 
9154 	if (action == PF_PASS && pd.badopts &&
9155 	    !((s && s->state_flags & PFSTATE_ALLOWOPTS) || r->allow_opts)) {
9156 		action = PF_DROP;
9157 		REASON_SET(&reason, PFRES_IPOPTIONS);
9158 		pd.act.log = PF_LOG_FORCE;
9159 		DPFPRINTF(PF_DEBUG_MISC,
9160 		    ("pf: dropping packet with dangerous headers\n"));
9161 	}
9162 
9163 	if (s) {
9164 		uint8_t log = pd.act.log;
9165 		memcpy(&pd.act, &s->act, sizeof(struct pf_rule_actions));
9166 		pd.act.log |= log;
9167 		tag = s->tag;
9168 		rt = s->rt;
9169 	} else {
9170 		tag = r->tag;
9171 		rt = r->rt;
9172 	}
9173 
9174 	if (tag > 0 && pf_tag_packet(m, &pd, tag)) {
9175 		action = PF_DROP;
9176 		REASON_SET(&reason, PFRES_MEMORY);
9177 	}
9178 
9179 	pf_scrub(m, &pd);
9180 	if (pd.proto == IPPROTO_TCP && pd.act.max_mss)
9181 		pf_normalize_mss(m, off, &pd);
9182 
9183 	if (pd.act.rtableid >= 0)
9184 		M_SETFIB(m, pd.act.rtableid);
9185 
9186 	if (pd.act.flags & PFSTATE_SETPRIO) {
9187 		if (pd.tos & IPTOS_LOWDELAY)
9188 			use_2nd_queue = 1;
9189 		if (vlan_set_pcp(m, pd.act.set_prio[use_2nd_queue])) {
9190 			action = PF_DROP;
9191 			REASON_SET(&reason, PFRES_MEMORY);
9192 			pd.act.log = PF_LOG_FORCE;
9193 			DPFPRINTF(PF_DEBUG_MISC,
9194 			    ("pf: failed to allocate 802.1q mtag\n"));
9195 		}
9196 	}
9197 
9198 #ifdef ALTQ
9199 	if (action == PF_PASS && pd.act.qid) {
9200 		if (pd.pf_mtag == NULL &&
9201 		    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
9202 			action = PF_DROP;
9203 			REASON_SET(&reason, PFRES_MEMORY);
9204 		} else {
9205 			if (s != NULL)
9206 				pd.pf_mtag->qid_hash = pf_state_hash(s);
9207 			if (use_2nd_queue || (pd.tos & IPTOS_LOWDELAY))
9208 				pd.pf_mtag->qid = pd.act.pqid;
9209 			else
9210 				pd.pf_mtag->qid = pd.act.qid;
9211 			/* Add hints for ecn. */
9212 			if (af == AF_INET)
9213 				pd.pf_mtag->hdr = h;
9214 			else
9215 				pd.pf_mtag->hdr = h6;
9216 		}
9217 	}
9218 #endif /* ALTQ */
9219 
9220 	/*
9221 	 * connections redirected to loopback should not match sockets
9222 	 * bound specifically to loopback due to security implications,
9223 	 * see tcp_input() and in_pcblookup_listen().
9224 	 */
9225 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
9226 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL &&
9227 	    (s->nat_rule.ptr->action == PF_RDR ||
9228 	    s->nat_rule.ptr->action == PF_BINAT) &&
9229 	    pf_is_loopback(af, pd.dst))
9230 		m->m_flags |= M_SKIP_FIREWALL;
9231 
9232 	if (af == AF_INET && __predict_false(ip_divert_ptr != NULL) &&
9233 	    action == PF_PASS && r->divert.port && !PACKET_LOOPED(&pd)) {
9234 		mtag = m_tag_alloc(MTAG_PF_DIVERT, 0,
9235 		    sizeof(struct pf_divert_mtag), M_NOWAIT | M_ZERO);
9236 		if (mtag != NULL) {
9237 			((struct pf_divert_mtag *)(mtag+1))->port =
9238 			    ntohs(r->divert.port);
9239 			((struct pf_divert_mtag *)(mtag+1))->idir =
9240 			    (dir == PF_IN) ? PF_DIVERT_MTAG_DIR_IN :
9241 			    PF_DIVERT_MTAG_DIR_OUT;
9242 
9243 			if (s)
9244 				PF_STATE_UNLOCK(s);
9245 
9246 			m_tag_prepend(m, mtag);
9247 			if (m->m_flags & M_FASTFWD_OURS) {
9248 				if (pd.pf_mtag == NULL &&
9249 				    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
9250 					action = PF_DROP;
9251 					REASON_SET(&reason, PFRES_MEMORY);
9252 					pd.act.log = PF_LOG_FORCE;
9253 					DPFPRINTF(PF_DEBUG_MISC,
9254 					    ("pf: failed to allocate tag\n"));
9255 				} else {
9256 					pd.pf_mtag->flags |=
9257 					    PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
9258 					m->m_flags &= ~M_FASTFWD_OURS;
9259 				}
9260 			}
9261 			ip_divert_ptr(*m0, dir == PF_IN);
9262 			*m0 = NULL;
9263 
9264 			return (action);
9265 		} else {
9266 			/* XXX: ipfw has the same behaviour! */
9267 			action = PF_DROP;
9268 			REASON_SET(&reason, PFRES_MEMORY);
9269 			pd.act.log = PF_LOG_FORCE;
9270 			DPFPRINTF(PF_DEBUG_MISC,
9271 			    ("pf: failed to allocate divert tag\n"));
9272 		}
9273 	}
9274 	/* XXX: Anybody working on it?! */
9275 	if (af == AF_INET6 && r->divert.port)
9276 		printf("pf: divert(9) is not supported for IPv6\n");
9277 
9278 	/* this flag will need revising if the pkt is forwarded */
9279 	if (pd.pf_mtag)
9280 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_PACKET_LOOPED;
9281 
9282 	if (pd.act.log) {
9283 		struct pf_krule		*lr;
9284 		struct pf_krule_item	*ri;
9285 
9286 		if (s != NULL && s->nat_rule.ptr != NULL &&
9287 		    s->nat_rule.ptr->log & PF_LOG_ALL)
9288 			lr = s->nat_rule.ptr;
9289 		else
9290 			lr = r;
9291 
9292 		if (pd.act.log & PF_LOG_FORCE || lr->log & PF_LOG_ALL)
9293 			PFLOG_PACKET(kif, m, action, reason, lr, a,
9294 			    ruleset, &pd, (s == NULL));
9295 		if (s) {
9296 			SLIST_FOREACH(ri, &s->match_rules, entry)
9297 				if (ri->r->log & PF_LOG_ALL)
9298 					PFLOG_PACKET(kif, m, action,
9299 					    reason, ri->r, a, ruleset, &pd, 0);
9300 		}
9301 	}
9302 
9303 	pf_counters_inc(action, &pd, kif, s, r, a);
9304 
9305 	switch (action) {
9306 	case PF_SYNPROXY_DROP:
9307 		m_freem(*m0);
9308 	case PF_DEFER:
9309 		*m0 = NULL;
9310 		action = PF_PASS;
9311 		break;
9312 	case PF_DROP:
9313 		m_freem(*m0);
9314 		*m0 = NULL;
9315 		break;
9316 	default:
9317 		if (rt) {
9318 			switch (af) {
9319 #ifdef INET
9320 			case AF_INET:
9321 				/* pf_route() returns unlocked. */
9322 				pf_route(m0, r, kif->pfik_ifp, s, &pd, inp);
9323 				break;
9324 #endif
9325 #ifdef INET6
9326 			case AF_INET6:
9327 				/* pf_route6() returns unlocked. */
9328 				pf_route6(m0, r, kif->pfik_ifp, s, &pd, inp);
9329 				break;
9330 #endif
9331 			default:
9332 				panic("Unknown af %d", af);
9333 			}
9334 			goto out;
9335 		}
9336 		if (pf_dummynet(&pd, s, r, m0) != 0) {
9337 			action = PF_DROP;
9338 			REASON_SET(&reason, PFRES_MEMORY);
9339 		}
9340 		break;
9341 	}
9342 
9343 	SDT_PROBE4(pf, ip, test, done, action, reason, r, s);
9344 
9345 	if (s && action != PF_DROP) {
9346 		if (!s->if_index_in && dir == PF_IN)
9347 			s->if_index_in = ifp->if_index;
9348 		else if (!s->if_index_out && dir == PF_OUT)
9349 			s->if_index_out = ifp->if_index;
9350 	}
9351 
9352 	if (s)
9353 		PF_STATE_UNLOCK(s);
9354 
9355 #ifdef INET6
9356 	/* If reassembled packet passed, create new fragments. */
9357 	if (af == AF_INET6 && action == PF_PASS && *m0 && dir == PF_OUT &&
9358 	    (mtag = m_tag_find(m, PACKET_TAG_PF_REASSEMBLED, NULL)) != NULL)
9359 		action = pf_refragment6(ifp, m0, mtag, pflags & PFIL_FWD);
9360 #endif
9361 
9362 out:
9363 	pf_sctp_multihome_delayed(&pd, off, kif, s, action);
9364 
9365 	return (action);
9366 }
9367 #endif /* INET || INET6 */
9368