xref: /freebsd/sys/netpfil/pf/pf.c (revision e7a629c851d747772cc138efcb0418809ecdea55)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2001 Daniel Hartmeier
5  * Copyright (c) 2002 - 2008 Henning Brauer
6  * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  *    - Redistributions of source code must retain the above copyright
14  *      notice, this list of conditions and the following disclaimer.
15  *    - Redistributions in binary form must reproduce the above
16  *      copyright notice, this list of conditions and the following
17  *      disclaimer in the documentation and/or other materials provided
18  *      with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGE.
32  *
33  * Effort sponsored in part by the Defense Advanced Research Projects
34  * Agency (DARPA) and Air Force Research Laboratory, Air Force
35  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
36  *
37  *	$OpenBSD: pf.c,v 1.634 2009/02/27 12:37:45 henning Exp $
38  */
39 
40 #include <sys/cdefs.h>
41 #include "opt_bpf.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_pf.h"
45 #include "opt_sctp.h"
46 
47 #include <sys/param.h>
48 #include <sys/bus.h>
49 #include <sys/endian.h>
50 #include <sys/gsb_crc32.h>
51 #include <sys/hash.h>
52 #include <sys/interrupt.h>
53 #include <sys/kernel.h>
54 #include <sys/kthread.h>
55 #include <sys/limits.h>
56 #include <sys/mbuf.h>
57 #include <sys/md5.h>
58 #include <sys/random.h>
59 #include <sys/refcount.h>
60 #include <sys/sdt.h>
61 #include <sys/socket.h>
62 #include <sys/sysctl.h>
63 #include <sys/taskqueue.h>
64 #include <sys/ucred.h>
65 
66 #include <net/if.h>
67 #include <net/if_var.h>
68 #include <net/if_private.h>
69 #include <net/if_types.h>
70 #include <net/if_vlan_var.h>
71 #include <net/route.h>
72 #include <net/route/nhop.h>
73 #include <net/vnet.h>
74 
75 #include <net/pfil.h>
76 #include <net/pfvar.h>
77 #include <net/if_pflog.h>
78 #include <net/if_pfsync.h>
79 
80 #include <netinet/in_pcb.h>
81 #include <netinet/in_var.h>
82 #include <netinet/in_fib.h>
83 #include <netinet/ip.h>
84 #include <netinet/ip_fw.h>
85 #include <netinet/ip_icmp.h>
86 #include <netinet/icmp_var.h>
87 #include <netinet/ip_var.h>
88 #include <netinet/tcp.h>
89 #include <netinet/tcp_fsm.h>
90 #include <netinet/tcp_seq.h>
91 #include <netinet/tcp_timer.h>
92 #include <netinet/tcp_var.h>
93 #include <netinet/udp.h>
94 #include <netinet/udp_var.h>
95 
96 /* dummynet */
97 #include <netinet/ip_dummynet.h>
98 #include <netinet/ip_fw.h>
99 #include <netpfil/ipfw/dn_heap.h>
100 #include <netpfil/ipfw/ip_fw_private.h>
101 #include <netpfil/ipfw/ip_dn_private.h>
102 
103 #ifdef INET6
104 #include <netinet/ip6.h>
105 #include <netinet/icmp6.h>
106 #include <netinet6/nd6.h>
107 #include <netinet6/ip6_var.h>
108 #include <netinet6/in6_pcb.h>
109 #include <netinet6/in6_fib.h>
110 #include <netinet6/scope6_var.h>
111 #endif /* INET6 */
112 
113 #include <netinet/sctp_header.h>
114 #include <netinet/sctp_crc32.h>
115 
116 #include <machine/in_cksum.h>
117 #include <security/mac/mac_framework.h>
118 
119 #define	DPFPRINTF(n, x)	if (V_pf_status.debug >= (n)) printf x
120 
121 SDT_PROVIDER_DEFINE(pf);
122 SDT_PROBE_DEFINE4(pf, ip, test, done, "int", "int", "struct pf_krule *",
123     "struct pf_kstate *");
124 SDT_PROBE_DEFINE5(pf, ip, state, lookup, "struct pfi_kkif *",
125     "struct pf_state_key_cmp *", "int", "struct pf_pdesc *",
126     "struct pf_kstate *");
127 SDT_PROBE_DEFINE2(pf, ip, , bound_iface, "struct pf_kstate *",
128     "struct pfi_kkif *");
129 SDT_PROBE_DEFINE4(pf, sctp, multihome, test, "struct pfi_kkif *",
130     "struct pf_krule *", "struct mbuf *", "int");
131 SDT_PROBE_DEFINE2(pf, sctp, multihome, add, "uint32_t",
132     "struct pf_sctp_source *");
133 SDT_PROBE_DEFINE3(pf, sctp, multihome, remove, "uint32_t",
134     "struct pf_kstate *", "struct pf_sctp_source *");
135 
136 SDT_PROBE_DEFINE3(pf, eth, test_rule, entry, "int", "struct ifnet *",
137     "struct mbuf *");
138 SDT_PROBE_DEFINE2(pf, eth, test_rule, test, "int", "struct pf_keth_rule *");
139 SDT_PROBE_DEFINE3(pf, eth, test_rule, mismatch,
140     "int", "struct pf_keth_rule *", "char *");
141 SDT_PROBE_DEFINE2(pf, eth, test_rule, match, "int", "struct pf_keth_rule *");
142 SDT_PROBE_DEFINE2(pf, eth, test_rule, final_match,
143     "int", "struct pf_keth_rule *");
144 SDT_PROBE_DEFINE2(pf, purge, state, rowcount, "int", "size_t");
145 
146 /*
147  * Global variables
148  */
149 
150 /* state tables */
151 VNET_DEFINE(struct pf_altqqueue,	 pf_altqs[4]);
152 VNET_DEFINE(struct pf_kpalist,		 pf_pabuf);
153 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_active);
154 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_active);
155 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_inactive);
156 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_inactive);
157 VNET_DEFINE(struct pf_kstatus,		 pf_status);
158 
159 VNET_DEFINE(u_int32_t,			 ticket_altqs_active);
160 VNET_DEFINE(u_int32_t,			 ticket_altqs_inactive);
161 VNET_DEFINE(int,			 altqs_inactive_open);
162 VNET_DEFINE(u_int32_t,			 ticket_pabuf);
163 
164 VNET_DEFINE(MD5_CTX,			 pf_tcp_secret_ctx);
165 #define	V_pf_tcp_secret_ctx		 VNET(pf_tcp_secret_ctx)
166 VNET_DEFINE(u_char,			 pf_tcp_secret[16]);
167 #define	V_pf_tcp_secret			 VNET(pf_tcp_secret)
168 VNET_DEFINE(int,			 pf_tcp_secret_init);
169 #define	V_pf_tcp_secret_init		 VNET(pf_tcp_secret_init)
170 VNET_DEFINE(int,			 pf_tcp_iss_off);
171 #define	V_pf_tcp_iss_off		 VNET(pf_tcp_iss_off)
172 VNET_DECLARE(int,			 pf_vnet_active);
173 #define	V_pf_vnet_active		 VNET(pf_vnet_active)
174 
175 VNET_DEFINE_STATIC(uint32_t, pf_purge_idx);
176 #define V_pf_purge_idx	VNET(pf_purge_idx)
177 
178 #ifdef PF_WANT_32_TO_64_COUNTER
179 VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter);
180 #define	V_pf_counter_periodic_iter	VNET(pf_counter_periodic_iter)
181 
182 VNET_DEFINE(struct allrulelist_head, pf_allrulelist);
183 VNET_DEFINE(size_t, pf_allrulecount);
184 VNET_DEFINE(struct pf_krule *, pf_rulemarker);
185 #endif
186 
187 struct pf_sctp_endpoint;
188 RB_HEAD(pf_sctp_endpoints, pf_sctp_endpoint);
189 struct pf_sctp_source {
190 	sa_family_t			af;
191 	struct pf_addr			addr;
192 	TAILQ_ENTRY(pf_sctp_source)	entry;
193 };
194 TAILQ_HEAD(pf_sctp_sources, pf_sctp_source);
195 struct pf_sctp_endpoint
196 {
197 	uint32_t		 v_tag;
198 	struct pf_sctp_sources	 sources;
199 	RB_ENTRY(pf_sctp_endpoint)	entry;
200 };
201 static int
202 pf_sctp_endpoint_compare(struct pf_sctp_endpoint *a, struct pf_sctp_endpoint *b)
203 {
204 	return (a->v_tag - b->v_tag);
205 }
206 RB_PROTOTYPE(pf_sctp_endpoints, pf_sctp_endpoint, entry, pf_sctp_endpoint_compare);
207 RB_GENERATE(pf_sctp_endpoints, pf_sctp_endpoint, entry, pf_sctp_endpoint_compare);
208 VNET_DEFINE_STATIC(struct pf_sctp_endpoints, pf_sctp_endpoints);
209 #define V_pf_sctp_endpoints	VNET(pf_sctp_endpoints)
210 static struct mtx_padalign pf_sctp_endpoints_mtx;
211 MTX_SYSINIT(pf_sctp_endpoints_mtx, &pf_sctp_endpoints_mtx, "SCTP endpoints", MTX_DEF);
212 #define	PF_SCTP_ENDPOINTS_LOCK()	mtx_lock(&pf_sctp_endpoints_mtx)
213 #define	PF_SCTP_ENDPOINTS_UNLOCK()	mtx_unlock(&pf_sctp_endpoints_mtx)
214 
215 /*
216  * Queue for pf_intr() sends.
217  */
218 static MALLOC_DEFINE(M_PFTEMP, "pf_temp", "pf(4) temporary allocations");
219 struct pf_send_entry {
220 	STAILQ_ENTRY(pf_send_entry)	pfse_next;
221 	struct mbuf			*pfse_m;
222 	enum {
223 		PFSE_IP,
224 		PFSE_IP6,
225 		PFSE_ICMP,
226 		PFSE_ICMP6,
227 	}				pfse_type;
228 	struct {
229 		int		type;
230 		int		code;
231 		int		mtu;
232 	} icmpopts;
233 };
234 
235 STAILQ_HEAD(pf_send_head, pf_send_entry);
236 VNET_DEFINE_STATIC(struct pf_send_head, pf_sendqueue);
237 #define	V_pf_sendqueue	VNET(pf_sendqueue)
238 
239 static struct mtx_padalign pf_sendqueue_mtx;
240 MTX_SYSINIT(pf_sendqueue_mtx, &pf_sendqueue_mtx, "pf send queue", MTX_DEF);
241 #define	PF_SENDQ_LOCK()		mtx_lock(&pf_sendqueue_mtx)
242 #define	PF_SENDQ_UNLOCK()	mtx_unlock(&pf_sendqueue_mtx)
243 
244 /*
245  * Queue for pf_overload_task() tasks.
246  */
247 struct pf_overload_entry {
248 	SLIST_ENTRY(pf_overload_entry)	next;
249 	struct pf_addr  		addr;
250 	sa_family_t			af;
251 	uint8_t				dir;
252 	struct pf_krule  		*rule;
253 };
254 
255 SLIST_HEAD(pf_overload_head, pf_overload_entry);
256 VNET_DEFINE_STATIC(struct pf_overload_head, pf_overloadqueue);
257 #define V_pf_overloadqueue	VNET(pf_overloadqueue)
258 VNET_DEFINE_STATIC(struct task, pf_overloadtask);
259 #define	V_pf_overloadtask	VNET(pf_overloadtask)
260 
261 static struct mtx_padalign pf_overloadqueue_mtx;
262 MTX_SYSINIT(pf_overloadqueue_mtx, &pf_overloadqueue_mtx,
263     "pf overload/flush queue", MTX_DEF);
264 #define	PF_OVERLOADQ_LOCK()	mtx_lock(&pf_overloadqueue_mtx)
265 #define	PF_OVERLOADQ_UNLOCK()	mtx_unlock(&pf_overloadqueue_mtx)
266 
267 VNET_DEFINE(struct pf_krulequeue, pf_unlinked_rules);
268 struct mtx_padalign pf_unlnkdrules_mtx;
269 MTX_SYSINIT(pf_unlnkdrules_mtx, &pf_unlnkdrules_mtx, "pf unlinked rules",
270     MTX_DEF);
271 
272 struct sx pf_config_lock;
273 SX_SYSINIT(pf_config_lock, &pf_config_lock, "pf config");
274 
275 struct mtx_padalign pf_table_stats_lock;
276 MTX_SYSINIT(pf_table_stats_lock, &pf_table_stats_lock, "pf table stats",
277     MTX_DEF);
278 
279 VNET_DEFINE_STATIC(uma_zone_t,	pf_sources_z);
280 #define	V_pf_sources_z	VNET(pf_sources_z)
281 uma_zone_t		pf_mtag_z;
282 VNET_DEFINE(uma_zone_t,	 pf_state_z);
283 VNET_DEFINE(uma_zone_t,	 pf_state_key_z);
284 VNET_DEFINE(uma_zone_t,	 pf_udp_mapping_z);
285 
286 VNET_DEFINE(struct unrhdr64, pf_stateid);
287 
288 static void		 pf_src_tree_remove_state(struct pf_kstate *);
289 static void		 pf_init_threshold(struct pf_threshold *, u_int32_t,
290 			    u_int32_t);
291 static void		 pf_add_threshold(struct pf_threshold *);
292 static int		 pf_check_threshold(struct pf_threshold *);
293 
294 static void		 pf_change_ap(struct mbuf *, struct pf_addr *, u_int16_t *,
295 			    u_int16_t *, u_int16_t *, struct pf_addr *,
296 			    u_int16_t, u_int8_t, sa_family_t);
297 static int		 pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *,
298 			    struct tcphdr *, struct pf_state_peer *);
299 int			 pf_icmp_mapping(struct pf_pdesc *, u_int8_t, int *,
300 			    int *, u_int16_t *, u_int16_t *);
301 static void		 pf_change_icmp(struct pf_addr *, u_int16_t *,
302 			    struct pf_addr *, struct pf_addr *, u_int16_t,
303 			    u_int16_t *, u_int16_t *, u_int16_t *,
304 			    u_int16_t *, u_int8_t, sa_family_t);
305 static void		 pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t,
306 			    sa_family_t, struct pf_krule *, int);
307 static void		 pf_detach_state(struct pf_kstate *);
308 static int		 pf_state_key_attach(struct pf_state_key *,
309 			    struct pf_state_key *, struct pf_kstate *);
310 static void		 pf_state_key_detach(struct pf_kstate *, int);
311 static int		 pf_state_key_ctor(void *, int, void *, int);
312 static u_int32_t	 pf_tcp_iss(struct pf_pdesc *);
313 static __inline void	 pf_dummynet_flag_remove(struct mbuf *m,
314 			    struct pf_mtag *pf_mtag);
315 static int		 pf_dummynet(struct pf_pdesc *, struct pf_kstate *,
316 			    struct pf_krule *, struct mbuf **);
317 static int		 pf_dummynet_route(struct pf_pdesc *,
318 			    struct pf_kstate *, struct pf_krule *,
319 			    struct ifnet *, struct sockaddr *, struct mbuf **);
320 static int		 pf_test_eth_rule(int, struct pfi_kkif *,
321 			    struct mbuf **);
322 static int		 pf_test_rule(struct pf_krule **, struct pf_kstate **,
323 			    struct pfi_kkif *, struct mbuf *, int,
324 			    struct pf_pdesc *, struct pf_krule **,
325 			    struct pf_kruleset **, struct inpcb *, int);
326 static int		 pf_create_state(struct pf_krule *, struct pf_krule *,
327 			    struct pf_krule *, struct pf_pdesc *,
328 			    struct pf_ksrc_node *, struct pf_state_key *,
329 			    struct pf_state_key *, struct mbuf *, int,
330 			    u_int16_t, u_int16_t, int *, struct pfi_kkif *,
331 			    struct pf_kstate **, int, u_int16_t, u_int16_t,
332 			    int, struct pf_krule_slist *, struct pf_udp_mapping *);
333 static int		 pf_state_key_addr_setup(struct pf_pdesc *, struct mbuf *,
334 			    int, struct pf_state_key_cmp *, int, struct pf_addr *,
335 			    int, struct pf_addr *, int);
336 static int		 pf_test_fragment(struct pf_krule **, struct pfi_kkif *,
337 			    struct mbuf *, struct pf_pdesc *,
338 			    struct pf_krule **, struct pf_kruleset **);
339 static int		 pf_tcp_track_full(struct pf_kstate **,
340 			    struct pfi_kkif *, struct mbuf *, int,
341 			    struct pf_pdesc *, u_short *, int *);
342 static int		 pf_tcp_track_sloppy(struct pf_kstate **,
343 			    struct pf_pdesc *, u_short *);
344 static int		 pf_test_state_tcp(struct pf_kstate **,
345 			    struct pfi_kkif *, struct mbuf *, int,
346 			    struct pf_pdesc *, u_short *);
347 static int		 pf_test_state_udp(struct pf_kstate **,
348 			    struct pfi_kkif *, struct mbuf *, int,
349 			    struct pf_pdesc *);
350 int			 pf_icmp_state_lookup(struct pf_state_key_cmp *,
351 			    struct pf_pdesc *, struct pf_kstate **, struct mbuf *,
352 			    int, int, struct pfi_kkif *, u_int16_t, u_int16_t,
353 			    int, int *, int, int);
354 static int		 pf_test_state_icmp(struct pf_kstate **,
355 			    struct pfi_kkif *, struct mbuf *, int,
356 			    struct pf_pdesc *, u_short *);
357 static void		 pf_sctp_multihome_detach_addr(const struct pf_kstate *);
358 static void		 pf_sctp_multihome_delayed(struct pf_pdesc *, int,
359 			    struct pfi_kkif *, struct pf_kstate *, int);
360 static int		 pf_test_state_sctp(struct pf_kstate **,
361 			    struct pfi_kkif *, struct mbuf *, int,
362 			    struct pf_pdesc *, u_short *);
363 static int		 pf_test_state_other(struct pf_kstate **,
364 			    struct pfi_kkif *, struct mbuf *, struct pf_pdesc *);
365 static u_int16_t	 pf_calc_mss(struct pf_addr *, sa_family_t,
366 				int, u_int16_t);
367 static int		 pf_check_proto_cksum(struct mbuf *, int, int,
368 			    u_int8_t, sa_family_t);
369 static void		 pf_print_state_parts(struct pf_kstate *,
370 			    struct pf_state_key *, struct pf_state_key *);
371 static void		 pf_patch_8(struct mbuf *, u_int16_t *, u_int8_t *, u_int8_t,
372 			    bool, u_int8_t);
373 static struct pf_kstate	*pf_find_state(struct pfi_kkif *,
374 			    const struct pf_state_key_cmp *, u_int);
375 static int		 pf_src_connlimit(struct pf_kstate **);
376 static int		 pf_match_rcvif(struct mbuf *, struct pf_krule *);
377 static void		 pf_counters_inc(int,
378 			    struct pf_pdesc *, struct pfi_kkif *,
379 			    struct pf_kstate *, struct pf_krule *,
380 			    struct pf_krule *);
381 static void		 pf_overload_task(void *v, int pending);
382 static u_short		 pf_insert_src_node(struct pf_ksrc_node **,
383 			    struct pf_krule *, struct pf_addr *, sa_family_t);
384 static u_int		 pf_purge_expired_states(u_int, int);
385 static void		 pf_purge_unlinked_rules(void);
386 static int		 pf_mtag_uminit(void *, int, int);
387 static void		 pf_mtag_free(struct m_tag *);
388 static void		 pf_packet_rework_nat(struct mbuf *, struct pf_pdesc *,
389 			    int, struct pf_state_key *);
390 #ifdef INET
391 static void		 pf_route(struct mbuf **, struct pf_krule *,
392 			    struct ifnet *, struct pf_kstate *,
393 			    struct pf_pdesc *, struct inpcb *);
394 #endif /* INET */
395 #ifdef INET6
396 static void		 pf_change_a6(struct pf_addr *, u_int16_t *,
397 			    struct pf_addr *, u_int8_t);
398 static void		 pf_route6(struct mbuf **, struct pf_krule *,
399 			    struct ifnet *, struct pf_kstate *,
400 			    struct pf_pdesc *, struct inpcb *);
401 #endif /* INET6 */
402 static __inline void pf_set_protostate(struct pf_kstate *, int, u_int8_t);
403 
404 int in4_cksum(struct mbuf *m, u_int8_t nxt, int off, int len);
405 
406 extern int pf_end_threads;
407 extern struct proc *pf_purge_proc;
408 
409 VNET_DEFINE(struct pf_limit, pf_limits[PF_LIMIT_MAX]);
410 
411 enum { PF_ICMP_MULTI_NONE, PF_ICMP_MULTI_LINK };
412 
413 #define	PACKET_UNDO_NAT(_m, _pd, _off, _s)		\
414 	do {								\
415 		struct pf_state_key *nk;				\
416 		if ((pd->dir) == PF_OUT)					\
417 			nk = (_s)->key[PF_SK_STACK];			\
418 		else							\
419 			nk = (_s)->key[PF_SK_WIRE];			\
420 		pf_packet_rework_nat(_m, _pd, _off, nk);		\
421 	} while (0)
422 
423 #define	PACKET_LOOPED(pd)	((pd)->pf_mtag &&			\
424 				 (pd)->pf_mtag->flags & PF_MTAG_FLAG_PACKET_LOOPED)
425 
426 #define	STATE_LOOKUP(i, k, s, pd)					\
427 	do {								\
428 		(s) = pf_find_state((i), (k), (pd->dir));			\
429 		SDT_PROBE5(pf, ip, state, lookup, i, k, (pd->dir), pd, (s));	\
430 		if ((s) == NULL)					\
431 			return (PF_DROP);				\
432 		if (PACKET_LOOPED(pd))					\
433 			return (PF_PASS);				\
434 	} while (0)
435 
436 static struct pfi_kkif *
437 BOUND_IFACE(struct pf_kstate *st, struct pfi_kkif *k)
438 {
439 	SDT_PROBE2(pf, ip, , bound_iface, st, k);
440 
441 	/* Floating unless otherwise specified. */
442 	if (! (st->rule.ptr->rule_flag & PFRULE_IFBOUND))
443 		return (V_pfi_all);
444 
445 	/*
446 	 * Initially set to all, because we don't know what interface we'll be
447 	 * sending this out when we create the state.
448 	 */
449 	if (st->rule.ptr->rt == PF_REPLYTO)
450 		return (V_pfi_all);
451 
452 	/* Don't overrule the interface for states created on incoming packets. */
453 	if (st->direction == PF_IN)
454 		return (k);
455 
456 	/* No route-to, so don't overrule. */
457 	if (st->rt != PF_ROUTETO)
458 		return (k);
459 
460 	/* Bind to the route-to interface. */
461 	return (st->rt_kif);
462 }
463 
464 #define	STATE_INC_COUNTERS(s)						\
465 	do {								\
466 		struct pf_krule_item *mrm;				\
467 		counter_u64_add(s->rule.ptr->states_cur, 1);		\
468 		counter_u64_add(s->rule.ptr->states_tot, 1);		\
469 		if (s->anchor.ptr != NULL) {				\
470 			counter_u64_add(s->anchor.ptr->states_cur, 1);	\
471 			counter_u64_add(s->anchor.ptr->states_tot, 1);	\
472 		}							\
473 		if (s->nat_rule.ptr != NULL) {				\
474 			counter_u64_add(s->nat_rule.ptr->states_cur, 1);\
475 			counter_u64_add(s->nat_rule.ptr->states_tot, 1);\
476 		}							\
477 		SLIST_FOREACH(mrm, &s->match_rules, entry) {		\
478 			counter_u64_add(mrm->r->states_cur, 1);		\
479 			counter_u64_add(mrm->r->states_tot, 1);		\
480 		}							\
481 	} while (0)
482 
483 #define	STATE_DEC_COUNTERS(s)						\
484 	do {								\
485 		struct pf_krule_item *mrm;				\
486 		if (s->nat_rule.ptr != NULL)				\
487 			counter_u64_add(s->nat_rule.ptr->states_cur, -1);\
488 		if (s->anchor.ptr != NULL)				\
489 			counter_u64_add(s->anchor.ptr->states_cur, -1);	\
490 		counter_u64_add(s->rule.ptr->states_cur, -1);		\
491 		SLIST_FOREACH(mrm, &s->match_rules, entry)		\
492 			counter_u64_add(mrm->r->states_cur, -1);	\
493 	} while (0)
494 
495 MALLOC_DEFINE(M_PFHASH, "pf_hash", "pf(4) hash header structures");
496 MALLOC_DEFINE(M_PF_RULE_ITEM, "pf_krule_item", "pf(4) rule items");
497 VNET_DEFINE(struct pf_keyhash *, pf_keyhash);
498 VNET_DEFINE(struct pf_idhash *, pf_idhash);
499 VNET_DEFINE(struct pf_srchash *, pf_srchash);
500 VNET_DEFINE(struct pf_udpendpointhash *, pf_udpendpointhash);
501 VNET_DEFINE(struct pf_udpendpointmapping *, pf_udpendpointmapping);
502 
503 SYSCTL_NODE(_net, OID_AUTO, pf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
504     "pf(4)");
505 
506 VNET_DEFINE(u_long, pf_hashmask);
507 VNET_DEFINE(u_long, pf_srchashmask);
508 VNET_DEFINE(u_long, pf_udpendpointhashmask);
509 VNET_DEFINE_STATIC(u_long, pf_hashsize);
510 #define V_pf_hashsize	VNET(pf_hashsize)
511 VNET_DEFINE_STATIC(u_long, pf_srchashsize);
512 #define V_pf_srchashsize	VNET(pf_srchashsize)
513 VNET_DEFINE_STATIC(u_long, pf_udpendpointhashsize);
514 #define V_pf_udpendpointhashsize	VNET(pf_udpendpointhashsize)
515 u_long	pf_ioctl_maxcount = 65535;
516 
517 SYSCTL_ULONG(_net_pf, OID_AUTO, states_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
518     &VNET_NAME(pf_hashsize), 0, "Size of pf(4) states hashtable");
519 SYSCTL_ULONG(_net_pf, OID_AUTO, source_nodes_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
520     &VNET_NAME(pf_srchashsize), 0, "Size of pf(4) source nodes hashtable");
521 SYSCTL_ULONG(_net_pf, OID_AUTO, udpendpoint_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
522     &VNET_NAME(pf_udpendpointhashsize), 0, "Size of pf(4) endpoint hashtable");
523 SYSCTL_ULONG(_net_pf, OID_AUTO, request_maxcount, CTLFLAG_RWTUN,
524     &pf_ioctl_maxcount, 0, "Maximum number of tables, addresses, ... in a single ioctl() call");
525 
526 VNET_DEFINE(void *, pf_swi_cookie);
527 VNET_DEFINE(struct intr_event *, pf_swi_ie);
528 
529 VNET_DEFINE(uint32_t, pf_hashseed);
530 #define	V_pf_hashseed	VNET(pf_hashseed)
531 
532 static void
533 pf_sctp_checksum(struct mbuf *m, int off)
534 {
535 	uint32_t sum = 0;
536 
537 	/* Zero out the checksum, to enable recalculation. */
538 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
539 	    sizeof(sum), (caddr_t)&sum);
540 
541 	sum = sctp_calculate_cksum(m, off);
542 
543 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
544 	    sizeof(sum), (caddr_t)&sum);
545 }
546 
547 int
548 pf_addr_cmp(struct pf_addr *a, struct pf_addr *b, sa_family_t af)
549 {
550 
551 	switch (af) {
552 #ifdef INET
553 	case AF_INET:
554 		if (a->addr32[0] > b->addr32[0])
555 			return (1);
556 		if (a->addr32[0] < b->addr32[0])
557 			return (-1);
558 		break;
559 #endif /* INET */
560 #ifdef INET6
561 	case AF_INET6:
562 		if (a->addr32[3] > b->addr32[3])
563 			return (1);
564 		if (a->addr32[3] < b->addr32[3])
565 			return (-1);
566 		if (a->addr32[2] > b->addr32[2])
567 			return (1);
568 		if (a->addr32[2] < b->addr32[2])
569 			return (-1);
570 		if (a->addr32[1] > b->addr32[1])
571 			return (1);
572 		if (a->addr32[1] < b->addr32[1])
573 			return (-1);
574 		if (a->addr32[0] > b->addr32[0])
575 			return (1);
576 		if (a->addr32[0] < b->addr32[0])
577 			return (-1);
578 		break;
579 #endif /* INET6 */
580 	default:
581 		panic("%s: unknown address family %u", __func__, af);
582 	}
583 	return (0);
584 }
585 
586 static bool
587 pf_is_loopback(sa_family_t af, struct pf_addr *addr)
588 {
589 	switch (af) {
590 	case AF_INET:
591 		return IN_LOOPBACK(ntohl(addr->v4.s_addr));
592 	case AF_INET6:
593 		return IN6_IS_ADDR_LOOPBACK(&addr->v6);
594 	default:
595 		panic("Unknown af %d", af);
596 	}
597 }
598 
599 static void
600 pf_packet_rework_nat(struct mbuf *m, struct pf_pdesc *pd, int off,
601 	struct pf_state_key *nk)
602 {
603 
604 	switch (pd->proto) {
605 	case IPPROTO_TCP: {
606 		struct tcphdr *th = &pd->hdr.tcp;
607 
608 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
609 			pf_change_ap(m, pd->src, &th->th_sport, pd->ip_sum,
610 			    &th->th_sum, &nk->addr[pd->sidx],
611 			    nk->port[pd->sidx], 0, pd->af);
612 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
613 			pf_change_ap(m, pd->dst, &th->th_dport, pd->ip_sum,
614 			    &th->th_sum, &nk->addr[pd->didx],
615 			    nk->port[pd->didx], 0, pd->af);
616 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
617 		break;
618 	}
619 	case IPPROTO_UDP: {
620 		struct udphdr *uh = &pd->hdr.udp;
621 
622 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
623 			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
624 			    &uh->uh_sum, &nk->addr[pd->sidx],
625 			    nk->port[pd->sidx], 1, pd->af);
626 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
627 			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
628 			    &uh->uh_sum, &nk->addr[pd->didx],
629 			    nk->port[pd->didx], 1, pd->af);
630 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
631 		break;
632 	}
633 	case IPPROTO_SCTP: {
634 		struct sctphdr *sh = &pd->hdr.sctp;
635 		uint16_t checksum = 0;
636 
637 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
638 			pf_change_ap(m, pd->src, &sh->src_port, pd->ip_sum,
639 			    &checksum, &nk->addr[pd->sidx],
640 			    nk->port[pd->sidx], 1, pd->af);
641 		}
642 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
643 			pf_change_ap(m, pd->dst, &sh->dest_port, pd->ip_sum,
644 			    &checksum, &nk->addr[pd->didx],
645 			    nk->port[pd->didx], 1, pd->af);
646 		}
647 
648 		break;
649 	}
650 	case IPPROTO_ICMP: {
651 		struct icmp *ih = &pd->hdr.icmp;
652 
653 		if (nk->port[pd->sidx] != ih->icmp_id) {
654 			pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
655 			    ih->icmp_cksum, ih->icmp_id,
656 			    nk->port[pd->sidx], 0);
657 			ih->icmp_id = nk->port[pd->sidx];
658 			pd->sport = &ih->icmp_id;
659 
660 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)ih);
661 		}
662 		/* FALLTHROUGH */
663 	}
664 	default:
665 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
666 			switch (pd->af) {
667 			case AF_INET:
668 				pf_change_a(&pd->src->v4.s_addr,
669 				    pd->ip_sum, nk->addr[pd->sidx].v4.s_addr,
670 				    0);
671 				break;
672 			case AF_INET6:
673 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
674 				break;
675 			}
676 		}
677 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
678 			switch (pd->af) {
679 			case AF_INET:
680 				pf_change_a(&pd->dst->v4.s_addr,
681 				    pd->ip_sum, nk->addr[pd->didx].v4.s_addr,
682 				    0);
683 				break;
684 			case AF_INET6:
685 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
686 				break;
687 			}
688 		}
689 		break;
690 	}
691 }
692 
693 static __inline uint32_t
694 pf_hashkey(const struct pf_state_key *sk)
695 {
696 	uint32_t h;
697 
698 	h = murmur3_32_hash32((const uint32_t *)sk,
699 	    sizeof(struct pf_state_key_cmp)/sizeof(uint32_t),
700 	    V_pf_hashseed);
701 
702 	return (h & V_pf_hashmask);
703 }
704 
705 static __inline uint32_t
706 pf_hashsrc(struct pf_addr *addr, sa_family_t af)
707 {
708 	uint32_t h;
709 
710 	switch (af) {
711 	case AF_INET:
712 		h = murmur3_32_hash32((uint32_t *)&addr->v4,
713 		    sizeof(addr->v4)/sizeof(uint32_t), V_pf_hashseed);
714 		break;
715 	case AF_INET6:
716 		h = murmur3_32_hash32((uint32_t *)&addr->v6,
717 		    sizeof(addr->v6)/sizeof(uint32_t), V_pf_hashseed);
718 		break;
719 	default:
720 		panic("%s: unknown address family %u", __func__, af);
721 	}
722 
723 	return (h & V_pf_srchashmask);
724 }
725 
726 static inline uint32_t
727 pf_hashudpendpoint(struct pf_udp_endpoint *endpoint)
728 {
729 	uint32_t h;
730 
731 	h = murmur3_32_hash32((uint32_t *)endpoint,
732 	    sizeof(struct pf_udp_endpoint_cmp)/sizeof(uint32_t),
733 	    V_pf_hashseed);
734 	return (h & V_pf_udpendpointhashmask);
735 }
736 
737 #ifdef ALTQ
738 static int
739 pf_state_hash(struct pf_kstate *s)
740 {
741 	u_int32_t hv = (intptr_t)s / sizeof(*s);
742 
743 	hv ^= crc32(&s->src, sizeof(s->src));
744 	hv ^= crc32(&s->dst, sizeof(s->dst));
745 	if (hv == 0)
746 		hv = 1;
747 	return (hv);
748 }
749 #endif
750 
751 static __inline void
752 pf_set_protostate(struct pf_kstate *s, int which, u_int8_t newstate)
753 {
754 	if (which == PF_PEER_DST || which == PF_PEER_BOTH)
755 		s->dst.state = newstate;
756 	if (which == PF_PEER_DST)
757 		return;
758 	if (s->src.state == newstate)
759 		return;
760 	if (s->creatorid == V_pf_status.hostid &&
761 	    s->key[PF_SK_STACK] != NULL &&
762 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP &&
763 	    !(TCPS_HAVEESTABLISHED(s->src.state) ||
764 	    s->src.state == TCPS_CLOSED) &&
765 	    (TCPS_HAVEESTABLISHED(newstate) || newstate == TCPS_CLOSED))
766 		atomic_add_32(&V_pf_status.states_halfopen, -1);
767 
768 	s->src.state = newstate;
769 }
770 
771 #ifdef INET6
772 void
773 pf_addrcpy(struct pf_addr *dst, struct pf_addr *src, sa_family_t af)
774 {
775 	switch (af) {
776 #ifdef INET
777 	case AF_INET:
778 		memcpy(&dst->v4, &src->v4, sizeof(dst->v4));
779 		break;
780 #endif /* INET */
781 	case AF_INET6:
782 		memcpy(&dst->v6, &src->v6, sizeof(dst->v6));
783 		break;
784 	}
785 }
786 #endif /* INET6 */
787 
788 static void
789 pf_init_threshold(struct pf_threshold *threshold,
790     u_int32_t limit, u_int32_t seconds)
791 {
792 	threshold->limit = limit * PF_THRESHOLD_MULT;
793 	threshold->seconds = seconds;
794 	threshold->count = 0;
795 	threshold->last = time_uptime;
796 }
797 
798 static void
799 pf_add_threshold(struct pf_threshold *threshold)
800 {
801 	u_int32_t t = time_uptime, diff = t - threshold->last;
802 
803 	if (diff >= threshold->seconds)
804 		threshold->count = 0;
805 	else
806 		threshold->count -= threshold->count * diff /
807 		    threshold->seconds;
808 	threshold->count += PF_THRESHOLD_MULT;
809 	threshold->last = t;
810 }
811 
812 static int
813 pf_check_threshold(struct pf_threshold *threshold)
814 {
815 	return (threshold->count > threshold->limit);
816 }
817 
818 static int
819 pf_src_connlimit(struct pf_kstate **state)
820 {
821 	struct pf_overload_entry *pfoe;
822 	int bad = 0;
823 
824 	PF_STATE_LOCK_ASSERT(*state);
825 	/*
826 	 * XXXKS: The src node is accessed unlocked!
827 	 * PF_SRC_NODE_LOCK_ASSERT((*state)->src_node);
828 	 */
829 
830 	(*state)->src_node->conn++;
831 	(*state)->src.tcp_est = 1;
832 	pf_add_threshold(&(*state)->src_node->conn_rate);
833 
834 	if ((*state)->rule.ptr->max_src_conn &&
835 	    (*state)->rule.ptr->max_src_conn <
836 	    (*state)->src_node->conn) {
837 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONN], 1);
838 		bad++;
839 	}
840 
841 	if ((*state)->rule.ptr->max_src_conn_rate.limit &&
842 	    pf_check_threshold(&(*state)->src_node->conn_rate)) {
843 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONNRATE], 1);
844 		bad++;
845 	}
846 
847 	if (!bad)
848 		return (0);
849 
850 	/* Kill this state. */
851 	(*state)->timeout = PFTM_PURGE;
852 	pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
853 
854 	if ((*state)->rule.ptr->overload_tbl == NULL)
855 		return (1);
856 
857 	/* Schedule overloading and flushing task. */
858 	pfoe = malloc(sizeof(*pfoe), M_PFTEMP, M_NOWAIT);
859 	if (pfoe == NULL)
860 		return (1);	/* too bad :( */
861 
862 	bcopy(&(*state)->src_node->addr, &pfoe->addr, sizeof(pfoe->addr));
863 	pfoe->af = (*state)->key[PF_SK_WIRE]->af;
864 	pfoe->rule = (*state)->rule.ptr;
865 	pfoe->dir = (*state)->direction;
866 	PF_OVERLOADQ_LOCK();
867 	SLIST_INSERT_HEAD(&V_pf_overloadqueue, pfoe, next);
868 	PF_OVERLOADQ_UNLOCK();
869 	taskqueue_enqueue(taskqueue_swi, &V_pf_overloadtask);
870 
871 	return (1);
872 }
873 
874 static void
875 pf_overload_task(void *v, int pending)
876 {
877 	struct pf_overload_head queue;
878 	struct pfr_addr p;
879 	struct pf_overload_entry *pfoe, *pfoe1;
880 	uint32_t killed = 0;
881 
882 	CURVNET_SET((struct vnet *)v);
883 
884 	PF_OVERLOADQ_LOCK();
885 	queue = V_pf_overloadqueue;
886 	SLIST_INIT(&V_pf_overloadqueue);
887 	PF_OVERLOADQ_UNLOCK();
888 
889 	bzero(&p, sizeof(p));
890 	SLIST_FOREACH(pfoe, &queue, next) {
891 		counter_u64_add(V_pf_status.lcounters[LCNT_OVERLOAD_TABLE], 1);
892 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
893 			printf("%s: blocking address ", __func__);
894 			pf_print_host(&pfoe->addr, 0, pfoe->af);
895 			printf("\n");
896 		}
897 
898 		p.pfra_af = pfoe->af;
899 		switch (pfoe->af) {
900 #ifdef INET
901 		case AF_INET:
902 			p.pfra_net = 32;
903 			p.pfra_ip4addr = pfoe->addr.v4;
904 			break;
905 #endif
906 #ifdef INET6
907 		case AF_INET6:
908 			p.pfra_net = 128;
909 			p.pfra_ip6addr = pfoe->addr.v6;
910 			break;
911 #endif
912 		}
913 
914 		PF_RULES_WLOCK();
915 		pfr_insert_kentry(pfoe->rule->overload_tbl, &p, time_second);
916 		PF_RULES_WUNLOCK();
917 	}
918 
919 	/*
920 	 * Remove those entries, that don't need flushing.
921 	 */
922 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
923 		if (pfoe->rule->flush == 0) {
924 			SLIST_REMOVE(&queue, pfoe, pf_overload_entry, next);
925 			free(pfoe, M_PFTEMP);
926 		} else
927 			counter_u64_add(
928 			    V_pf_status.lcounters[LCNT_OVERLOAD_FLUSH], 1);
929 
930 	/* If nothing to flush, return. */
931 	if (SLIST_EMPTY(&queue)) {
932 		CURVNET_RESTORE();
933 		return;
934 	}
935 
936 	for (int i = 0; i <= V_pf_hashmask; i++) {
937 		struct pf_idhash *ih = &V_pf_idhash[i];
938 		struct pf_state_key *sk;
939 		struct pf_kstate *s;
940 
941 		PF_HASHROW_LOCK(ih);
942 		LIST_FOREACH(s, &ih->states, entry) {
943 		    sk = s->key[PF_SK_WIRE];
944 		    SLIST_FOREACH(pfoe, &queue, next)
945 			if (sk->af == pfoe->af &&
946 			    ((pfoe->rule->flush & PF_FLUSH_GLOBAL) ||
947 			    pfoe->rule == s->rule.ptr) &&
948 			    ((pfoe->dir == PF_OUT &&
949 			    PF_AEQ(&pfoe->addr, &sk->addr[1], sk->af)) ||
950 			    (pfoe->dir == PF_IN &&
951 			    PF_AEQ(&pfoe->addr, &sk->addr[0], sk->af)))) {
952 				s->timeout = PFTM_PURGE;
953 				pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
954 				killed++;
955 			}
956 		}
957 		PF_HASHROW_UNLOCK(ih);
958 	}
959 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
960 		free(pfoe, M_PFTEMP);
961 	if (V_pf_status.debug >= PF_DEBUG_MISC)
962 		printf("%s: %u states killed", __func__, killed);
963 
964 	CURVNET_RESTORE();
965 }
966 
967 /*
968  * Can return locked on failure, so that we can consistently
969  * allocate and insert a new one.
970  */
971 struct pf_ksrc_node *
972 pf_find_src_node(struct pf_addr *src, struct pf_krule *rule, sa_family_t af,
973 	struct pf_srchash **sh, bool returnlocked)
974 {
975 	struct pf_ksrc_node *n;
976 
977 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_SEARCH], 1);
978 
979 	*sh = &V_pf_srchash[pf_hashsrc(src, af)];
980 	PF_HASHROW_LOCK(*sh);
981 	LIST_FOREACH(n, &(*sh)->nodes, entry)
982 		if (n->rule.ptr == rule && n->af == af &&
983 		    ((af == AF_INET && n->addr.v4.s_addr == src->v4.s_addr) ||
984 		    (af == AF_INET6 && bcmp(&n->addr, src, sizeof(*src)) == 0)))
985 			break;
986 
987 	if (n != NULL) {
988 		n->states++;
989 		PF_HASHROW_UNLOCK(*sh);
990 	} else if (returnlocked == false)
991 		PF_HASHROW_UNLOCK(*sh);
992 
993 	return (n);
994 }
995 
996 static void
997 pf_free_src_node(struct pf_ksrc_node *sn)
998 {
999 
1000 	for (int i = 0; i < 2; i++) {
1001 		counter_u64_free(sn->bytes[i]);
1002 		counter_u64_free(sn->packets[i]);
1003 	}
1004 	uma_zfree(V_pf_sources_z, sn);
1005 }
1006 
1007 static u_short
1008 pf_insert_src_node(struct pf_ksrc_node **sn, struct pf_krule *rule,
1009     struct pf_addr *src, sa_family_t af)
1010 {
1011 	u_short			 reason = 0;
1012 	struct pf_srchash	*sh = NULL;
1013 
1014 	KASSERT((rule->rule_flag & PFRULE_SRCTRACK ||
1015 	    rule->rpool.opts & PF_POOL_STICKYADDR),
1016 	    ("%s for non-tracking rule %p", __func__, rule));
1017 
1018 	if (*sn == NULL)
1019 		*sn = pf_find_src_node(src, rule, af, &sh, true);
1020 
1021 	if (*sn == NULL) {
1022 		PF_HASHROW_ASSERT(sh);
1023 
1024 		if (rule->max_src_nodes &&
1025 		    counter_u64_fetch(rule->src_nodes) >= rule->max_src_nodes) {
1026 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCNODES], 1);
1027 			PF_HASHROW_UNLOCK(sh);
1028 			reason = PFRES_SRCLIMIT;
1029 			goto done;
1030 		}
1031 
1032 		(*sn) = uma_zalloc(V_pf_sources_z, M_NOWAIT | M_ZERO);
1033 		if ((*sn) == NULL) {
1034 			PF_HASHROW_UNLOCK(sh);
1035 			reason = PFRES_MEMORY;
1036 			goto done;
1037 		}
1038 
1039 		for (int i = 0; i < 2; i++) {
1040 			(*sn)->bytes[i] = counter_u64_alloc(M_NOWAIT);
1041 			(*sn)->packets[i] = counter_u64_alloc(M_NOWAIT);
1042 
1043 			if ((*sn)->bytes[i] == NULL || (*sn)->packets[i] == NULL) {
1044 				pf_free_src_node(*sn);
1045 				PF_HASHROW_UNLOCK(sh);
1046 				reason = PFRES_MEMORY;
1047 				goto done;
1048 			}
1049 		}
1050 
1051 		pf_init_threshold(&(*sn)->conn_rate,
1052 		    rule->max_src_conn_rate.limit,
1053 		    rule->max_src_conn_rate.seconds);
1054 
1055 		MPASS((*sn)->lock == NULL);
1056 		(*sn)->lock = &sh->lock;
1057 
1058 		(*sn)->af = af;
1059 		(*sn)->rule.ptr = rule;
1060 		PF_ACPY(&(*sn)->addr, src, af);
1061 		LIST_INSERT_HEAD(&sh->nodes, *sn, entry);
1062 		(*sn)->creation = time_uptime;
1063 		(*sn)->ruletype = rule->action;
1064 		(*sn)->states = 1;
1065 		if ((*sn)->rule.ptr != NULL)
1066 			counter_u64_add((*sn)->rule.ptr->src_nodes, 1);
1067 		PF_HASHROW_UNLOCK(sh);
1068 		counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_INSERT], 1);
1069 	} else {
1070 		if (rule->max_src_states &&
1071 		    (*sn)->states >= rule->max_src_states) {
1072 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCSTATES],
1073 			    1);
1074 			reason = PFRES_SRCLIMIT;
1075 			goto done;
1076 		}
1077 	}
1078 done:
1079 	return (reason);
1080 }
1081 
1082 void
1083 pf_unlink_src_node(struct pf_ksrc_node *src)
1084 {
1085 	PF_SRC_NODE_LOCK_ASSERT(src);
1086 
1087 	LIST_REMOVE(src, entry);
1088 	if (src->rule.ptr)
1089 		counter_u64_add(src->rule.ptr->src_nodes, -1);
1090 }
1091 
1092 u_int
1093 pf_free_src_nodes(struct pf_ksrc_node_list *head)
1094 {
1095 	struct pf_ksrc_node *sn, *tmp;
1096 	u_int count = 0;
1097 
1098 	LIST_FOREACH_SAFE(sn, head, entry, tmp) {
1099 		pf_free_src_node(sn);
1100 		count++;
1101 	}
1102 
1103 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], count);
1104 
1105 	return (count);
1106 }
1107 
1108 void
1109 pf_mtag_initialize(void)
1110 {
1111 
1112 	pf_mtag_z = uma_zcreate("pf mtags", sizeof(struct m_tag) +
1113 	    sizeof(struct pf_mtag), NULL, NULL, pf_mtag_uminit, NULL,
1114 	    UMA_ALIGN_PTR, 0);
1115 }
1116 
1117 /* Per-vnet data storage structures initialization. */
1118 void
1119 pf_initialize(void)
1120 {
1121 	struct pf_keyhash	*kh;
1122 	struct pf_idhash	*ih;
1123 	struct pf_srchash	*sh;
1124 	struct pf_udpendpointhash	*uh;
1125 	u_int i;
1126 
1127 	if (V_pf_hashsize == 0 || !powerof2(V_pf_hashsize))
1128 		V_pf_hashsize = PF_HASHSIZ;
1129 	if (V_pf_srchashsize == 0 || !powerof2(V_pf_srchashsize))
1130 		V_pf_srchashsize = PF_SRCHASHSIZ;
1131 	if (V_pf_udpendpointhashsize == 0 || !powerof2(V_pf_udpendpointhashsize))
1132 		V_pf_udpendpointhashsize = PF_UDPENDHASHSIZ;
1133 
1134 	V_pf_hashseed = arc4random();
1135 
1136 	/* States and state keys storage. */
1137 	V_pf_state_z = uma_zcreate("pf states", sizeof(struct pf_kstate),
1138 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1139 	V_pf_limits[PF_LIMIT_STATES].zone = V_pf_state_z;
1140 	uma_zone_set_max(V_pf_state_z, PFSTATE_HIWAT);
1141 	uma_zone_set_warning(V_pf_state_z, "PF states limit reached");
1142 
1143 	V_pf_state_key_z = uma_zcreate("pf state keys",
1144 	    sizeof(struct pf_state_key), pf_state_key_ctor, NULL, NULL, NULL,
1145 	    UMA_ALIGN_PTR, 0);
1146 
1147 	V_pf_keyhash = mallocarray(V_pf_hashsize, sizeof(struct pf_keyhash),
1148 	    M_PFHASH, M_NOWAIT | M_ZERO);
1149 	V_pf_idhash = mallocarray(V_pf_hashsize, sizeof(struct pf_idhash),
1150 	    M_PFHASH, M_NOWAIT | M_ZERO);
1151 	if (V_pf_keyhash == NULL || V_pf_idhash == NULL) {
1152 		printf("pf: Unable to allocate memory for "
1153 		    "state_hashsize %lu.\n", V_pf_hashsize);
1154 
1155 		free(V_pf_keyhash, M_PFHASH);
1156 		free(V_pf_idhash, M_PFHASH);
1157 
1158 		V_pf_hashsize = PF_HASHSIZ;
1159 		V_pf_keyhash = mallocarray(V_pf_hashsize,
1160 		    sizeof(struct pf_keyhash), M_PFHASH, M_WAITOK | M_ZERO);
1161 		V_pf_idhash = mallocarray(V_pf_hashsize,
1162 		    sizeof(struct pf_idhash), M_PFHASH, M_WAITOK | M_ZERO);
1163 	}
1164 
1165 	V_pf_hashmask = V_pf_hashsize - 1;
1166 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= V_pf_hashmask;
1167 	    i++, kh++, ih++) {
1168 		mtx_init(&kh->lock, "pf_keyhash", NULL, MTX_DEF | MTX_DUPOK);
1169 		mtx_init(&ih->lock, "pf_idhash", NULL, MTX_DEF);
1170 	}
1171 
1172 	/* Source nodes. */
1173 	V_pf_sources_z = uma_zcreate("pf source nodes",
1174 	    sizeof(struct pf_ksrc_node), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
1175 	    0);
1176 	V_pf_limits[PF_LIMIT_SRC_NODES].zone = V_pf_sources_z;
1177 	uma_zone_set_max(V_pf_sources_z, PFSNODE_HIWAT);
1178 	uma_zone_set_warning(V_pf_sources_z, "PF source nodes limit reached");
1179 
1180 	V_pf_srchash = mallocarray(V_pf_srchashsize,
1181 	    sizeof(struct pf_srchash), M_PFHASH, M_NOWAIT | M_ZERO);
1182 	if (V_pf_srchash == NULL) {
1183 		printf("pf: Unable to allocate memory for "
1184 		    "source_hashsize %lu.\n", V_pf_srchashsize);
1185 
1186 		V_pf_srchashsize = PF_SRCHASHSIZ;
1187 		V_pf_srchash = mallocarray(V_pf_srchashsize,
1188 		    sizeof(struct pf_srchash), M_PFHASH, M_WAITOK | M_ZERO);
1189 	}
1190 
1191 	V_pf_srchashmask = V_pf_srchashsize - 1;
1192 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++)
1193 		mtx_init(&sh->lock, "pf_srchash", NULL, MTX_DEF);
1194 
1195 
1196 	/* UDP endpoint mappings. */
1197 	V_pf_udp_mapping_z = uma_zcreate("pf UDP mappings",
1198 	    sizeof(struct pf_udp_mapping), NULL, NULL, NULL, NULL,
1199 	    UMA_ALIGN_PTR, 0);
1200 	V_pf_udpendpointhash = mallocarray(V_pf_udpendpointhashsize,
1201 	    sizeof(struct pf_udpendpointhash), M_PFHASH, M_NOWAIT | M_ZERO);
1202 	if (V_pf_udpendpointhash == NULL) {
1203 		printf("pf: Unable to allocate memory for "
1204 		    "udpendpoint_hashsize %lu.\n", V_pf_udpendpointhashsize);
1205 
1206 		V_pf_udpendpointhashsize = PF_UDPENDHASHSIZ;
1207 		V_pf_udpendpointhash = mallocarray(V_pf_udpendpointhashsize,
1208 		    sizeof(struct pf_udpendpointhash), M_PFHASH, M_WAITOK | M_ZERO);
1209 	}
1210 
1211 	V_pf_udpendpointhashmask = V_pf_udpendpointhashsize - 1;
1212 	for (i = 0, uh = V_pf_udpendpointhash;
1213 	    i <= V_pf_udpendpointhashmask;
1214 	    i++, uh++) {
1215 		mtx_init(&uh->lock, "pf_udpendpointhash", NULL,
1216 		    MTX_DEF | MTX_DUPOK);
1217 	}
1218 
1219 	/* ALTQ */
1220 	TAILQ_INIT(&V_pf_altqs[0]);
1221 	TAILQ_INIT(&V_pf_altqs[1]);
1222 	TAILQ_INIT(&V_pf_altqs[2]);
1223 	TAILQ_INIT(&V_pf_altqs[3]);
1224 	TAILQ_INIT(&V_pf_pabuf);
1225 	V_pf_altqs_active = &V_pf_altqs[0];
1226 	V_pf_altq_ifs_active = &V_pf_altqs[1];
1227 	V_pf_altqs_inactive = &V_pf_altqs[2];
1228 	V_pf_altq_ifs_inactive = &V_pf_altqs[3];
1229 
1230 	/* Send & overload+flush queues. */
1231 	STAILQ_INIT(&V_pf_sendqueue);
1232 	SLIST_INIT(&V_pf_overloadqueue);
1233 	TASK_INIT(&V_pf_overloadtask, 0, pf_overload_task, curvnet);
1234 
1235 	/* Unlinked, but may be referenced rules. */
1236 	TAILQ_INIT(&V_pf_unlinked_rules);
1237 }
1238 
1239 void
1240 pf_mtag_cleanup(void)
1241 {
1242 
1243 	uma_zdestroy(pf_mtag_z);
1244 }
1245 
1246 void
1247 pf_cleanup(void)
1248 {
1249 	struct pf_keyhash	*kh;
1250 	struct pf_idhash	*ih;
1251 	struct pf_srchash	*sh;
1252 	struct pf_udpendpointhash	*uh;
1253 	struct pf_send_entry	*pfse, *next;
1254 	u_int i;
1255 
1256 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash;
1257 	    i <= V_pf_hashmask;
1258 	    i++, kh++, ih++) {
1259 		KASSERT(LIST_EMPTY(&kh->keys), ("%s: key hash not empty",
1260 		    __func__));
1261 		KASSERT(LIST_EMPTY(&ih->states), ("%s: id hash not empty",
1262 		    __func__));
1263 		mtx_destroy(&kh->lock);
1264 		mtx_destroy(&ih->lock);
1265 	}
1266 	free(V_pf_keyhash, M_PFHASH);
1267 	free(V_pf_idhash, M_PFHASH);
1268 
1269 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
1270 		KASSERT(LIST_EMPTY(&sh->nodes),
1271 		    ("%s: source node hash not empty", __func__));
1272 		mtx_destroy(&sh->lock);
1273 	}
1274 	free(V_pf_srchash, M_PFHASH);
1275 
1276 	for (i = 0, uh = V_pf_udpendpointhash;
1277 	    i <= V_pf_udpendpointhashmask;
1278 	    i++, uh++) {
1279 		KASSERT(LIST_EMPTY(&uh->endpoints),
1280 		    ("%s: udp endpoint hash not empty", __func__));
1281 		mtx_destroy(&uh->lock);
1282 	}
1283 	free(V_pf_udpendpointhash, M_PFHASH);
1284 
1285 	STAILQ_FOREACH_SAFE(pfse, &V_pf_sendqueue, pfse_next, next) {
1286 		m_freem(pfse->pfse_m);
1287 		free(pfse, M_PFTEMP);
1288 	}
1289 	MPASS(RB_EMPTY(&V_pf_sctp_endpoints));
1290 
1291 	uma_zdestroy(V_pf_sources_z);
1292 	uma_zdestroy(V_pf_state_z);
1293 	uma_zdestroy(V_pf_state_key_z);
1294 	uma_zdestroy(V_pf_udp_mapping_z);
1295 }
1296 
1297 static int
1298 pf_mtag_uminit(void *mem, int size, int how)
1299 {
1300 	struct m_tag *t;
1301 
1302 	t = (struct m_tag *)mem;
1303 	t->m_tag_cookie = MTAG_ABI_COMPAT;
1304 	t->m_tag_id = PACKET_TAG_PF;
1305 	t->m_tag_len = sizeof(struct pf_mtag);
1306 	t->m_tag_free = pf_mtag_free;
1307 
1308 	return (0);
1309 }
1310 
1311 static void
1312 pf_mtag_free(struct m_tag *t)
1313 {
1314 
1315 	uma_zfree(pf_mtag_z, t);
1316 }
1317 
1318 struct pf_mtag *
1319 pf_get_mtag(struct mbuf *m)
1320 {
1321 	struct m_tag *mtag;
1322 
1323 	if ((mtag = m_tag_find(m, PACKET_TAG_PF, NULL)) != NULL)
1324 		return ((struct pf_mtag *)(mtag + 1));
1325 
1326 	mtag = uma_zalloc(pf_mtag_z, M_NOWAIT);
1327 	if (mtag == NULL)
1328 		return (NULL);
1329 	bzero(mtag + 1, sizeof(struct pf_mtag));
1330 	m_tag_prepend(m, mtag);
1331 
1332 	return ((struct pf_mtag *)(mtag + 1));
1333 }
1334 
1335 static int
1336 pf_state_key_attach(struct pf_state_key *skw, struct pf_state_key *sks,
1337     struct pf_kstate *s)
1338 {
1339 	struct pf_keyhash	*khs, *khw, *kh;
1340 	struct pf_state_key	*sk, *cur;
1341 	struct pf_kstate	*si, *olds = NULL;
1342 	int idx;
1343 
1344 	NET_EPOCH_ASSERT();
1345 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1346 	KASSERT(s->key[PF_SK_WIRE] == NULL, ("%s: state has key", __func__));
1347 	KASSERT(s->key[PF_SK_STACK] == NULL, ("%s: state has key", __func__));
1348 
1349 	/*
1350 	 * We need to lock hash slots of both keys. To avoid deadlock
1351 	 * we always lock the slot with lower address first. Unlock order
1352 	 * isn't important.
1353 	 *
1354 	 * We also need to lock ID hash slot before dropping key
1355 	 * locks. On success we return with ID hash slot locked.
1356 	 */
1357 
1358 	if (skw == sks) {
1359 		khs = khw = &V_pf_keyhash[pf_hashkey(skw)];
1360 		PF_HASHROW_LOCK(khs);
1361 	} else {
1362 		khs = &V_pf_keyhash[pf_hashkey(sks)];
1363 		khw = &V_pf_keyhash[pf_hashkey(skw)];
1364 		if (khs == khw) {
1365 			PF_HASHROW_LOCK(khs);
1366 		} else if (khs < khw) {
1367 			PF_HASHROW_LOCK(khs);
1368 			PF_HASHROW_LOCK(khw);
1369 		} else {
1370 			PF_HASHROW_LOCK(khw);
1371 			PF_HASHROW_LOCK(khs);
1372 		}
1373 	}
1374 
1375 #define	KEYS_UNLOCK()	do {			\
1376 	if (khs != khw) {			\
1377 		PF_HASHROW_UNLOCK(khs);		\
1378 		PF_HASHROW_UNLOCK(khw);		\
1379 	} else					\
1380 		PF_HASHROW_UNLOCK(khs);		\
1381 } while (0)
1382 
1383 	/*
1384 	 * First run: start with wire key.
1385 	 */
1386 	sk = skw;
1387 	kh = khw;
1388 	idx = PF_SK_WIRE;
1389 
1390 	MPASS(s->lock == NULL);
1391 	s->lock = &V_pf_idhash[PF_IDHASH(s)].lock;
1392 
1393 keyattach:
1394 	LIST_FOREACH(cur, &kh->keys, entry)
1395 		if (bcmp(cur, sk, sizeof(struct pf_state_key_cmp)) == 0)
1396 			break;
1397 
1398 	if (cur != NULL) {
1399 		/* Key exists. Check for same kif, if none, add to key. */
1400 		TAILQ_FOREACH(si, &cur->states[idx], key_list[idx]) {
1401 			struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(si)];
1402 
1403 			PF_HASHROW_LOCK(ih);
1404 			if (si->kif == s->kif &&
1405 			    si->direction == s->direction) {
1406 				if (sk->proto == IPPROTO_TCP &&
1407 				    si->src.state >= TCPS_FIN_WAIT_2 &&
1408 				    si->dst.state >= TCPS_FIN_WAIT_2) {
1409 					/*
1410 					 * New state matches an old >FIN_WAIT_2
1411 					 * state. We can't drop key hash locks,
1412 					 * thus we can't unlink it properly.
1413 					 *
1414 					 * As a workaround we drop it into
1415 					 * TCPS_CLOSED state, schedule purge
1416 					 * ASAP and push it into the very end
1417 					 * of the slot TAILQ, so that it won't
1418 					 * conflict with our new state.
1419 					 */
1420 					pf_set_protostate(si, PF_PEER_BOTH,
1421 					    TCPS_CLOSED);
1422 					si->timeout = PFTM_PURGE;
1423 					olds = si;
1424 				} else {
1425 					if (V_pf_status.debug >= PF_DEBUG_MISC) {
1426 						printf("pf: %s key attach "
1427 						    "failed on %s: ",
1428 						    (idx == PF_SK_WIRE) ?
1429 						    "wire" : "stack",
1430 						    s->kif->pfik_name);
1431 						pf_print_state_parts(s,
1432 						    (idx == PF_SK_WIRE) ?
1433 						    sk : NULL,
1434 						    (idx == PF_SK_STACK) ?
1435 						    sk : NULL);
1436 						printf(", existing: ");
1437 						pf_print_state_parts(si,
1438 						    (idx == PF_SK_WIRE) ?
1439 						    sk : NULL,
1440 						    (idx == PF_SK_STACK) ?
1441 						    sk : NULL);
1442 						printf("\n");
1443 					}
1444 					s->timeout = PFTM_UNLINKED;
1445 					PF_HASHROW_UNLOCK(ih);
1446 					KEYS_UNLOCK();
1447 					uma_zfree(V_pf_state_key_z, sk);
1448 					if (idx == PF_SK_STACK)
1449 						pf_detach_state(s);
1450 					return (EEXIST); /* collision! */
1451 				}
1452 			}
1453 			PF_HASHROW_UNLOCK(ih);
1454 		}
1455 		uma_zfree(V_pf_state_key_z, sk);
1456 		s->key[idx] = cur;
1457 	} else {
1458 		LIST_INSERT_HEAD(&kh->keys, sk, entry);
1459 		s->key[idx] = sk;
1460 	}
1461 
1462 stateattach:
1463 	/* List is sorted, if-bound states before floating. */
1464 	if (s->kif == V_pfi_all)
1465 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], s, key_list[idx]);
1466 	else
1467 		TAILQ_INSERT_HEAD(&s->key[idx]->states[idx], s, key_list[idx]);
1468 
1469 	if (olds) {
1470 		TAILQ_REMOVE(&s->key[idx]->states[idx], olds, key_list[idx]);
1471 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], olds,
1472 		    key_list[idx]);
1473 		olds = NULL;
1474 	}
1475 
1476 	/*
1477 	 * Attach done. See how should we (or should not?)
1478 	 * attach a second key.
1479 	 */
1480 	if (sks == skw) {
1481 		s->key[PF_SK_STACK] = s->key[PF_SK_WIRE];
1482 		idx = PF_SK_STACK;
1483 		sks = NULL;
1484 		goto stateattach;
1485 	} else if (sks != NULL) {
1486 		/*
1487 		 * Continue attaching with stack key.
1488 		 */
1489 		sk = sks;
1490 		kh = khs;
1491 		idx = PF_SK_STACK;
1492 		sks = NULL;
1493 		goto keyattach;
1494 	}
1495 
1496 	PF_STATE_LOCK(s);
1497 	KEYS_UNLOCK();
1498 
1499 	KASSERT(s->key[PF_SK_WIRE] != NULL && s->key[PF_SK_STACK] != NULL,
1500 	    ("%s failure", __func__));
1501 
1502 	return (0);
1503 #undef	KEYS_UNLOCK
1504 }
1505 
1506 static void
1507 pf_detach_state(struct pf_kstate *s)
1508 {
1509 	struct pf_state_key *sks = s->key[PF_SK_STACK];
1510 	struct pf_keyhash *kh;
1511 
1512 	NET_EPOCH_ASSERT();
1513 	MPASS(s->timeout >= PFTM_MAX);
1514 
1515 	pf_sctp_multihome_detach_addr(s);
1516 
1517 	if ((s->state_flags & PFSTATE_PFLOW) && V_pflow_export_state_ptr)
1518 		V_pflow_export_state_ptr(s);
1519 
1520 	if (sks != NULL) {
1521 		kh = &V_pf_keyhash[pf_hashkey(sks)];
1522 		PF_HASHROW_LOCK(kh);
1523 		if (s->key[PF_SK_STACK] != NULL)
1524 			pf_state_key_detach(s, PF_SK_STACK);
1525 		/*
1526 		 * If both point to same key, then we are done.
1527 		 */
1528 		if (sks == s->key[PF_SK_WIRE]) {
1529 			pf_state_key_detach(s, PF_SK_WIRE);
1530 			PF_HASHROW_UNLOCK(kh);
1531 			return;
1532 		}
1533 		PF_HASHROW_UNLOCK(kh);
1534 	}
1535 
1536 	if (s->key[PF_SK_WIRE] != NULL) {
1537 		kh = &V_pf_keyhash[pf_hashkey(s->key[PF_SK_WIRE])];
1538 		PF_HASHROW_LOCK(kh);
1539 		if (s->key[PF_SK_WIRE] != NULL)
1540 			pf_state_key_detach(s, PF_SK_WIRE);
1541 		PF_HASHROW_UNLOCK(kh);
1542 	}
1543 }
1544 
1545 static void
1546 pf_state_key_detach(struct pf_kstate *s, int idx)
1547 {
1548 	struct pf_state_key *sk = s->key[idx];
1549 #ifdef INVARIANTS
1550 	struct pf_keyhash *kh = &V_pf_keyhash[pf_hashkey(sk)];
1551 
1552 	PF_HASHROW_ASSERT(kh);
1553 #endif
1554 	TAILQ_REMOVE(&sk->states[idx], s, key_list[idx]);
1555 	s->key[idx] = NULL;
1556 
1557 	if (TAILQ_EMPTY(&sk->states[0]) && TAILQ_EMPTY(&sk->states[1])) {
1558 		LIST_REMOVE(sk, entry);
1559 		uma_zfree(V_pf_state_key_z, sk);
1560 	}
1561 }
1562 
1563 static int
1564 pf_state_key_ctor(void *mem, int size, void *arg, int flags)
1565 {
1566 	struct pf_state_key *sk = mem;
1567 
1568 	bzero(sk, sizeof(struct pf_state_key_cmp));
1569 	TAILQ_INIT(&sk->states[PF_SK_WIRE]);
1570 	TAILQ_INIT(&sk->states[PF_SK_STACK]);
1571 
1572 	return (0);
1573 }
1574 
1575 static int
1576 pf_state_key_addr_setup(struct pf_pdesc *pd, struct mbuf *m, int off,
1577     struct pf_state_key_cmp *key, int sidx, struct pf_addr *saddr,
1578     int didx, struct pf_addr *daddr, int multi)
1579 {
1580 #ifdef INET6
1581 	struct nd_neighbor_solicit nd;
1582 	struct pf_addr *target;
1583 	u_short action, reason;
1584 
1585 	if (pd->af == AF_INET || pd->proto != IPPROTO_ICMPV6)
1586 		goto copy;
1587 
1588 	switch (pd->hdr.icmp6.icmp6_type) {
1589 	case ND_NEIGHBOR_SOLICIT:
1590 		if (multi)
1591 			return (-1);
1592 		if (!pf_pull_hdr(m, off, &nd, sizeof(nd), &action, &reason, pd->af))
1593 			return (-1);
1594 		target = (struct pf_addr *)&nd.nd_ns_target;
1595 		daddr = target;
1596 		break;
1597 	case ND_NEIGHBOR_ADVERT:
1598 		if (multi)
1599 			return (-1);
1600 		if (!pf_pull_hdr(m, off, &nd, sizeof(nd), &action, &reason, pd->af))
1601 			return (-1);
1602 		target = (struct pf_addr *)&nd.nd_ns_target;
1603 		saddr = target;
1604 		if (IN6_IS_ADDR_MULTICAST(&pd->dst->v6)) {
1605 			key->addr[didx].addr32[0] = 0;
1606 			key->addr[didx].addr32[1] = 0;
1607 			key->addr[didx].addr32[2] = 0;
1608 			key->addr[didx].addr32[3] = 0;
1609 			daddr = NULL; /* overwritten */
1610 		}
1611 		break;
1612 	default:
1613 		if (multi == PF_ICMP_MULTI_LINK) {
1614 			key->addr[sidx].addr32[0] = IPV6_ADDR_INT32_MLL;
1615 			key->addr[sidx].addr32[1] = 0;
1616 			key->addr[sidx].addr32[2] = 0;
1617 			key->addr[sidx].addr32[3] = IPV6_ADDR_INT32_ONE;
1618 			saddr = NULL; /* overwritten */
1619 		}
1620 	}
1621 copy:
1622 #endif
1623 	if (saddr)
1624 		PF_ACPY(&key->addr[sidx], saddr, pd->af);
1625 	if (daddr)
1626 		PF_ACPY(&key->addr[didx], daddr, pd->af);
1627 
1628 	return (0);
1629 }
1630 
1631 struct pf_state_key *
1632 pf_state_key_setup(struct pf_pdesc *pd, struct mbuf *m, int off,
1633     struct pf_addr *saddr, struct pf_addr *daddr, u_int16_t sport,
1634     u_int16_t dport)
1635 {
1636 	struct pf_state_key *sk;
1637 
1638 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1639 	if (sk == NULL)
1640 		return (NULL);
1641 
1642 	if (pf_state_key_addr_setup(pd, m, off, (struct pf_state_key_cmp *)sk,
1643 	    pd->sidx, pd->src, pd->didx, pd->dst, 0)) {
1644 		uma_zfree(V_pf_state_key_z, sk);
1645 		return (NULL);
1646 	}
1647 
1648 	sk->port[pd->sidx] = sport;
1649 	sk->port[pd->didx] = dport;
1650 	sk->proto = pd->proto;
1651 	sk->af = pd->af;
1652 
1653 	return (sk);
1654 }
1655 
1656 struct pf_state_key *
1657 pf_state_key_clone(const struct pf_state_key *orig)
1658 {
1659 	struct pf_state_key *sk;
1660 
1661 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1662 	if (sk == NULL)
1663 		return (NULL);
1664 
1665 	bcopy(orig, sk, sizeof(struct pf_state_key_cmp));
1666 
1667 	return (sk);
1668 }
1669 
1670 int
1671 pf_state_insert(struct pfi_kkif *kif, struct pfi_kkif *orig_kif,
1672     struct pf_state_key *skw, struct pf_state_key *sks, struct pf_kstate *s)
1673 {
1674 	struct pf_idhash *ih;
1675 	struct pf_kstate *cur;
1676 	int error;
1677 
1678 	NET_EPOCH_ASSERT();
1679 
1680 	KASSERT(TAILQ_EMPTY(&sks->states[0]) && TAILQ_EMPTY(&sks->states[1]),
1681 	    ("%s: sks not pristine", __func__));
1682 	KASSERT(TAILQ_EMPTY(&skw->states[0]) && TAILQ_EMPTY(&skw->states[1]),
1683 	    ("%s: skw not pristine", __func__));
1684 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1685 
1686 	s->kif = kif;
1687 	s->orig_kif = orig_kif;
1688 
1689 	if (s->id == 0 && s->creatorid == 0) {
1690 		s->id = alloc_unr64(&V_pf_stateid);
1691 		s->id = htobe64(s->id);
1692 		s->creatorid = V_pf_status.hostid;
1693 	}
1694 
1695 	/* Returns with ID locked on success. */
1696 	if ((error = pf_state_key_attach(skw, sks, s)) != 0)
1697 		return (error);
1698 
1699 	ih = &V_pf_idhash[PF_IDHASH(s)];
1700 	PF_HASHROW_ASSERT(ih);
1701 	LIST_FOREACH(cur, &ih->states, entry)
1702 		if (cur->id == s->id && cur->creatorid == s->creatorid)
1703 			break;
1704 
1705 	if (cur != NULL) {
1706 		s->timeout = PFTM_UNLINKED;
1707 		PF_HASHROW_UNLOCK(ih);
1708 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
1709 			printf("pf: state ID collision: "
1710 			    "id: %016llx creatorid: %08x\n",
1711 			    (unsigned long long)be64toh(s->id),
1712 			    ntohl(s->creatorid));
1713 		}
1714 		pf_detach_state(s);
1715 		return (EEXIST);
1716 	}
1717 	LIST_INSERT_HEAD(&ih->states, s, entry);
1718 	/* One for keys, one for ID hash. */
1719 	refcount_init(&s->refs, 2);
1720 
1721 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_INSERT], 1);
1722 	if (V_pfsync_insert_state_ptr != NULL)
1723 		V_pfsync_insert_state_ptr(s);
1724 
1725 	/* Returns locked. */
1726 	return (0);
1727 }
1728 
1729 /*
1730  * Find state by ID: returns with locked row on success.
1731  */
1732 struct pf_kstate *
1733 pf_find_state_byid(uint64_t id, uint32_t creatorid)
1734 {
1735 	struct pf_idhash *ih;
1736 	struct pf_kstate *s;
1737 
1738 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1739 
1740 	ih = &V_pf_idhash[(be64toh(id) % (V_pf_hashmask + 1))];
1741 
1742 	PF_HASHROW_LOCK(ih);
1743 	LIST_FOREACH(s, &ih->states, entry)
1744 		if (s->id == id && s->creatorid == creatorid)
1745 			break;
1746 
1747 	if (s == NULL)
1748 		PF_HASHROW_UNLOCK(ih);
1749 
1750 	return (s);
1751 }
1752 
1753 /*
1754  * Find state by key.
1755  * Returns with ID hash slot locked on success.
1756  */
1757 static struct pf_kstate *
1758 pf_find_state(struct pfi_kkif *kif, const struct pf_state_key_cmp *key,
1759     u_int dir)
1760 {
1761 	struct pf_keyhash	*kh;
1762 	struct pf_state_key	*sk;
1763 	struct pf_kstate	*s;
1764 	int idx;
1765 
1766 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1767 
1768 	kh = &V_pf_keyhash[pf_hashkey((const struct pf_state_key *)key)];
1769 
1770 	PF_HASHROW_LOCK(kh);
1771 	LIST_FOREACH(sk, &kh->keys, entry)
1772 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1773 			break;
1774 	if (sk == NULL) {
1775 		PF_HASHROW_UNLOCK(kh);
1776 		return (NULL);
1777 	}
1778 
1779 	idx = (dir == PF_IN ? PF_SK_WIRE : PF_SK_STACK);
1780 
1781 	/* List is sorted, if-bound states before floating ones. */
1782 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx])
1783 		if (s->kif == V_pfi_all || s->kif == kif || s->orig_kif == kif) {
1784 			PF_STATE_LOCK(s);
1785 			PF_HASHROW_UNLOCK(kh);
1786 			if (__predict_false(s->timeout >= PFTM_MAX)) {
1787 				/*
1788 				 * State is either being processed by
1789 				 * pf_unlink_state() in an other thread, or
1790 				 * is scheduled for immediate expiry.
1791 				 */
1792 				PF_STATE_UNLOCK(s);
1793 				return (NULL);
1794 			}
1795 			return (s);
1796 		}
1797 	PF_HASHROW_UNLOCK(kh);
1798 
1799 	return (NULL);
1800 }
1801 
1802 /*
1803  * Returns with ID hash slot locked on success.
1804  */
1805 struct pf_kstate *
1806 pf_find_state_all(const struct pf_state_key_cmp *key, u_int dir, int *more)
1807 {
1808 	struct pf_keyhash	*kh;
1809 	struct pf_state_key	*sk;
1810 	struct pf_kstate	*s, *ret = NULL;
1811 	int			 idx, inout = 0;
1812 
1813 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1814 
1815 	kh = &V_pf_keyhash[pf_hashkey((const struct pf_state_key *)key)];
1816 
1817 	PF_HASHROW_LOCK(kh);
1818 	LIST_FOREACH(sk, &kh->keys, entry)
1819 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1820 			break;
1821 	if (sk == NULL) {
1822 		PF_HASHROW_UNLOCK(kh);
1823 		return (NULL);
1824 	}
1825 	switch (dir) {
1826 	case PF_IN:
1827 		idx = PF_SK_WIRE;
1828 		break;
1829 	case PF_OUT:
1830 		idx = PF_SK_STACK;
1831 		break;
1832 	case PF_INOUT:
1833 		idx = PF_SK_WIRE;
1834 		inout = 1;
1835 		break;
1836 	default:
1837 		panic("%s: dir %u", __func__, dir);
1838 	}
1839 second_run:
1840 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx]) {
1841 		if (more == NULL) {
1842 			PF_STATE_LOCK(s);
1843 			PF_HASHROW_UNLOCK(kh);
1844 			return (s);
1845 		}
1846 
1847 		if (ret)
1848 			(*more)++;
1849 		else {
1850 			ret = s;
1851 			PF_STATE_LOCK(s);
1852 		}
1853 	}
1854 	if (inout == 1) {
1855 		inout = 0;
1856 		idx = PF_SK_STACK;
1857 		goto second_run;
1858 	}
1859 	PF_HASHROW_UNLOCK(kh);
1860 
1861 	return (ret);
1862 }
1863 
1864 /*
1865  * FIXME
1866  * This routine is inefficient -- locks the state only to unlock immediately on
1867  * return.
1868  * It is racy -- after the state is unlocked nothing stops other threads from
1869  * removing it.
1870  */
1871 bool
1872 pf_find_state_all_exists(const struct pf_state_key_cmp *key, u_int dir)
1873 {
1874 	struct pf_kstate *s;
1875 
1876 	s = pf_find_state_all(key, dir, NULL);
1877 	if (s != NULL) {
1878 		PF_STATE_UNLOCK(s);
1879 		return (true);
1880 	}
1881 	return (false);
1882 }
1883 
1884 struct pf_udp_mapping *
1885 pf_udp_mapping_create(sa_family_t af, struct pf_addr *src_addr, uint16_t src_port,
1886     struct pf_addr *nat_addr, uint16_t nat_port)
1887 {
1888 	struct pf_udp_mapping *mapping;
1889 
1890 	mapping = uma_zalloc(V_pf_udp_mapping_z, M_NOWAIT | M_ZERO);
1891 	if (mapping == NULL)
1892 		return (NULL);
1893 	PF_ACPY(&mapping->endpoints[0].addr, src_addr, af);
1894 	mapping->endpoints[0].port = src_port;
1895 	mapping->endpoints[0].af = af;
1896 	mapping->endpoints[0].mapping = mapping;
1897 	PF_ACPY(&mapping->endpoints[1].addr, nat_addr, af);
1898 	mapping->endpoints[1].port = nat_port;
1899 	mapping->endpoints[1].af = af;
1900 	mapping->endpoints[1].mapping = mapping;
1901 	refcount_init(&mapping->refs, 1);
1902 	return (mapping);
1903 }
1904 
1905 int
1906 pf_udp_mapping_insert(struct pf_udp_mapping *mapping)
1907 {
1908 	struct pf_udpendpointhash *h0, *h1;
1909 	struct pf_udp_endpoint *endpoint;
1910 	int ret = EEXIST;
1911 
1912 	h0 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[0])];
1913 	h1 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[1])];
1914 	if (h0 == h1) {
1915 		PF_HASHROW_LOCK(h0);
1916 	} else if (h0 < h1) {
1917 		PF_HASHROW_LOCK(h0);
1918 		PF_HASHROW_LOCK(h1);
1919 	} else {
1920 		PF_HASHROW_LOCK(h1);
1921 		PF_HASHROW_LOCK(h0);
1922 	}
1923 
1924 	LIST_FOREACH(endpoint, &h0->endpoints, entry) {
1925 		if (bcmp(endpoint, &mapping->endpoints[0],
1926 		    sizeof(struct pf_udp_endpoint_cmp)) == 0)
1927 			break;
1928 	}
1929 	if (endpoint != NULL)
1930 		goto cleanup;
1931 	LIST_FOREACH(endpoint, &h1->endpoints, entry) {
1932 		if (bcmp(endpoint, &mapping->endpoints[1],
1933 		    sizeof(struct pf_udp_endpoint_cmp)) == 0)
1934 			break;
1935 	}
1936 	if (endpoint != NULL)
1937 		goto cleanup;
1938 	LIST_INSERT_HEAD(&h0->endpoints, &mapping->endpoints[0], entry);
1939 	LIST_INSERT_HEAD(&h1->endpoints, &mapping->endpoints[1], entry);
1940 	ret = 0;
1941 
1942 cleanup:
1943 	if (h0 != h1) {
1944 		PF_HASHROW_UNLOCK(h0);
1945 		PF_HASHROW_UNLOCK(h1);
1946 	} else {
1947 		PF_HASHROW_UNLOCK(h0);
1948 	}
1949 	return (ret);
1950 }
1951 
1952 void
1953 pf_udp_mapping_release(struct pf_udp_mapping *mapping)
1954 {
1955 	/* refcount is synchronized on the source endpoint's row lock */
1956 	struct pf_udpendpointhash *h0, *h1;
1957 
1958 	if (mapping == NULL)
1959 		return;
1960 
1961 	h0 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[0])];
1962 	PF_HASHROW_LOCK(h0);
1963 	if (refcount_release(&mapping->refs)) {
1964 		LIST_REMOVE(&mapping->endpoints[0], entry);
1965 		PF_HASHROW_UNLOCK(h0);
1966 		h1 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[1])];
1967 		PF_HASHROW_LOCK(h1);
1968 		LIST_REMOVE(&mapping->endpoints[1], entry);
1969 		PF_HASHROW_UNLOCK(h1);
1970 
1971 		uma_zfree(V_pf_udp_mapping_z, mapping);
1972 	} else {
1973 			PF_HASHROW_UNLOCK(h0);
1974 	}
1975 }
1976 
1977 
1978 struct pf_udp_mapping *
1979 pf_udp_mapping_find(struct pf_udp_endpoint_cmp *key)
1980 {
1981 	struct pf_udpendpointhash *uh;
1982 	struct pf_udp_endpoint *endpoint;
1983 
1984 	uh = &V_pf_udpendpointhash[pf_hashudpendpoint((struct pf_udp_endpoint*)key)];
1985 
1986 	PF_HASHROW_LOCK(uh);
1987 	LIST_FOREACH(endpoint, &uh->endpoints, entry) {
1988 		if (bcmp(endpoint, key, sizeof(struct pf_udp_endpoint_cmp)) == 0 &&
1989 			bcmp(endpoint, &endpoint->mapping->endpoints[0],
1990 			    sizeof(struct pf_udp_endpoint_cmp)) == 0)
1991 			break;
1992 	}
1993 	if (endpoint == NULL) {
1994 		PF_HASHROW_UNLOCK(uh);
1995 		return (NULL);
1996 	}
1997 	refcount_acquire(&endpoint->mapping->refs);
1998 	PF_HASHROW_UNLOCK(uh);
1999 	return (endpoint->mapping);
2000 }
2001 /* END state table stuff */
2002 
2003 static void
2004 pf_send(struct pf_send_entry *pfse)
2005 {
2006 
2007 	PF_SENDQ_LOCK();
2008 	STAILQ_INSERT_TAIL(&V_pf_sendqueue, pfse, pfse_next);
2009 	PF_SENDQ_UNLOCK();
2010 	swi_sched(V_pf_swi_cookie, 0);
2011 }
2012 
2013 static bool
2014 pf_isforlocal(struct mbuf *m, int af)
2015 {
2016 	switch (af) {
2017 #ifdef INET
2018 	case AF_INET: {
2019 		struct ip *ip = mtod(m, struct ip *);
2020 
2021 		return (in_localip(ip->ip_dst));
2022 	}
2023 #endif
2024 #ifdef INET6
2025 	case AF_INET6: {
2026 		struct ip6_hdr *ip6;
2027 		struct in6_ifaddr *ia;
2028 		ip6 = mtod(m, struct ip6_hdr *);
2029 		ia = in6ifa_ifwithaddr(&ip6->ip6_dst, 0 /* XXX */, false);
2030 		if (ia == NULL)
2031 			return (false);
2032 		return (! (ia->ia6_flags & IN6_IFF_NOTREADY));
2033 	}
2034 #endif
2035 	default:
2036 		panic("Unsupported af %d", af);
2037 	}
2038 
2039 	return (false);
2040 }
2041 
2042 int
2043 pf_icmp_mapping(struct pf_pdesc *pd, u_int8_t type,
2044     int *icmp_dir, int *multi, u_int16_t *virtual_id, u_int16_t *virtual_type)
2045 {
2046 	/*
2047 	 * ICMP types marked with PF_OUT are typically responses to
2048 	 * PF_IN, and will match states in the opposite direction.
2049 	 * PF_IN ICMP types need to match a state with that type.
2050 	 */
2051 	*icmp_dir = PF_OUT;
2052 	*multi = PF_ICMP_MULTI_LINK;
2053 	/* Queries (and responses) */
2054 	switch (pd->af) {
2055 #ifdef INET
2056 	case AF_INET:
2057 		switch (type) {
2058 		case ICMP_ECHO:
2059 			*icmp_dir = PF_IN;
2060 		case ICMP_ECHOREPLY:
2061 			*virtual_type = ICMP_ECHO;
2062 			*virtual_id = pd->hdr.icmp.icmp_id;
2063 			break;
2064 
2065 		case ICMP_TSTAMP:
2066 			*icmp_dir = PF_IN;
2067 		case ICMP_TSTAMPREPLY:
2068 			*virtual_type = ICMP_TSTAMP;
2069 			*virtual_id = pd->hdr.icmp.icmp_id;
2070 			break;
2071 
2072 		case ICMP_IREQ:
2073 			*icmp_dir = PF_IN;
2074 		case ICMP_IREQREPLY:
2075 			*virtual_type = ICMP_IREQ;
2076 			*virtual_id = pd->hdr.icmp.icmp_id;
2077 			break;
2078 
2079 		case ICMP_MASKREQ:
2080 			*icmp_dir = PF_IN;
2081 		case ICMP_MASKREPLY:
2082 			*virtual_type = ICMP_MASKREQ;
2083 			*virtual_id = pd->hdr.icmp.icmp_id;
2084 			break;
2085 
2086 		case ICMP_IPV6_WHEREAREYOU:
2087 			*icmp_dir = PF_IN;
2088 		case ICMP_IPV6_IAMHERE:
2089 			*virtual_type = ICMP_IPV6_WHEREAREYOU;
2090 			*virtual_id = 0; /* Nothing sane to match on! */
2091 			break;
2092 
2093 		case ICMP_MOBILE_REGREQUEST:
2094 			*icmp_dir = PF_IN;
2095 		case ICMP_MOBILE_REGREPLY:
2096 			*virtual_type = ICMP_MOBILE_REGREQUEST;
2097 			*virtual_id = 0; /* Nothing sane to match on! */
2098 			break;
2099 
2100 		case ICMP_ROUTERSOLICIT:
2101 			*icmp_dir = PF_IN;
2102 		case ICMP_ROUTERADVERT:
2103 			*virtual_type = ICMP_ROUTERSOLICIT;
2104 			*virtual_id = 0; /* Nothing sane to match on! */
2105 			break;
2106 
2107 		/* These ICMP types map to other connections */
2108 		case ICMP_UNREACH:
2109 		case ICMP_SOURCEQUENCH:
2110 		case ICMP_REDIRECT:
2111 		case ICMP_TIMXCEED:
2112 		case ICMP_PARAMPROB:
2113 			/* These will not be used, but set them anyway */
2114 			*icmp_dir = PF_IN;
2115 			*virtual_type = type;
2116 			*virtual_id = 0;
2117 			HTONS(*virtual_type);
2118 			return (1);  /* These types match to another state */
2119 
2120 		/*
2121 		 * All remaining ICMP types get their own states,
2122 		 * and will only match in one direction.
2123 		 */
2124 		default:
2125 			*icmp_dir = PF_IN;
2126 			*virtual_type = type;
2127 			*virtual_id = 0;
2128 			break;
2129 		}
2130 		break;
2131 #endif /* INET */
2132 #ifdef INET6
2133 	case AF_INET6:
2134 		switch (type) {
2135 		case ICMP6_ECHO_REQUEST:
2136 			*icmp_dir = PF_IN;
2137 		case ICMP6_ECHO_REPLY:
2138 			*virtual_type = ICMP6_ECHO_REQUEST;
2139 			*virtual_id = pd->hdr.icmp6.icmp6_id;
2140 			break;
2141 
2142 		case MLD_LISTENER_QUERY:
2143 		case MLD_LISTENER_REPORT: {
2144 			/*
2145 			 * Listener Report can be sent by clients
2146 			 * without an associated Listener Query.
2147 			 * In addition to that, when Report is sent as a
2148 			 * reply to a Query its source and destination
2149 			 * address are different.
2150 			 */
2151 			*icmp_dir = PF_IN;
2152 			*virtual_type = MLD_LISTENER_QUERY;
2153 			*virtual_id = 0;
2154 			break;
2155 		}
2156 		case MLD_MTRACE:
2157 			*icmp_dir = PF_IN;
2158 		case MLD_MTRACE_RESP:
2159 			*virtual_type = MLD_MTRACE;
2160 			*virtual_id = 0; /* Nothing sane to match on! */
2161 			break;
2162 
2163 		case ND_NEIGHBOR_SOLICIT:
2164 			*icmp_dir = PF_IN;
2165 		case ND_NEIGHBOR_ADVERT: {
2166 			*virtual_type = ND_NEIGHBOR_SOLICIT;
2167 			*virtual_id = 0;
2168 			break;
2169 		}
2170 
2171 		/*
2172 		 * These ICMP types map to other connections.
2173 		 * ND_REDIRECT can't be in this list because the triggering
2174 		 * packet header is optional.
2175 		 */
2176 		case ICMP6_DST_UNREACH:
2177 		case ICMP6_PACKET_TOO_BIG:
2178 		case ICMP6_TIME_EXCEEDED:
2179 		case ICMP6_PARAM_PROB:
2180 			/* These will not be used, but set them anyway */
2181 			*icmp_dir = PF_IN;
2182 			*virtual_type = type;
2183 			*virtual_id = 0;
2184 			HTONS(*virtual_type);
2185 			return (1);  /* These types match to another state */
2186 		/*
2187 		 * All remaining ICMP6 types get their own states,
2188 		 * and will only match in one direction.
2189 		 */
2190 		default:
2191 			*icmp_dir = PF_IN;
2192 			*virtual_type = type;
2193 			*virtual_id = 0;
2194 			break;
2195 		}
2196 		break;
2197 #endif /* INET6 */
2198 	default:
2199 		*icmp_dir = PF_IN;
2200 		*virtual_type = type;
2201 		*virtual_id = 0;
2202 		break;
2203 	}
2204 	HTONS(*virtual_type);
2205 	return (0);  /* These types match to their own state */
2206 }
2207 
2208 void
2209 pf_intr(void *v)
2210 {
2211 	struct epoch_tracker et;
2212 	struct pf_send_head queue;
2213 	struct pf_send_entry *pfse, *next;
2214 
2215 	CURVNET_SET((struct vnet *)v);
2216 
2217 	PF_SENDQ_LOCK();
2218 	queue = V_pf_sendqueue;
2219 	STAILQ_INIT(&V_pf_sendqueue);
2220 	PF_SENDQ_UNLOCK();
2221 
2222 	NET_EPOCH_ENTER(et);
2223 
2224 	STAILQ_FOREACH_SAFE(pfse, &queue, pfse_next, next) {
2225 		switch (pfse->pfse_type) {
2226 #ifdef INET
2227 		case PFSE_IP: {
2228 			if (pf_isforlocal(pfse->pfse_m, AF_INET)) {
2229 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
2230 				pfse->pfse_m->m_pkthdr.csum_flags |=
2231 				    CSUM_IP_VALID | CSUM_IP_CHECKED;
2232 				ip_input(pfse->pfse_m);
2233 			} else {
2234 				ip_output(pfse->pfse_m, NULL, NULL, 0, NULL,
2235 				    NULL);
2236 			}
2237 			break;
2238 		}
2239 		case PFSE_ICMP:
2240 			icmp_error(pfse->pfse_m, pfse->icmpopts.type,
2241 			    pfse->icmpopts.code, 0, pfse->icmpopts.mtu);
2242 			break;
2243 #endif /* INET */
2244 #ifdef INET6
2245 		case PFSE_IP6:
2246 			if (pf_isforlocal(pfse->pfse_m, AF_INET6)) {
2247 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
2248 				ip6_input(pfse->pfse_m);
2249 			} else {
2250 				ip6_output(pfse->pfse_m, NULL, NULL, 0, NULL,
2251 				    NULL, NULL);
2252 			}
2253 			break;
2254 		case PFSE_ICMP6:
2255 			icmp6_error(pfse->pfse_m, pfse->icmpopts.type,
2256 			    pfse->icmpopts.code, pfse->icmpopts.mtu);
2257 			break;
2258 #endif /* INET6 */
2259 		default:
2260 			panic("%s: unknown type", __func__);
2261 		}
2262 		free(pfse, M_PFTEMP);
2263 	}
2264 	NET_EPOCH_EXIT(et);
2265 	CURVNET_RESTORE();
2266 }
2267 
2268 #define	pf_purge_thread_period	(hz / 10)
2269 
2270 #ifdef PF_WANT_32_TO_64_COUNTER
2271 static void
2272 pf_status_counter_u64_periodic(void)
2273 {
2274 
2275 	PF_RULES_RASSERT();
2276 
2277 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 60)) != 0) {
2278 		return;
2279 	}
2280 
2281 	for (int i = 0; i < FCNT_MAX; i++) {
2282 		pf_counter_u64_periodic(&V_pf_status.fcounters[i]);
2283 	}
2284 }
2285 
2286 static void
2287 pf_kif_counter_u64_periodic(void)
2288 {
2289 	struct pfi_kkif *kif;
2290 	size_t r, run;
2291 
2292 	PF_RULES_RASSERT();
2293 
2294 	if (__predict_false(V_pf_allkifcount == 0)) {
2295 		return;
2296 	}
2297 
2298 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
2299 		return;
2300 	}
2301 
2302 	run = V_pf_allkifcount / 10;
2303 	if (run < 5)
2304 		run = 5;
2305 
2306 	for (r = 0; r < run; r++) {
2307 		kif = LIST_NEXT(V_pf_kifmarker, pfik_allkiflist);
2308 		if (kif == NULL) {
2309 			LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
2310 			LIST_INSERT_HEAD(&V_pf_allkiflist, V_pf_kifmarker, pfik_allkiflist);
2311 			break;
2312 		}
2313 
2314 		LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
2315 		LIST_INSERT_AFTER(kif, V_pf_kifmarker, pfik_allkiflist);
2316 
2317 		for (int i = 0; i < 2; i++) {
2318 			for (int j = 0; j < 2; j++) {
2319 				for (int k = 0; k < 2; k++) {
2320 					pf_counter_u64_periodic(&kif->pfik_packets[i][j][k]);
2321 					pf_counter_u64_periodic(&kif->pfik_bytes[i][j][k]);
2322 				}
2323 			}
2324 		}
2325 	}
2326 }
2327 
2328 static void
2329 pf_rule_counter_u64_periodic(void)
2330 {
2331 	struct pf_krule *rule;
2332 	size_t r, run;
2333 
2334 	PF_RULES_RASSERT();
2335 
2336 	if (__predict_false(V_pf_allrulecount == 0)) {
2337 		return;
2338 	}
2339 
2340 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
2341 		return;
2342 	}
2343 
2344 	run = V_pf_allrulecount / 10;
2345 	if (run < 5)
2346 		run = 5;
2347 
2348 	for (r = 0; r < run; r++) {
2349 		rule = LIST_NEXT(V_pf_rulemarker, allrulelist);
2350 		if (rule == NULL) {
2351 			LIST_REMOVE(V_pf_rulemarker, allrulelist);
2352 			LIST_INSERT_HEAD(&V_pf_allrulelist, V_pf_rulemarker, allrulelist);
2353 			break;
2354 		}
2355 
2356 		LIST_REMOVE(V_pf_rulemarker, allrulelist);
2357 		LIST_INSERT_AFTER(rule, V_pf_rulemarker, allrulelist);
2358 
2359 		pf_counter_u64_periodic(&rule->evaluations);
2360 		for (int i = 0; i < 2; i++) {
2361 			pf_counter_u64_periodic(&rule->packets[i]);
2362 			pf_counter_u64_periodic(&rule->bytes[i]);
2363 		}
2364 	}
2365 }
2366 
2367 static void
2368 pf_counter_u64_periodic_main(void)
2369 {
2370 	PF_RULES_RLOCK_TRACKER;
2371 
2372 	V_pf_counter_periodic_iter++;
2373 
2374 	PF_RULES_RLOCK();
2375 	pf_counter_u64_critical_enter();
2376 	pf_status_counter_u64_periodic();
2377 	pf_kif_counter_u64_periodic();
2378 	pf_rule_counter_u64_periodic();
2379 	pf_counter_u64_critical_exit();
2380 	PF_RULES_RUNLOCK();
2381 }
2382 #else
2383 #define	pf_counter_u64_periodic_main()	do { } while (0)
2384 #endif
2385 
2386 void
2387 pf_purge_thread(void *unused __unused)
2388 {
2389 	struct epoch_tracker	 et;
2390 
2391 	VNET_ITERATOR_DECL(vnet_iter);
2392 
2393 	sx_xlock(&pf_end_lock);
2394 	while (pf_end_threads == 0) {
2395 		sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
2396 
2397 		VNET_LIST_RLOCK();
2398 		NET_EPOCH_ENTER(et);
2399 		VNET_FOREACH(vnet_iter) {
2400 			CURVNET_SET(vnet_iter);
2401 
2402 			/* Wait until V_pf_default_rule is initialized. */
2403 			if (V_pf_vnet_active == 0) {
2404 				CURVNET_RESTORE();
2405 				continue;
2406 			}
2407 
2408 			pf_counter_u64_periodic_main();
2409 
2410 			/*
2411 			 *  Process 1/interval fraction of the state
2412 			 * table every run.
2413 			 */
2414 			V_pf_purge_idx =
2415 			    pf_purge_expired_states(V_pf_purge_idx, V_pf_hashmask /
2416 			    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
2417 
2418 			/*
2419 			 * Purge other expired types every
2420 			 * PFTM_INTERVAL seconds.
2421 			 */
2422 			if (V_pf_purge_idx == 0) {
2423 				/*
2424 				 * Order is important:
2425 				 * - states and src nodes reference rules
2426 				 * - states and rules reference kifs
2427 				 */
2428 				pf_purge_expired_fragments();
2429 				pf_purge_expired_src_nodes();
2430 				pf_purge_unlinked_rules();
2431 				pfi_kkif_purge();
2432 			}
2433 			CURVNET_RESTORE();
2434 		}
2435 		NET_EPOCH_EXIT(et);
2436 		VNET_LIST_RUNLOCK();
2437 	}
2438 
2439 	pf_end_threads++;
2440 	sx_xunlock(&pf_end_lock);
2441 	kproc_exit(0);
2442 }
2443 
2444 void
2445 pf_unload_vnet_purge(void)
2446 {
2447 
2448 	/*
2449 	 * To cleanse up all kifs and rules we need
2450 	 * two runs: first one clears reference flags,
2451 	 * then pf_purge_expired_states() doesn't
2452 	 * raise them, and then second run frees.
2453 	 */
2454 	pf_purge_unlinked_rules();
2455 	pfi_kkif_purge();
2456 
2457 	/*
2458 	 * Now purge everything.
2459 	 */
2460 	pf_purge_expired_states(0, V_pf_hashmask);
2461 	pf_purge_fragments(UINT_MAX);
2462 	pf_purge_expired_src_nodes();
2463 
2464 	/*
2465 	 * Now all kifs & rules should be unreferenced,
2466 	 * thus should be successfully freed.
2467 	 */
2468 	pf_purge_unlinked_rules();
2469 	pfi_kkif_purge();
2470 }
2471 
2472 u_int32_t
2473 pf_state_expires(const struct pf_kstate *state)
2474 {
2475 	u_int32_t	timeout;
2476 	u_int32_t	start;
2477 	u_int32_t	end;
2478 	u_int32_t	states;
2479 
2480 	/* handle all PFTM_* > PFTM_MAX here */
2481 	if (state->timeout == PFTM_PURGE)
2482 		return (time_uptime);
2483 	KASSERT(state->timeout != PFTM_UNLINKED,
2484 	    ("pf_state_expires: timeout == PFTM_UNLINKED"));
2485 	KASSERT((state->timeout < PFTM_MAX),
2486 	    ("pf_state_expires: timeout > PFTM_MAX"));
2487 	timeout = state->rule.ptr->timeout[state->timeout];
2488 	if (!timeout)
2489 		timeout = V_pf_default_rule.timeout[state->timeout];
2490 	start = state->rule.ptr->timeout[PFTM_ADAPTIVE_START];
2491 	if (start && state->rule.ptr != &V_pf_default_rule) {
2492 		end = state->rule.ptr->timeout[PFTM_ADAPTIVE_END];
2493 		states = counter_u64_fetch(state->rule.ptr->states_cur);
2494 	} else {
2495 		start = V_pf_default_rule.timeout[PFTM_ADAPTIVE_START];
2496 		end = V_pf_default_rule.timeout[PFTM_ADAPTIVE_END];
2497 		states = V_pf_status.states;
2498 	}
2499 	if (end && states > start && start < end) {
2500 		if (states < end) {
2501 			timeout = (u_int64_t)timeout * (end - states) /
2502 			    (end - start);
2503 			return ((state->expire / 1000) + timeout);
2504 		}
2505 		else
2506 			return (time_uptime);
2507 	}
2508 	return ((state->expire / 1000) + timeout);
2509 }
2510 
2511 void
2512 pf_purge_expired_src_nodes(void)
2513 {
2514 	struct pf_ksrc_node_list	 freelist;
2515 	struct pf_srchash	*sh;
2516 	struct pf_ksrc_node	*cur, *next;
2517 	int i;
2518 
2519 	LIST_INIT(&freelist);
2520 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
2521 	    PF_HASHROW_LOCK(sh);
2522 	    LIST_FOREACH_SAFE(cur, &sh->nodes, entry, next)
2523 		if (cur->states == 0 && cur->expire <= time_uptime) {
2524 			pf_unlink_src_node(cur);
2525 			LIST_INSERT_HEAD(&freelist, cur, entry);
2526 		} else if (cur->rule.ptr != NULL)
2527 			cur->rule.ptr->rule_ref |= PFRULE_REFS;
2528 	    PF_HASHROW_UNLOCK(sh);
2529 	}
2530 
2531 	pf_free_src_nodes(&freelist);
2532 
2533 	V_pf_status.src_nodes = uma_zone_get_cur(V_pf_sources_z);
2534 }
2535 
2536 static void
2537 pf_src_tree_remove_state(struct pf_kstate *s)
2538 {
2539 	struct pf_ksrc_node *sn;
2540 	uint32_t timeout;
2541 
2542 	timeout = s->rule.ptr->timeout[PFTM_SRC_NODE] ?
2543 	    s->rule.ptr->timeout[PFTM_SRC_NODE] :
2544 	    V_pf_default_rule.timeout[PFTM_SRC_NODE];
2545 
2546 	if (s->src_node != NULL) {
2547 		sn = s->src_node;
2548 		PF_SRC_NODE_LOCK(sn);
2549 		if (s->src.tcp_est)
2550 			--sn->conn;
2551 		if (--sn->states == 0)
2552 			sn->expire = time_uptime + timeout;
2553 		PF_SRC_NODE_UNLOCK(sn);
2554 	}
2555 	if (s->nat_src_node != s->src_node && s->nat_src_node != NULL) {
2556 		sn = s->nat_src_node;
2557 		PF_SRC_NODE_LOCK(sn);
2558 		if (--sn->states == 0)
2559 			sn->expire = time_uptime + timeout;
2560 		PF_SRC_NODE_UNLOCK(sn);
2561 	}
2562 	s->src_node = s->nat_src_node = NULL;
2563 }
2564 
2565 /*
2566  * Unlink and potentilly free a state. Function may be
2567  * called with ID hash row locked, but always returns
2568  * unlocked, since it needs to go through key hash locking.
2569  */
2570 int
2571 pf_unlink_state(struct pf_kstate *s)
2572 {
2573 	struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)];
2574 
2575 	NET_EPOCH_ASSERT();
2576 	PF_HASHROW_ASSERT(ih);
2577 
2578 	if (s->timeout == PFTM_UNLINKED) {
2579 		/*
2580 		 * State is being processed
2581 		 * by pf_unlink_state() in
2582 		 * an other thread.
2583 		 */
2584 		PF_HASHROW_UNLOCK(ih);
2585 		return (0);	/* XXXGL: undefined actually */
2586 	}
2587 
2588 	if (s->src.state == PF_TCPS_PROXY_DST) {
2589 		/* XXX wire key the right one? */
2590 		pf_send_tcp(s->rule.ptr, s->key[PF_SK_WIRE]->af,
2591 		    &s->key[PF_SK_WIRE]->addr[1],
2592 		    &s->key[PF_SK_WIRE]->addr[0],
2593 		    s->key[PF_SK_WIRE]->port[1],
2594 		    s->key[PF_SK_WIRE]->port[0],
2595 		    s->src.seqhi, s->src.seqlo + 1,
2596 		    TH_RST|TH_ACK, 0, 0, 0, true, s->tag, 0, s->act.rtableid);
2597 	}
2598 
2599 	LIST_REMOVE(s, entry);
2600 	pf_src_tree_remove_state(s);
2601 
2602 	if (V_pfsync_delete_state_ptr != NULL)
2603 		V_pfsync_delete_state_ptr(s);
2604 
2605 	STATE_DEC_COUNTERS(s);
2606 
2607 	s->timeout = PFTM_UNLINKED;
2608 
2609 	/* Ensure we remove it from the list of halfopen states, if needed. */
2610 	if (s->key[PF_SK_STACK] != NULL &&
2611 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP)
2612 		pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
2613 
2614 	PF_HASHROW_UNLOCK(ih);
2615 
2616 	pf_detach_state(s);
2617 
2618 	pf_udp_mapping_release(s->udp_mapping);
2619 
2620 	/* pf_state_insert() initialises refs to 2 */
2621 	return (pf_release_staten(s, 2));
2622 }
2623 
2624 struct pf_kstate *
2625 pf_alloc_state(int flags)
2626 {
2627 
2628 	return (uma_zalloc(V_pf_state_z, flags | M_ZERO));
2629 }
2630 
2631 void
2632 pf_free_state(struct pf_kstate *cur)
2633 {
2634 	struct pf_krule_item *ri;
2635 
2636 	KASSERT(cur->refs == 0, ("%s: %p has refs", __func__, cur));
2637 	KASSERT(cur->timeout == PFTM_UNLINKED, ("%s: timeout %u", __func__,
2638 	    cur->timeout));
2639 
2640 	while ((ri = SLIST_FIRST(&cur->match_rules))) {
2641 		SLIST_REMOVE_HEAD(&cur->match_rules, entry);
2642 		free(ri, M_PF_RULE_ITEM);
2643 	}
2644 
2645 	pf_normalize_tcp_cleanup(cur);
2646 	uma_zfree(V_pf_state_z, cur);
2647 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_REMOVALS], 1);
2648 }
2649 
2650 /*
2651  * Called only from pf_purge_thread(), thus serialized.
2652  */
2653 static u_int
2654 pf_purge_expired_states(u_int i, int maxcheck)
2655 {
2656 	struct pf_idhash *ih;
2657 	struct pf_kstate *s;
2658 	struct pf_krule_item *mrm;
2659 	size_t count __unused;
2660 
2661 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2662 
2663 	/*
2664 	 * Go through hash and unlink states that expire now.
2665 	 */
2666 	while (maxcheck > 0) {
2667 		count = 0;
2668 		ih = &V_pf_idhash[i];
2669 
2670 		/* only take the lock if we expect to do work */
2671 		if (!LIST_EMPTY(&ih->states)) {
2672 relock:
2673 			PF_HASHROW_LOCK(ih);
2674 			LIST_FOREACH(s, &ih->states, entry) {
2675 				if (pf_state_expires(s) <= time_uptime) {
2676 					V_pf_status.states -=
2677 					    pf_unlink_state(s);
2678 					goto relock;
2679 				}
2680 				s->rule.ptr->rule_ref |= PFRULE_REFS;
2681 				if (s->nat_rule.ptr != NULL)
2682 					s->nat_rule.ptr->rule_ref |= PFRULE_REFS;
2683 				if (s->anchor.ptr != NULL)
2684 					s->anchor.ptr->rule_ref |= PFRULE_REFS;
2685 				s->kif->pfik_flags |= PFI_IFLAG_REFS;
2686 				SLIST_FOREACH(mrm, &s->match_rules, entry)
2687 					mrm->r->rule_ref |= PFRULE_REFS;
2688 				if (s->rt_kif)
2689 					s->rt_kif->pfik_flags |= PFI_IFLAG_REFS;
2690 				count++;
2691 			}
2692 			PF_HASHROW_UNLOCK(ih);
2693 		}
2694 
2695 		SDT_PROBE2(pf, purge, state, rowcount, i, count);
2696 
2697 		/* Return when we hit end of hash. */
2698 		if (++i > V_pf_hashmask) {
2699 			V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2700 			return (0);
2701 		}
2702 
2703 		maxcheck--;
2704 	}
2705 
2706 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2707 
2708 	return (i);
2709 }
2710 
2711 static void
2712 pf_purge_unlinked_rules(void)
2713 {
2714 	struct pf_krulequeue tmpq;
2715 	struct pf_krule *r, *r1;
2716 
2717 	/*
2718 	 * If we have overloading task pending, then we'd
2719 	 * better skip purging this time. There is a tiny
2720 	 * probability that overloading task references
2721 	 * an already unlinked rule.
2722 	 */
2723 	PF_OVERLOADQ_LOCK();
2724 	if (!SLIST_EMPTY(&V_pf_overloadqueue)) {
2725 		PF_OVERLOADQ_UNLOCK();
2726 		return;
2727 	}
2728 	PF_OVERLOADQ_UNLOCK();
2729 
2730 	/*
2731 	 * Do naive mark-and-sweep garbage collecting of old rules.
2732 	 * Reference flag is raised by pf_purge_expired_states()
2733 	 * and pf_purge_expired_src_nodes().
2734 	 *
2735 	 * To avoid LOR between PF_UNLNKDRULES_LOCK/PF_RULES_WLOCK,
2736 	 * use a temporary queue.
2737 	 */
2738 	TAILQ_INIT(&tmpq);
2739 	PF_UNLNKDRULES_LOCK();
2740 	TAILQ_FOREACH_SAFE(r, &V_pf_unlinked_rules, entries, r1) {
2741 		if (!(r->rule_ref & PFRULE_REFS)) {
2742 			TAILQ_REMOVE(&V_pf_unlinked_rules, r, entries);
2743 			TAILQ_INSERT_TAIL(&tmpq, r, entries);
2744 		} else
2745 			r->rule_ref &= ~PFRULE_REFS;
2746 	}
2747 	PF_UNLNKDRULES_UNLOCK();
2748 
2749 	if (!TAILQ_EMPTY(&tmpq)) {
2750 		PF_CONFIG_LOCK();
2751 		PF_RULES_WLOCK();
2752 		TAILQ_FOREACH_SAFE(r, &tmpq, entries, r1) {
2753 			TAILQ_REMOVE(&tmpq, r, entries);
2754 			pf_free_rule(r);
2755 		}
2756 		PF_RULES_WUNLOCK();
2757 		PF_CONFIG_UNLOCK();
2758 	}
2759 }
2760 
2761 void
2762 pf_print_host(struct pf_addr *addr, u_int16_t p, sa_family_t af)
2763 {
2764 	switch (af) {
2765 #ifdef INET
2766 	case AF_INET: {
2767 		u_int32_t a = ntohl(addr->addr32[0]);
2768 		printf("%u.%u.%u.%u", (a>>24)&255, (a>>16)&255,
2769 		    (a>>8)&255, a&255);
2770 		if (p) {
2771 			p = ntohs(p);
2772 			printf(":%u", p);
2773 		}
2774 		break;
2775 	}
2776 #endif /* INET */
2777 #ifdef INET6
2778 	case AF_INET6: {
2779 		u_int16_t b;
2780 		u_int8_t i, curstart, curend, maxstart, maxend;
2781 		curstart = curend = maxstart = maxend = 255;
2782 		for (i = 0; i < 8; i++) {
2783 			if (!addr->addr16[i]) {
2784 				if (curstart == 255)
2785 					curstart = i;
2786 				curend = i;
2787 			} else {
2788 				if ((curend - curstart) >
2789 				    (maxend - maxstart)) {
2790 					maxstart = curstart;
2791 					maxend = curend;
2792 				}
2793 				curstart = curend = 255;
2794 			}
2795 		}
2796 		if ((curend - curstart) >
2797 		    (maxend - maxstart)) {
2798 			maxstart = curstart;
2799 			maxend = curend;
2800 		}
2801 		for (i = 0; i < 8; i++) {
2802 			if (i >= maxstart && i <= maxend) {
2803 				if (i == 0)
2804 					printf(":");
2805 				if (i == maxend)
2806 					printf(":");
2807 			} else {
2808 				b = ntohs(addr->addr16[i]);
2809 				printf("%x", b);
2810 				if (i < 7)
2811 					printf(":");
2812 			}
2813 		}
2814 		if (p) {
2815 			p = ntohs(p);
2816 			printf("[%u]", p);
2817 		}
2818 		break;
2819 	}
2820 #endif /* INET6 */
2821 	}
2822 }
2823 
2824 void
2825 pf_print_state(struct pf_kstate *s)
2826 {
2827 	pf_print_state_parts(s, NULL, NULL);
2828 }
2829 
2830 static void
2831 pf_print_state_parts(struct pf_kstate *s,
2832     struct pf_state_key *skwp, struct pf_state_key *sksp)
2833 {
2834 	struct pf_state_key *skw, *sks;
2835 	u_int8_t proto, dir;
2836 
2837 	/* Do our best to fill these, but they're skipped if NULL */
2838 	skw = skwp ? skwp : (s ? s->key[PF_SK_WIRE] : NULL);
2839 	sks = sksp ? sksp : (s ? s->key[PF_SK_STACK] : NULL);
2840 	proto = skw ? skw->proto : (sks ? sks->proto : 0);
2841 	dir = s ? s->direction : 0;
2842 
2843 	switch (proto) {
2844 	case IPPROTO_IPV4:
2845 		printf("IPv4");
2846 		break;
2847 	case IPPROTO_IPV6:
2848 		printf("IPv6");
2849 		break;
2850 	case IPPROTO_TCP:
2851 		printf("TCP");
2852 		break;
2853 	case IPPROTO_UDP:
2854 		printf("UDP");
2855 		break;
2856 	case IPPROTO_ICMP:
2857 		printf("ICMP");
2858 		break;
2859 	case IPPROTO_ICMPV6:
2860 		printf("ICMPv6");
2861 		break;
2862 	default:
2863 		printf("%u", proto);
2864 		break;
2865 	}
2866 	switch (dir) {
2867 	case PF_IN:
2868 		printf(" in");
2869 		break;
2870 	case PF_OUT:
2871 		printf(" out");
2872 		break;
2873 	}
2874 	if (skw) {
2875 		printf(" wire: ");
2876 		pf_print_host(&skw->addr[0], skw->port[0], skw->af);
2877 		printf(" ");
2878 		pf_print_host(&skw->addr[1], skw->port[1], skw->af);
2879 	}
2880 	if (sks) {
2881 		printf(" stack: ");
2882 		if (sks != skw) {
2883 			pf_print_host(&sks->addr[0], sks->port[0], sks->af);
2884 			printf(" ");
2885 			pf_print_host(&sks->addr[1], sks->port[1], sks->af);
2886 		} else
2887 			printf("-");
2888 	}
2889 	if (s) {
2890 		if (proto == IPPROTO_TCP) {
2891 			printf(" [lo=%u high=%u win=%u modulator=%u",
2892 			    s->src.seqlo, s->src.seqhi,
2893 			    s->src.max_win, s->src.seqdiff);
2894 			if (s->src.wscale && s->dst.wscale)
2895 				printf(" wscale=%u",
2896 				    s->src.wscale & PF_WSCALE_MASK);
2897 			printf("]");
2898 			printf(" [lo=%u high=%u win=%u modulator=%u",
2899 			    s->dst.seqlo, s->dst.seqhi,
2900 			    s->dst.max_win, s->dst.seqdiff);
2901 			if (s->src.wscale && s->dst.wscale)
2902 				printf(" wscale=%u",
2903 				s->dst.wscale & PF_WSCALE_MASK);
2904 			printf("]");
2905 		}
2906 		printf(" %u:%u", s->src.state, s->dst.state);
2907 		if (s->rule.ptr)
2908 			printf(" @%d", s->rule.ptr->nr);
2909 	}
2910 }
2911 
2912 void
2913 pf_print_flags(u_int8_t f)
2914 {
2915 	if (f)
2916 		printf(" ");
2917 	if (f & TH_FIN)
2918 		printf("F");
2919 	if (f & TH_SYN)
2920 		printf("S");
2921 	if (f & TH_RST)
2922 		printf("R");
2923 	if (f & TH_PUSH)
2924 		printf("P");
2925 	if (f & TH_ACK)
2926 		printf("A");
2927 	if (f & TH_URG)
2928 		printf("U");
2929 	if (f & TH_ECE)
2930 		printf("E");
2931 	if (f & TH_CWR)
2932 		printf("W");
2933 }
2934 
2935 #define	PF_SET_SKIP_STEPS(i)					\
2936 	do {							\
2937 		while (head[i] != cur) {			\
2938 			head[i]->skip[i].ptr = cur;		\
2939 			head[i] = TAILQ_NEXT(head[i], entries);	\
2940 		}						\
2941 	} while (0)
2942 
2943 void
2944 pf_calc_skip_steps(struct pf_krulequeue *rules)
2945 {
2946 	struct pf_krule *cur, *prev, *head[PF_SKIP_COUNT];
2947 	int i;
2948 
2949 	cur = TAILQ_FIRST(rules);
2950 	prev = cur;
2951 	for (i = 0; i < PF_SKIP_COUNT; ++i)
2952 		head[i] = cur;
2953 	while (cur != NULL) {
2954 		if (cur->kif != prev->kif || cur->ifnot != prev->ifnot)
2955 			PF_SET_SKIP_STEPS(PF_SKIP_IFP);
2956 		if (cur->direction != prev->direction)
2957 			PF_SET_SKIP_STEPS(PF_SKIP_DIR);
2958 		if (cur->af != prev->af)
2959 			PF_SET_SKIP_STEPS(PF_SKIP_AF);
2960 		if (cur->proto != prev->proto)
2961 			PF_SET_SKIP_STEPS(PF_SKIP_PROTO);
2962 		if (cur->src.neg != prev->src.neg ||
2963 		    pf_addr_wrap_neq(&cur->src.addr, &prev->src.addr))
2964 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_ADDR);
2965 		if (cur->src.port[0] != prev->src.port[0] ||
2966 		    cur->src.port[1] != prev->src.port[1] ||
2967 		    cur->src.port_op != prev->src.port_op)
2968 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_PORT);
2969 		if (cur->dst.neg != prev->dst.neg ||
2970 		    pf_addr_wrap_neq(&cur->dst.addr, &prev->dst.addr))
2971 			PF_SET_SKIP_STEPS(PF_SKIP_DST_ADDR);
2972 		if (cur->dst.port[0] != prev->dst.port[0] ||
2973 		    cur->dst.port[1] != prev->dst.port[1] ||
2974 		    cur->dst.port_op != prev->dst.port_op)
2975 			PF_SET_SKIP_STEPS(PF_SKIP_DST_PORT);
2976 
2977 		prev = cur;
2978 		cur = TAILQ_NEXT(cur, entries);
2979 	}
2980 	for (i = 0; i < PF_SKIP_COUNT; ++i)
2981 		PF_SET_SKIP_STEPS(i);
2982 }
2983 
2984 int
2985 pf_addr_wrap_neq(struct pf_addr_wrap *aw1, struct pf_addr_wrap *aw2)
2986 {
2987 	if (aw1->type != aw2->type)
2988 		return (1);
2989 	switch (aw1->type) {
2990 	case PF_ADDR_ADDRMASK:
2991 	case PF_ADDR_RANGE:
2992 		if (PF_ANEQ(&aw1->v.a.addr, &aw2->v.a.addr, AF_INET6))
2993 			return (1);
2994 		if (PF_ANEQ(&aw1->v.a.mask, &aw2->v.a.mask, AF_INET6))
2995 			return (1);
2996 		return (0);
2997 	case PF_ADDR_DYNIFTL:
2998 		return (aw1->p.dyn->pfid_kt != aw2->p.dyn->pfid_kt);
2999 	case PF_ADDR_NOROUTE:
3000 	case PF_ADDR_URPFFAILED:
3001 		return (0);
3002 	case PF_ADDR_TABLE:
3003 		return (aw1->p.tbl != aw2->p.tbl);
3004 	default:
3005 		printf("invalid address type: %d\n", aw1->type);
3006 		return (1);
3007 	}
3008 }
3009 
3010 /**
3011  * Checksum updates are a little complicated because the checksum in the TCP/UDP
3012  * header isn't always a full checksum. In some cases (i.e. output) it's a
3013  * pseudo-header checksum, which is a partial checksum over src/dst IP
3014  * addresses, protocol number and length.
3015  *
3016  * That means we have the following cases:
3017  *  * Input or forwarding: we don't have TSO, the checksum fields are full
3018  *  	checksums, we need to update the checksum whenever we change anything.
3019  *  * Output (i.e. the checksum is a pseudo-header checksum):
3020  *  	x The field being updated is src/dst address or affects the length of
3021  *  	the packet. We need to update the pseudo-header checksum (note that this
3022  *  	checksum is not ones' complement).
3023  *  	x Some other field is being modified (e.g. src/dst port numbers): We
3024  *  	don't have to update anything.
3025  **/
3026 u_int16_t
3027 pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
3028 {
3029 	u_int32_t x;
3030 
3031 	x = cksum + old - new;
3032 	x = (x + (x >> 16)) & 0xffff;
3033 
3034 	/* optimise: eliminate a branch when not udp */
3035 	if (udp && cksum == 0x0000)
3036 		return cksum;
3037 	if (udp && x == 0x0000)
3038 		x = 0xffff;
3039 
3040 	return (u_int16_t)(x);
3041 }
3042 
3043 static void
3044 pf_patch_8(struct mbuf *m, u_int16_t *cksum, u_int8_t *f, u_int8_t v, bool hi,
3045     u_int8_t udp)
3046 {
3047 	u_int16_t old = htons(hi ? (*f << 8) : *f);
3048 	u_int16_t new = htons(hi ? ( v << 8) :  v);
3049 
3050 	if (*f == v)
3051 		return;
3052 
3053 	*f = v;
3054 
3055 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3056 		return;
3057 
3058 	*cksum = pf_cksum_fixup(*cksum, old, new, udp);
3059 }
3060 
3061 void
3062 pf_patch_16_unaligned(struct mbuf *m, u_int16_t *cksum, void *f, u_int16_t v,
3063     bool hi, u_int8_t udp)
3064 {
3065 	u_int8_t *fb = (u_int8_t *)f;
3066 	u_int8_t *vb = (u_int8_t *)&v;
3067 
3068 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
3069 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
3070 }
3071 
3072 void
3073 pf_patch_32_unaligned(struct mbuf *m, u_int16_t *cksum, void *f, u_int32_t v,
3074     bool hi, u_int8_t udp)
3075 {
3076 	u_int8_t *fb = (u_int8_t *)f;
3077 	u_int8_t *vb = (u_int8_t *)&v;
3078 
3079 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
3080 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
3081 	pf_patch_8(m, cksum, fb++, *vb++, hi, udp);
3082 	pf_patch_8(m, cksum, fb++, *vb++, !hi, udp);
3083 }
3084 
3085 u_int16_t
3086 pf_proto_cksum_fixup(struct mbuf *m, u_int16_t cksum, u_int16_t old,
3087         u_int16_t new, u_int8_t udp)
3088 {
3089 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3090 		return (cksum);
3091 
3092 	return (pf_cksum_fixup(cksum, old, new, udp));
3093 }
3094 
3095 static void
3096 pf_change_ap(struct mbuf *m, struct pf_addr *a, u_int16_t *p, u_int16_t *ic,
3097         u_int16_t *pc, struct pf_addr *an, u_int16_t pn, u_int8_t u,
3098         sa_family_t af)
3099 {
3100 	struct pf_addr	ao;
3101 	u_int16_t	po = *p;
3102 
3103 	PF_ACPY(&ao, a, af);
3104 	PF_ACPY(a, an, af);
3105 
3106 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3107 		*pc = ~*pc;
3108 
3109 	*p = pn;
3110 
3111 	switch (af) {
3112 #ifdef INET
3113 	case AF_INET:
3114 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
3115 		    ao.addr16[0], an->addr16[0], 0),
3116 		    ao.addr16[1], an->addr16[1], 0);
3117 		*p = pn;
3118 
3119 		*pc = pf_cksum_fixup(pf_cksum_fixup(*pc,
3120 		    ao.addr16[0], an->addr16[0], u),
3121 		    ao.addr16[1], an->addr16[1], u);
3122 
3123 		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
3124 		break;
3125 #endif /* INET */
3126 #ifdef INET6
3127 	case AF_INET6:
3128 		*pc = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3129 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3130 		    pf_cksum_fixup(pf_cksum_fixup(*pc,
3131 		    ao.addr16[0], an->addr16[0], u),
3132 		    ao.addr16[1], an->addr16[1], u),
3133 		    ao.addr16[2], an->addr16[2], u),
3134 		    ao.addr16[3], an->addr16[3], u),
3135 		    ao.addr16[4], an->addr16[4], u),
3136 		    ao.addr16[5], an->addr16[5], u),
3137 		    ao.addr16[6], an->addr16[6], u),
3138 		    ao.addr16[7], an->addr16[7], u);
3139 
3140 		*pc = pf_proto_cksum_fixup(m, *pc, po, pn, u);
3141 		break;
3142 #endif /* INET6 */
3143 	}
3144 
3145 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA |
3146 	    CSUM_DELAY_DATA_IPV6)) {
3147 		*pc = ~*pc;
3148 		if (! *pc)
3149 			*pc = 0xffff;
3150 	}
3151 }
3152 
3153 /* Changes a u_int32_t.  Uses a void * so there are no align restrictions */
3154 void
3155 pf_change_a(void *a, u_int16_t *c, u_int32_t an, u_int8_t u)
3156 {
3157 	u_int32_t	ao;
3158 
3159 	memcpy(&ao, a, sizeof(ao));
3160 	memcpy(a, &an, sizeof(u_int32_t));
3161 	*c = pf_cksum_fixup(pf_cksum_fixup(*c, ao / 65536, an / 65536, u),
3162 	    ao % 65536, an % 65536, u);
3163 }
3164 
3165 void
3166 pf_change_proto_a(struct mbuf *m, void *a, u_int16_t *c, u_int32_t an, u_int8_t udp)
3167 {
3168 	u_int32_t	ao;
3169 
3170 	memcpy(&ao, a, sizeof(ao));
3171 	memcpy(a, &an, sizeof(u_int32_t));
3172 
3173 	*c = pf_proto_cksum_fixup(m,
3174 	    pf_proto_cksum_fixup(m, *c, ao / 65536, an / 65536, udp),
3175 	    ao % 65536, an % 65536, udp);
3176 }
3177 
3178 #ifdef INET6
3179 static void
3180 pf_change_a6(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u)
3181 {
3182 	struct pf_addr	ao;
3183 
3184 	PF_ACPY(&ao, a, AF_INET6);
3185 	PF_ACPY(a, an, AF_INET6);
3186 
3187 	*c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3188 	    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3189 	    pf_cksum_fixup(pf_cksum_fixup(*c,
3190 	    ao.addr16[0], an->addr16[0], u),
3191 	    ao.addr16[1], an->addr16[1], u),
3192 	    ao.addr16[2], an->addr16[2], u),
3193 	    ao.addr16[3], an->addr16[3], u),
3194 	    ao.addr16[4], an->addr16[4], u),
3195 	    ao.addr16[5], an->addr16[5], u),
3196 	    ao.addr16[6], an->addr16[6], u),
3197 	    ao.addr16[7], an->addr16[7], u);
3198 }
3199 #endif /* INET6 */
3200 
3201 static void
3202 pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa,
3203     struct pf_addr *na, u_int16_t np, u_int16_t *pc, u_int16_t *h2c,
3204     u_int16_t *ic, u_int16_t *hc, u_int8_t u, sa_family_t af)
3205 {
3206 	struct pf_addr	oia, ooa;
3207 
3208 	PF_ACPY(&oia, ia, af);
3209 	if (oa)
3210 		PF_ACPY(&ooa, oa, af);
3211 
3212 	/* Change inner protocol port, fix inner protocol checksum. */
3213 	if (ip != NULL) {
3214 		u_int16_t	oip = *ip;
3215 		u_int32_t	opc;
3216 
3217 		if (pc != NULL)
3218 			opc = *pc;
3219 		*ip = np;
3220 		if (pc != NULL)
3221 			*pc = pf_cksum_fixup(*pc, oip, *ip, u);
3222 		*ic = pf_cksum_fixup(*ic, oip, *ip, 0);
3223 		if (pc != NULL)
3224 			*ic = pf_cksum_fixup(*ic, opc, *pc, 0);
3225 	}
3226 	/* Change inner ip address, fix inner ip and icmp checksums. */
3227 	PF_ACPY(ia, na, af);
3228 	switch (af) {
3229 #ifdef INET
3230 	case AF_INET: {
3231 		u_int32_t	 oh2c = *h2c;
3232 
3233 		*h2c = pf_cksum_fixup(pf_cksum_fixup(*h2c,
3234 		    oia.addr16[0], ia->addr16[0], 0),
3235 		    oia.addr16[1], ia->addr16[1], 0);
3236 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
3237 		    oia.addr16[0], ia->addr16[0], 0),
3238 		    oia.addr16[1], ia->addr16[1], 0);
3239 		*ic = pf_cksum_fixup(*ic, oh2c, *h2c, 0);
3240 		break;
3241 	}
3242 #endif /* INET */
3243 #ifdef INET6
3244 	case AF_INET6:
3245 		*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3246 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3247 		    pf_cksum_fixup(pf_cksum_fixup(*ic,
3248 		    oia.addr16[0], ia->addr16[0], u),
3249 		    oia.addr16[1], ia->addr16[1], u),
3250 		    oia.addr16[2], ia->addr16[2], u),
3251 		    oia.addr16[3], ia->addr16[3], u),
3252 		    oia.addr16[4], ia->addr16[4], u),
3253 		    oia.addr16[5], ia->addr16[5], u),
3254 		    oia.addr16[6], ia->addr16[6], u),
3255 		    oia.addr16[7], ia->addr16[7], u);
3256 		break;
3257 #endif /* INET6 */
3258 	}
3259 	/* Outer ip address, fix outer ip or icmpv6 checksum, if necessary. */
3260 	if (oa) {
3261 		PF_ACPY(oa, na, af);
3262 		switch (af) {
3263 #ifdef INET
3264 		case AF_INET:
3265 			*hc = pf_cksum_fixup(pf_cksum_fixup(*hc,
3266 			    ooa.addr16[0], oa->addr16[0], 0),
3267 			    ooa.addr16[1], oa->addr16[1], 0);
3268 			break;
3269 #endif /* INET */
3270 #ifdef INET6
3271 		case AF_INET6:
3272 			*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3273 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3274 			    pf_cksum_fixup(pf_cksum_fixup(*ic,
3275 			    ooa.addr16[0], oa->addr16[0], u),
3276 			    ooa.addr16[1], oa->addr16[1], u),
3277 			    ooa.addr16[2], oa->addr16[2], u),
3278 			    ooa.addr16[3], oa->addr16[3], u),
3279 			    ooa.addr16[4], oa->addr16[4], u),
3280 			    ooa.addr16[5], oa->addr16[5], u),
3281 			    ooa.addr16[6], oa->addr16[6], u),
3282 			    ooa.addr16[7], oa->addr16[7], u);
3283 			break;
3284 #endif /* INET6 */
3285 		}
3286 	}
3287 }
3288 
3289 /*
3290  * Need to modulate the sequence numbers in the TCP SACK option
3291  * (credits to Krzysztof Pfaff for report and patch)
3292  */
3293 static int
3294 pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd,
3295     struct tcphdr *th, struct pf_state_peer *dst)
3296 {
3297 	int hlen = (th->th_off << 2) - sizeof(*th), thoptlen = hlen;
3298 	u_int8_t opts[TCP_MAXOLEN], *opt = opts;
3299 	int copyback = 0, i, olen;
3300 	struct sackblk sack;
3301 
3302 #define	TCPOLEN_SACKLEN	(TCPOLEN_SACK + 2)
3303 	if (hlen < TCPOLEN_SACKLEN ||
3304 	    !pf_pull_hdr(m, off + sizeof(*th), opts, hlen, NULL, NULL, pd->af))
3305 		return 0;
3306 
3307 	while (hlen >= TCPOLEN_SACKLEN) {
3308 		size_t startoff = opt - opts;
3309 		olen = opt[1];
3310 		switch (*opt) {
3311 		case TCPOPT_EOL:	/* FALLTHROUGH */
3312 		case TCPOPT_NOP:
3313 			opt++;
3314 			hlen--;
3315 			break;
3316 		case TCPOPT_SACK:
3317 			if (olen > hlen)
3318 				olen = hlen;
3319 			if (olen >= TCPOLEN_SACKLEN) {
3320 				for (i = 2; i + TCPOLEN_SACK <= olen;
3321 				    i += TCPOLEN_SACK) {
3322 					memcpy(&sack, &opt[i], sizeof(sack));
3323 					pf_patch_32_unaligned(m,
3324 					    &th->th_sum, &sack.start,
3325 					    htonl(ntohl(sack.start) - dst->seqdiff),
3326 					    PF_ALGNMNT(startoff),
3327 					    0);
3328 					pf_patch_32_unaligned(m, &th->th_sum,
3329 					    &sack.end,
3330 					    htonl(ntohl(sack.end) - dst->seqdiff),
3331 					    PF_ALGNMNT(startoff),
3332 					    0);
3333 					memcpy(&opt[i], &sack, sizeof(sack));
3334 				}
3335 				copyback = 1;
3336 			}
3337 			/* FALLTHROUGH */
3338 		default:
3339 			if (olen < 2)
3340 				olen = 2;
3341 			hlen -= olen;
3342 			opt += olen;
3343 		}
3344 	}
3345 
3346 	if (copyback)
3347 		m_copyback(m, off + sizeof(*th), thoptlen, (caddr_t)opts);
3348 	return (copyback);
3349 }
3350 
3351 struct mbuf *
3352 pf_build_tcp(const struct pf_krule *r, sa_family_t af,
3353     const struct pf_addr *saddr, const struct pf_addr *daddr,
3354     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
3355     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
3356     bool skip_firewall, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
3357 {
3358 	struct mbuf	*m;
3359 	int		 len, tlen;
3360 #ifdef INET
3361 	struct ip	*h = NULL;
3362 #endif /* INET */
3363 #ifdef INET6
3364 	struct ip6_hdr	*h6 = NULL;
3365 #endif /* INET6 */
3366 	struct tcphdr	*th;
3367 	char		*opt;
3368 	struct pf_mtag  *pf_mtag;
3369 
3370 	len = 0;
3371 	th = NULL;
3372 
3373 	/* maximum segment size tcp option */
3374 	tlen = sizeof(struct tcphdr);
3375 	if (mss)
3376 		tlen += 4;
3377 
3378 	switch (af) {
3379 #ifdef INET
3380 	case AF_INET:
3381 		len = sizeof(struct ip) + tlen;
3382 		break;
3383 #endif /* INET */
3384 #ifdef INET6
3385 	case AF_INET6:
3386 		len = sizeof(struct ip6_hdr) + tlen;
3387 		break;
3388 #endif /* INET6 */
3389 	default:
3390 		panic("%s: unsupported af %d", __func__, af);
3391 	}
3392 
3393 	m = m_gethdr(M_NOWAIT, MT_DATA);
3394 	if (m == NULL)
3395 		return (NULL);
3396 
3397 #ifdef MAC
3398 	mac_netinet_firewall_send(m);
3399 #endif
3400 	if ((pf_mtag = pf_get_mtag(m)) == NULL) {
3401 		m_freem(m);
3402 		return (NULL);
3403 	}
3404 	if (skip_firewall)
3405 		m->m_flags |= M_SKIP_FIREWALL;
3406 	pf_mtag->tag = mtag_tag;
3407 	pf_mtag->flags = mtag_flags;
3408 
3409 	if (rtableid >= 0)
3410 		M_SETFIB(m, rtableid);
3411 
3412 #ifdef ALTQ
3413 	if (r != NULL && r->qid) {
3414 		pf_mtag->qid = r->qid;
3415 
3416 		/* add hints for ecn */
3417 		pf_mtag->hdr = mtod(m, struct ip *);
3418 	}
3419 #endif /* ALTQ */
3420 	m->m_data += max_linkhdr;
3421 	m->m_pkthdr.len = m->m_len = len;
3422 	/* The rest of the stack assumes a rcvif, so provide one.
3423 	 * This is a locally generated packet, so .. close enough. */
3424 	m->m_pkthdr.rcvif = V_loif;
3425 	bzero(m->m_data, len);
3426 	switch (af) {
3427 #ifdef INET
3428 	case AF_INET:
3429 		h = mtod(m, struct ip *);
3430 
3431 		/* IP header fields included in the TCP checksum */
3432 		h->ip_p = IPPROTO_TCP;
3433 		h->ip_len = htons(tlen);
3434 		h->ip_src.s_addr = saddr->v4.s_addr;
3435 		h->ip_dst.s_addr = daddr->v4.s_addr;
3436 
3437 		th = (struct tcphdr *)((caddr_t)h + sizeof(struct ip));
3438 		break;
3439 #endif /* INET */
3440 #ifdef INET6
3441 	case AF_INET6:
3442 		h6 = mtod(m, struct ip6_hdr *);
3443 
3444 		/* IP header fields included in the TCP checksum */
3445 		h6->ip6_nxt = IPPROTO_TCP;
3446 		h6->ip6_plen = htons(tlen);
3447 		memcpy(&h6->ip6_src, &saddr->v6, sizeof(struct in6_addr));
3448 		memcpy(&h6->ip6_dst, &daddr->v6, sizeof(struct in6_addr));
3449 
3450 		th = (struct tcphdr *)((caddr_t)h6 + sizeof(struct ip6_hdr));
3451 		break;
3452 #endif /* INET6 */
3453 	}
3454 
3455 	/* TCP header */
3456 	th->th_sport = sport;
3457 	th->th_dport = dport;
3458 	th->th_seq = htonl(seq);
3459 	th->th_ack = htonl(ack);
3460 	th->th_off = tlen >> 2;
3461 	th->th_flags = tcp_flags;
3462 	th->th_win = htons(win);
3463 
3464 	if (mss) {
3465 		opt = (char *)(th + 1);
3466 		opt[0] = TCPOPT_MAXSEG;
3467 		opt[1] = 4;
3468 		HTONS(mss);
3469 		bcopy((caddr_t)&mss, (caddr_t)(opt + 2), 2);
3470 	}
3471 
3472 	switch (af) {
3473 #ifdef INET
3474 	case AF_INET:
3475 		/* TCP checksum */
3476 		th->th_sum = in_cksum(m, len);
3477 
3478 		/* Finish the IP header */
3479 		h->ip_v = 4;
3480 		h->ip_hl = sizeof(*h) >> 2;
3481 		h->ip_tos = IPTOS_LOWDELAY;
3482 		h->ip_off = htons(V_path_mtu_discovery ? IP_DF : 0);
3483 		h->ip_len = htons(len);
3484 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
3485 		h->ip_sum = 0;
3486 		break;
3487 #endif /* INET */
3488 #ifdef INET6
3489 	case AF_INET6:
3490 		/* TCP checksum */
3491 		th->th_sum = in6_cksum(m, IPPROTO_TCP,
3492 		    sizeof(struct ip6_hdr), tlen);
3493 
3494 		h6->ip6_vfc |= IPV6_VERSION;
3495 		h6->ip6_hlim = IPV6_DEFHLIM;
3496 		break;
3497 #endif /* INET6 */
3498 	}
3499 
3500 	return (m);
3501 }
3502 
3503 static void
3504 pf_send_sctp_abort(sa_family_t af, struct pf_pdesc *pd,
3505     uint8_t ttl, int rtableid)
3506 {
3507 	struct mbuf		*m;
3508 #ifdef INET
3509 	struct ip		*h = NULL;
3510 #endif /* INET */
3511 #ifdef INET6
3512 	struct ip6_hdr		*h6 = NULL;
3513 #endif /* INET6 */
3514 	struct sctphdr		*hdr;
3515 	struct sctp_chunkhdr	*chunk;
3516 	struct pf_send_entry	*pfse;
3517 	int			 off = 0;
3518 
3519 	MPASS(af == pd->af);
3520 
3521 	m = m_gethdr(M_NOWAIT, MT_DATA);
3522 	if (m == NULL)
3523 		return;
3524 
3525 	m->m_data += max_linkhdr;
3526 	m->m_flags |= M_SKIP_FIREWALL;
3527 	/* The rest of the stack assumes a rcvif, so provide one.
3528 	 * This is a locally generated packet, so .. close enough. */
3529 	m->m_pkthdr.rcvif = V_loif;
3530 
3531 	/* IPv4|6 header */
3532 	switch (af) {
3533 #ifdef INET
3534 	case AF_INET:
3535 		bzero(m->m_data, sizeof(struct ip) + sizeof(*hdr) + sizeof(*chunk));
3536 
3537 		h = mtod(m, struct ip *);
3538 
3539 		/* IP header fields included in the TCP checksum */
3540 
3541 		h->ip_p = IPPROTO_SCTP;
3542 		h->ip_len = htons(sizeof(*h) + sizeof(*hdr) + sizeof(*chunk));
3543 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
3544 		h->ip_src = pd->dst->v4;
3545 		h->ip_dst = pd->src->v4;
3546 
3547 		off += sizeof(struct ip);
3548 		break;
3549 #endif /* INET */
3550 #ifdef INET6
3551 	case AF_INET6:
3552 		bzero(m->m_data, sizeof(struct ip6_hdr) + sizeof(*hdr) + sizeof(*chunk));
3553 
3554 		h6 = mtod(m, struct ip6_hdr *);
3555 
3556 		/* IP header fields included in the TCP checksum */
3557 		h6->ip6_vfc |= IPV6_VERSION;
3558 		h6->ip6_nxt = IPPROTO_SCTP;
3559 		h6->ip6_plen = htons(sizeof(*h6) + sizeof(*hdr) + sizeof(*chunk));
3560 		h6->ip6_hlim = ttl ? ttl : V_ip6_defhlim;
3561 		memcpy(&h6->ip6_src, &pd->dst->v6, sizeof(struct in6_addr));
3562 		memcpy(&h6->ip6_dst, &pd->src->v6, sizeof(struct in6_addr));
3563 
3564 		off += sizeof(struct ip6_hdr);
3565 		break;
3566 #endif /* INET6 */
3567 	}
3568 
3569 	/* SCTP header */
3570 	hdr = mtodo(m, off);
3571 
3572 	hdr->src_port = pd->hdr.sctp.dest_port;
3573 	hdr->dest_port = pd->hdr.sctp.src_port;
3574 	hdr->v_tag = pd->sctp_initiate_tag;
3575 	hdr->checksum = 0;
3576 
3577 	/* Abort chunk. */
3578 	off += sizeof(struct sctphdr);
3579 	chunk = mtodo(m, off);
3580 
3581 	chunk->chunk_type = SCTP_ABORT_ASSOCIATION;
3582 	chunk->chunk_length = htons(sizeof(*chunk));
3583 
3584 	/* SCTP checksum */
3585 	off += sizeof(*chunk);
3586 	m->m_pkthdr.len = m->m_len = off;
3587 
3588 	pf_sctp_checksum(m, off - sizeof(*hdr) - sizeof(*chunk));
3589 
3590 	if (rtableid >= 0)
3591 		M_SETFIB(m, rtableid);
3592 
3593 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3594 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3595 	if (pfse == NULL) {
3596 		m_freem(m);
3597 		return;
3598 	}
3599 
3600 	switch (af) {
3601 #ifdef INET
3602 	case AF_INET:
3603 		pfse->pfse_type = PFSE_IP;
3604 		break;
3605 #endif /* INET */
3606 #ifdef INET6
3607 	case AF_INET6:
3608 		pfse->pfse_type = PFSE_IP6;
3609 		break;
3610 #endif /* INET6 */
3611 	}
3612 
3613 	pfse->pfse_m = m;
3614 	pf_send(pfse);
3615 }
3616 
3617 void
3618 pf_send_tcp(const struct pf_krule *r, sa_family_t af,
3619     const struct pf_addr *saddr, const struct pf_addr *daddr,
3620     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
3621     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
3622     bool skip_firewall, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
3623 {
3624 	struct pf_send_entry *pfse;
3625 	struct mbuf	*m;
3626 
3627 	m = pf_build_tcp(r, af, saddr, daddr, sport, dport, seq, ack, tcp_flags,
3628 	    win, mss, ttl, skip_firewall, mtag_tag, mtag_flags, rtableid);
3629 	if (m == NULL)
3630 		return;
3631 
3632 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3633 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3634 	if (pfse == NULL) {
3635 		m_freem(m);
3636 		return;
3637 	}
3638 
3639 	switch (af) {
3640 #ifdef INET
3641 	case AF_INET:
3642 		pfse->pfse_type = PFSE_IP;
3643 		break;
3644 #endif /* INET */
3645 #ifdef INET6
3646 	case AF_INET6:
3647 		pfse->pfse_type = PFSE_IP6;
3648 		break;
3649 #endif /* INET6 */
3650 	}
3651 
3652 	pfse->pfse_m = m;
3653 	pf_send(pfse);
3654 }
3655 
3656 static void
3657 pf_return(struct pf_krule *r, struct pf_krule *nr, struct pf_pdesc *pd,
3658     struct pf_state_key *sk, int off, struct mbuf *m, struct tcphdr *th,
3659     struct pfi_kkif *kif, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen,
3660     u_short *reason, int rtableid)
3661 {
3662 	struct pf_addr	* const saddr = pd->src;
3663 	struct pf_addr	* const daddr = pd->dst;
3664 	sa_family_t	 af = pd->af;
3665 
3666 	/* undo NAT changes, if they have taken place */
3667 	if (nr != NULL) {
3668 		PF_ACPY(saddr, &sk->addr[pd->sidx], af);
3669 		PF_ACPY(daddr, &sk->addr[pd->didx], af);
3670 		if (pd->sport)
3671 			*pd->sport = sk->port[pd->sidx];
3672 		if (pd->dport)
3673 			*pd->dport = sk->port[pd->didx];
3674 		if (pd->proto_sum)
3675 			*pd->proto_sum = bproto_sum;
3676 		if (pd->ip_sum)
3677 			*pd->ip_sum = bip_sum;
3678 		m_copyback(m, off, hdrlen, pd->hdr.any);
3679 	}
3680 	if (pd->proto == IPPROTO_TCP &&
3681 	    ((r->rule_flag & PFRULE_RETURNRST) ||
3682 	    (r->rule_flag & PFRULE_RETURN)) &&
3683 	    !(th->th_flags & TH_RST)) {
3684 		u_int32_t	 ack = ntohl(th->th_seq) + pd->p_len;
3685 		int		 len = 0;
3686 #ifdef INET
3687 		struct ip	*h4;
3688 #endif
3689 #ifdef INET6
3690 		struct ip6_hdr	*h6;
3691 #endif
3692 
3693 		switch (af) {
3694 #ifdef INET
3695 		case AF_INET:
3696 			h4 = mtod(m, struct ip *);
3697 			len = ntohs(h4->ip_len) - off;
3698 			break;
3699 #endif
3700 #ifdef INET6
3701 		case AF_INET6:
3702 			h6 = mtod(m, struct ip6_hdr *);
3703 			len = ntohs(h6->ip6_plen) - (off - sizeof(*h6));
3704 			break;
3705 #endif
3706 		}
3707 
3708 		if (pf_check_proto_cksum(m, off, len, IPPROTO_TCP, af))
3709 			REASON_SET(reason, PFRES_PROTCKSUM);
3710 		else {
3711 			if (th->th_flags & TH_SYN)
3712 				ack++;
3713 			if (th->th_flags & TH_FIN)
3714 				ack++;
3715 			pf_send_tcp(r, af, pd->dst,
3716 				pd->src, th->th_dport, th->th_sport,
3717 				ntohl(th->th_ack), ack, TH_RST|TH_ACK, 0, 0,
3718 				r->return_ttl, true, 0, 0, rtableid);
3719 		}
3720 	} else if (pd->proto == IPPROTO_SCTP &&
3721 	    (r->rule_flag & PFRULE_RETURN)) {
3722 		pf_send_sctp_abort(af, pd, r->return_ttl, rtableid);
3723 	} else if (pd->proto != IPPROTO_ICMP && af == AF_INET &&
3724 		r->return_icmp)
3725 		pf_send_icmp(m, r->return_icmp >> 8,
3726 			r->return_icmp & 255, af, r, rtableid);
3727 	else if (pd->proto != IPPROTO_ICMPV6 && af == AF_INET6 &&
3728 		r->return_icmp6)
3729 		pf_send_icmp(m, r->return_icmp6 >> 8,
3730 			r->return_icmp6 & 255, af, r, rtableid);
3731 }
3732 
3733 static int
3734 pf_match_ieee8021q_pcp(u_int8_t prio, struct mbuf *m)
3735 {
3736 	struct m_tag *mtag;
3737 	u_int8_t mpcp;
3738 
3739 	mtag = m_tag_locate(m, MTAG_8021Q, MTAG_8021Q_PCP_IN, NULL);
3740 	if (mtag == NULL)
3741 		return (0);
3742 
3743 	if (prio == PF_PRIO_ZERO)
3744 		prio = 0;
3745 
3746 	mpcp = *(uint8_t *)(mtag + 1);
3747 
3748 	return (mpcp == prio);
3749 }
3750 
3751 static int
3752 pf_icmp_to_bandlim(uint8_t type)
3753 {
3754 	switch (type) {
3755 		case ICMP_ECHO:
3756 		case ICMP_ECHOREPLY:
3757 			return (BANDLIM_ICMP_ECHO);
3758 		case ICMP_TSTAMP:
3759 		case ICMP_TSTAMPREPLY:
3760 			return (BANDLIM_ICMP_TSTAMP);
3761 		case ICMP_UNREACH:
3762 		default:
3763 			return (BANDLIM_ICMP_UNREACH);
3764 	}
3765 }
3766 
3767 static void
3768 pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af,
3769     struct pf_krule *r, int rtableid)
3770 {
3771 	struct pf_send_entry *pfse;
3772 	struct mbuf *m0;
3773 	struct pf_mtag *pf_mtag;
3774 
3775 	/* ICMP packet rate limitation. */
3776 #ifdef INET6
3777 	if (af == AF_INET6) {
3778 		if (icmp6_ratelimit(NULL, type, code))
3779 			return;
3780 	}
3781 #endif
3782 #ifdef INET
3783 	if (af == AF_INET) {
3784 		if (badport_bandlim(pf_icmp_to_bandlim(type)) != 0)
3785 			return;
3786 	}
3787 #endif
3788 
3789 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
3790 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
3791 	if (pfse == NULL)
3792 		return;
3793 
3794 	if ((m0 = m_copypacket(m, M_NOWAIT)) == NULL) {
3795 		free(pfse, M_PFTEMP);
3796 		return;
3797 	}
3798 
3799 	if ((pf_mtag = pf_get_mtag(m0)) == NULL) {
3800 		free(pfse, M_PFTEMP);
3801 		return;
3802 	}
3803 	/* XXX: revisit */
3804 	m0->m_flags |= M_SKIP_FIREWALL;
3805 
3806 	if (rtableid >= 0)
3807 		M_SETFIB(m0, rtableid);
3808 
3809 #ifdef ALTQ
3810 	if (r->qid) {
3811 		pf_mtag->qid = r->qid;
3812 		/* add hints for ecn */
3813 		pf_mtag->hdr = mtod(m0, struct ip *);
3814 	}
3815 #endif /* ALTQ */
3816 
3817 	switch (af) {
3818 #ifdef INET
3819 	case AF_INET:
3820 		pfse->pfse_type = PFSE_ICMP;
3821 		break;
3822 #endif /* INET */
3823 #ifdef INET6
3824 	case AF_INET6:
3825 		pfse->pfse_type = PFSE_ICMP6;
3826 		break;
3827 #endif /* INET6 */
3828 	}
3829 	pfse->pfse_m = m0;
3830 	pfse->icmpopts.type = type;
3831 	pfse->icmpopts.code = code;
3832 	pf_send(pfse);
3833 }
3834 
3835 /*
3836  * Return 1 if the addresses a and b match (with mask m), otherwise return 0.
3837  * If n is 0, they match if they are equal. If n is != 0, they match if they
3838  * are different.
3839  */
3840 int
3841 pf_match_addr(u_int8_t n, struct pf_addr *a, struct pf_addr *m,
3842     struct pf_addr *b, sa_family_t af)
3843 {
3844 	int	match = 0;
3845 
3846 	switch (af) {
3847 #ifdef INET
3848 	case AF_INET:
3849 		if (IN_ARE_MASKED_ADDR_EQUAL(a->v4, b->v4, m->v4))
3850 			match++;
3851 		break;
3852 #endif /* INET */
3853 #ifdef INET6
3854 	case AF_INET6:
3855 		if (IN6_ARE_MASKED_ADDR_EQUAL(&a->v6, &b->v6, &m->v6))
3856 			match++;
3857 		break;
3858 #endif /* INET6 */
3859 	}
3860 	if (match) {
3861 		if (n)
3862 			return (0);
3863 		else
3864 			return (1);
3865 	} else {
3866 		if (n)
3867 			return (1);
3868 		else
3869 			return (0);
3870 	}
3871 }
3872 
3873 /*
3874  * Return 1 if b <= a <= e, otherwise return 0.
3875  */
3876 int
3877 pf_match_addr_range(struct pf_addr *b, struct pf_addr *e,
3878     struct pf_addr *a, sa_family_t af)
3879 {
3880 	switch (af) {
3881 #ifdef INET
3882 	case AF_INET:
3883 		if ((ntohl(a->addr32[0]) < ntohl(b->addr32[0])) ||
3884 		    (ntohl(a->addr32[0]) > ntohl(e->addr32[0])))
3885 			return (0);
3886 		break;
3887 #endif /* INET */
3888 #ifdef INET6
3889 	case AF_INET6: {
3890 		int	i;
3891 
3892 		/* check a >= b */
3893 		for (i = 0; i < 4; ++i)
3894 			if (ntohl(a->addr32[i]) > ntohl(b->addr32[i]))
3895 				break;
3896 			else if (ntohl(a->addr32[i]) < ntohl(b->addr32[i]))
3897 				return (0);
3898 		/* check a <= e */
3899 		for (i = 0; i < 4; ++i)
3900 			if (ntohl(a->addr32[i]) < ntohl(e->addr32[i]))
3901 				break;
3902 			else if (ntohl(a->addr32[i]) > ntohl(e->addr32[i]))
3903 				return (0);
3904 		break;
3905 	}
3906 #endif /* INET6 */
3907 	}
3908 	return (1);
3909 }
3910 
3911 static int
3912 pf_match(u_int8_t op, u_int32_t a1, u_int32_t a2, u_int32_t p)
3913 {
3914 	switch (op) {
3915 	case PF_OP_IRG:
3916 		return ((p > a1) && (p < a2));
3917 	case PF_OP_XRG:
3918 		return ((p < a1) || (p > a2));
3919 	case PF_OP_RRG:
3920 		return ((p >= a1) && (p <= a2));
3921 	case PF_OP_EQ:
3922 		return (p == a1);
3923 	case PF_OP_NE:
3924 		return (p != a1);
3925 	case PF_OP_LT:
3926 		return (p < a1);
3927 	case PF_OP_LE:
3928 		return (p <= a1);
3929 	case PF_OP_GT:
3930 		return (p > a1);
3931 	case PF_OP_GE:
3932 		return (p >= a1);
3933 	}
3934 	return (0); /* never reached */
3935 }
3936 
3937 int
3938 pf_match_port(u_int8_t op, u_int16_t a1, u_int16_t a2, u_int16_t p)
3939 {
3940 	NTOHS(a1);
3941 	NTOHS(a2);
3942 	NTOHS(p);
3943 	return (pf_match(op, a1, a2, p));
3944 }
3945 
3946 static int
3947 pf_match_uid(u_int8_t op, uid_t a1, uid_t a2, uid_t u)
3948 {
3949 	if (u == UID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
3950 		return (0);
3951 	return (pf_match(op, a1, a2, u));
3952 }
3953 
3954 static int
3955 pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g)
3956 {
3957 	if (g == GID_MAX && op != PF_OP_EQ && op != PF_OP_NE)
3958 		return (0);
3959 	return (pf_match(op, a1, a2, g));
3960 }
3961 
3962 int
3963 pf_match_tag(struct mbuf *m, struct pf_krule *r, int *tag, int mtag)
3964 {
3965 	if (*tag == -1)
3966 		*tag = mtag;
3967 
3968 	return ((!r->match_tag_not && r->match_tag == *tag) ||
3969 	    (r->match_tag_not && r->match_tag != *tag));
3970 }
3971 
3972 static int
3973 pf_match_rcvif(struct mbuf *m, struct pf_krule *r)
3974 {
3975 	struct ifnet *ifp = m->m_pkthdr.rcvif;
3976 	struct pfi_kkif *kif;
3977 
3978 	if (ifp == NULL)
3979 		return (0);
3980 
3981 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
3982 
3983 	if (kif == NULL) {
3984 		DPFPRINTF(PF_DEBUG_URGENT,
3985 		    ("pf_test_via: kif == NULL, @%d via %s\n", r->nr,
3986 			r->rcv_ifname));
3987 		return (0);
3988 	}
3989 
3990 	return (pfi_kkif_match(r->rcv_kif, kif));
3991 }
3992 
3993 int
3994 pf_tag_packet(struct mbuf *m, struct pf_pdesc *pd, int tag)
3995 {
3996 
3997 	KASSERT(tag > 0, ("%s: tag %d", __func__, tag));
3998 
3999 	if (pd->pf_mtag == NULL && ((pd->pf_mtag = pf_get_mtag(m)) == NULL))
4000 		return (ENOMEM);
4001 
4002 	pd->pf_mtag->tag = tag;
4003 
4004 	return (0);
4005 }
4006 
4007 #define	PF_ANCHOR_STACKSIZE	32
4008 struct pf_kanchor_stackframe {
4009 	struct pf_kruleset	*rs;
4010 	struct pf_krule		*r;	/* XXX: + match bit */
4011 	struct pf_kanchor	*child;
4012 };
4013 
4014 /*
4015  * XXX: We rely on malloc(9) returning pointer aligned addresses.
4016  */
4017 #define	PF_ANCHORSTACK_MATCH	0x00000001
4018 #define	PF_ANCHORSTACK_MASK	(PF_ANCHORSTACK_MATCH)
4019 
4020 #define	PF_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
4021 #define	PF_ANCHOR_RULE(f)	(struct pf_krule *)			\
4022 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
4023 #define	PF_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 			\
4024 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
4025 } while (0)
4026 
4027 void
4028 pf_step_into_anchor(struct pf_kanchor_stackframe *stack, int *depth,
4029     struct pf_kruleset **rs, int n, struct pf_krule **r, struct pf_krule **a,
4030     int *match)
4031 {
4032 	struct pf_kanchor_stackframe	*f;
4033 
4034 	PF_RULES_RASSERT();
4035 
4036 	if (match)
4037 		*match = 0;
4038 	if (*depth >= PF_ANCHOR_STACKSIZE) {
4039 		printf("%s: anchor stack overflow on %s\n",
4040 		    __func__, (*r)->anchor->name);
4041 		*r = TAILQ_NEXT(*r, entries);
4042 		return;
4043 	} else if (*depth == 0 && a != NULL)
4044 		*a = *r;
4045 	f = stack + (*depth)++;
4046 	f->rs = *rs;
4047 	f->r = *r;
4048 	if ((*r)->anchor_wildcard) {
4049 		struct pf_kanchor_node *parent = &(*r)->anchor->children;
4050 
4051 		if ((f->child = RB_MIN(pf_kanchor_node, parent)) == NULL) {
4052 			*r = NULL;
4053 			return;
4054 		}
4055 		*rs = &f->child->ruleset;
4056 	} else {
4057 		f->child = NULL;
4058 		*rs = &(*r)->anchor->ruleset;
4059 	}
4060 	*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
4061 }
4062 
4063 int
4064 pf_step_out_of_anchor(struct pf_kanchor_stackframe *stack, int *depth,
4065     struct pf_kruleset **rs, int n, struct pf_krule **r, struct pf_krule **a,
4066     int *match)
4067 {
4068 	struct pf_kanchor_stackframe	*f;
4069 	struct pf_krule *fr;
4070 	int quick = 0;
4071 
4072 	PF_RULES_RASSERT();
4073 
4074 	do {
4075 		if (*depth <= 0)
4076 			break;
4077 		f = stack + *depth - 1;
4078 		fr = PF_ANCHOR_RULE(f);
4079 		if (f->child != NULL) {
4080 			/*
4081 			 * This block traverses through
4082 			 * a wildcard anchor.
4083 			 */
4084 			if (match != NULL && *match) {
4085 				/*
4086 				 * If any of "*" matched, then
4087 				 * "foo/ *" matched, mark frame
4088 				 * appropriately.
4089 				 */
4090 				PF_ANCHOR_SET_MATCH(f);
4091 				*match = 0;
4092 			}
4093 			f->child = RB_NEXT(pf_kanchor_node,
4094 			    &fr->anchor->children, f->child);
4095 			if (f->child != NULL) {
4096 				*rs = &f->child->ruleset;
4097 				*r = TAILQ_FIRST((*rs)->rules[n].active.ptr);
4098 				if (*r == NULL)
4099 					continue;
4100 				else
4101 					break;
4102 			}
4103 		}
4104 		(*depth)--;
4105 		if (*depth == 0 && a != NULL)
4106 			*a = NULL;
4107 		*rs = f->rs;
4108 		if (PF_ANCHOR_MATCH(f) || (match != NULL && *match))
4109 			quick = fr->quick;
4110 		*r = TAILQ_NEXT(fr, entries);
4111 	} while (*r == NULL);
4112 
4113 	return (quick);
4114 }
4115 
4116 struct pf_keth_anchor_stackframe {
4117 	struct pf_keth_ruleset	*rs;
4118 	struct pf_keth_rule	*r;	/* XXX: + match bit */
4119 	struct pf_keth_anchor	*child;
4120 };
4121 
4122 #define	PF_ETH_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
4123 #define	PF_ETH_ANCHOR_RULE(f)	(struct pf_keth_rule *)			\
4124 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
4125 #define	PF_ETH_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 		\
4126 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
4127 } while (0)
4128 
4129 void
4130 pf_step_into_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
4131     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
4132     struct pf_keth_rule **a, int *match)
4133 {
4134 	struct pf_keth_anchor_stackframe	*f;
4135 
4136 	NET_EPOCH_ASSERT();
4137 
4138 	if (match)
4139 		*match = 0;
4140 	if (*depth >= PF_ANCHOR_STACKSIZE) {
4141 		printf("%s: anchor stack overflow on %s\n",
4142 		    __func__, (*r)->anchor->name);
4143 		*r = TAILQ_NEXT(*r, entries);
4144 		return;
4145 	} else if (*depth == 0 && a != NULL)
4146 		*a = *r;
4147 	f = stack + (*depth)++;
4148 	f->rs = *rs;
4149 	f->r = *r;
4150 	if ((*r)->anchor_wildcard) {
4151 		struct pf_keth_anchor_node *parent = &(*r)->anchor->children;
4152 
4153 		if ((f->child = RB_MIN(pf_keth_anchor_node, parent)) == NULL) {
4154 			*r = NULL;
4155 			return;
4156 		}
4157 		*rs = &f->child->ruleset;
4158 	} else {
4159 		f->child = NULL;
4160 		*rs = &(*r)->anchor->ruleset;
4161 	}
4162 	*r = TAILQ_FIRST((*rs)->active.rules);
4163 }
4164 
4165 int
4166 pf_step_out_of_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
4167     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
4168     struct pf_keth_rule **a, int *match)
4169 {
4170 	struct pf_keth_anchor_stackframe	*f;
4171 	struct pf_keth_rule *fr;
4172 	int quick = 0;
4173 
4174 	NET_EPOCH_ASSERT();
4175 
4176 	do {
4177 		if (*depth <= 0)
4178 			break;
4179 		f = stack + *depth - 1;
4180 		fr = PF_ETH_ANCHOR_RULE(f);
4181 		if (f->child != NULL) {
4182 			/*
4183 			 * This block traverses through
4184 			 * a wildcard anchor.
4185 			 */
4186 			if (match != NULL && *match) {
4187 				/*
4188 				 * If any of "*" matched, then
4189 				 * "foo/ *" matched, mark frame
4190 				 * appropriately.
4191 				 */
4192 				PF_ETH_ANCHOR_SET_MATCH(f);
4193 				*match = 0;
4194 			}
4195 			f->child = RB_NEXT(pf_keth_anchor_node,
4196 			    &fr->anchor->children, f->child);
4197 			if (f->child != NULL) {
4198 				*rs = &f->child->ruleset;
4199 				*r = TAILQ_FIRST((*rs)->active.rules);
4200 				if (*r == NULL)
4201 					continue;
4202 				else
4203 					break;
4204 			}
4205 		}
4206 		(*depth)--;
4207 		if (*depth == 0 && a != NULL)
4208 			*a = NULL;
4209 		*rs = f->rs;
4210 		if (PF_ETH_ANCHOR_MATCH(f) || (match != NULL && *match))
4211 			quick = fr->quick;
4212 		*r = TAILQ_NEXT(fr, entries);
4213 	} while (*r == NULL);
4214 
4215 	return (quick);
4216 }
4217 
4218 #ifdef INET6
4219 void
4220 pf_poolmask(struct pf_addr *naddr, struct pf_addr *raddr,
4221     struct pf_addr *rmask, struct pf_addr *saddr, sa_family_t af)
4222 {
4223 	switch (af) {
4224 #ifdef INET
4225 	case AF_INET:
4226 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
4227 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
4228 		break;
4229 #endif /* INET */
4230 	case AF_INET6:
4231 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
4232 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
4233 		naddr->addr32[1] = (raddr->addr32[1] & rmask->addr32[1]) |
4234 		((rmask->addr32[1] ^ 0xffffffff ) & saddr->addr32[1]);
4235 		naddr->addr32[2] = (raddr->addr32[2] & rmask->addr32[2]) |
4236 		((rmask->addr32[2] ^ 0xffffffff ) & saddr->addr32[2]);
4237 		naddr->addr32[3] = (raddr->addr32[3] & rmask->addr32[3]) |
4238 		((rmask->addr32[3] ^ 0xffffffff ) & saddr->addr32[3]);
4239 		break;
4240 	}
4241 }
4242 
4243 void
4244 pf_addr_inc(struct pf_addr *addr, sa_family_t af)
4245 {
4246 	switch (af) {
4247 #ifdef INET
4248 	case AF_INET:
4249 		addr->addr32[0] = htonl(ntohl(addr->addr32[0]) + 1);
4250 		break;
4251 #endif /* INET */
4252 	case AF_INET6:
4253 		if (addr->addr32[3] == 0xffffffff) {
4254 			addr->addr32[3] = 0;
4255 			if (addr->addr32[2] == 0xffffffff) {
4256 				addr->addr32[2] = 0;
4257 				if (addr->addr32[1] == 0xffffffff) {
4258 					addr->addr32[1] = 0;
4259 					addr->addr32[0] =
4260 					    htonl(ntohl(addr->addr32[0]) + 1);
4261 				} else
4262 					addr->addr32[1] =
4263 					    htonl(ntohl(addr->addr32[1]) + 1);
4264 			} else
4265 				addr->addr32[2] =
4266 				    htonl(ntohl(addr->addr32[2]) + 1);
4267 		} else
4268 			addr->addr32[3] =
4269 			    htonl(ntohl(addr->addr32[3]) + 1);
4270 		break;
4271 	}
4272 }
4273 #endif /* INET6 */
4274 
4275 void
4276 pf_rule_to_actions(struct pf_krule *r, struct pf_rule_actions *a)
4277 {
4278 	/*
4279 	 * Modern rules use the same flags in rules as they do in states.
4280 	 */
4281 	a->flags |= (r->scrub_flags & (PFSTATE_NODF|PFSTATE_RANDOMID|
4282 	    PFSTATE_SCRUB_TCP|PFSTATE_SETPRIO));
4283 
4284 	/*
4285 	 * Old-style scrub rules have different flags which need to be translated.
4286 	 */
4287 	if (r->rule_flag & PFRULE_RANDOMID)
4288 		a->flags |= PFSTATE_RANDOMID;
4289 	if (r->scrub_flags & PFSTATE_SETTOS || r->rule_flag & PFRULE_SET_TOS ) {
4290 		a->flags |= PFSTATE_SETTOS;
4291 		a->set_tos = r->set_tos;
4292 	}
4293 
4294 	if (r->qid)
4295 		a->qid = r->qid;
4296 	if (r->pqid)
4297 		a->pqid = r->pqid;
4298 	if (r->rtableid >= 0)
4299 		a->rtableid = r->rtableid;
4300 	a->log |= r->log;
4301 	if (r->min_ttl)
4302 		a->min_ttl = r->min_ttl;
4303 	if (r->max_mss)
4304 		a->max_mss = r->max_mss;
4305 	if (r->dnpipe)
4306 		a->dnpipe = r->dnpipe;
4307 	if (r->dnrpipe)
4308 		a->dnrpipe = r->dnrpipe;
4309 	if (r->dnpipe || r->dnrpipe) {
4310 		if (r->free_flags & PFRULE_DN_IS_PIPE)
4311 			a->flags |= PFSTATE_DN_IS_PIPE;
4312 		else
4313 			a->flags &= ~PFSTATE_DN_IS_PIPE;
4314 	}
4315 	if (r->scrub_flags & PFSTATE_SETPRIO) {
4316 		a->set_prio[0] = r->set_prio[0];
4317 		a->set_prio[1] = r->set_prio[1];
4318 	}
4319 }
4320 
4321 int
4322 pf_socket_lookup(struct pf_pdesc *pd, struct mbuf *m)
4323 {
4324 	struct pf_addr		*saddr, *daddr;
4325 	u_int16_t		 sport, dport;
4326 	struct inpcbinfo	*pi;
4327 	struct inpcb		*inp;
4328 
4329 	pd->lookup.uid = UID_MAX;
4330 	pd->lookup.gid = GID_MAX;
4331 
4332 	switch (pd->proto) {
4333 	case IPPROTO_TCP:
4334 		sport = pd->hdr.tcp.th_sport;
4335 		dport = pd->hdr.tcp.th_dport;
4336 		pi = &V_tcbinfo;
4337 		break;
4338 	case IPPROTO_UDP:
4339 		sport = pd->hdr.udp.uh_sport;
4340 		dport = pd->hdr.udp.uh_dport;
4341 		pi = &V_udbinfo;
4342 		break;
4343 	default:
4344 		return (-1);
4345 	}
4346 	if (pd->dir == PF_IN) {
4347 		saddr = pd->src;
4348 		daddr = pd->dst;
4349 	} else {
4350 		u_int16_t	p;
4351 
4352 		p = sport;
4353 		sport = dport;
4354 		dport = p;
4355 		saddr = pd->dst;
4356 		daddr = pd->src;
4357 	}
4358 	switch (pd->af) {
4359 #ifdef INET
4360 	case AF_INET:
4361 		inp = in_pcblookup_mbuf(pi, saddr->v4, sport, daddr->v4,
4362 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
4363 		if (inp == NULL) {
4364 			inp = in_pcblookup_mbuf(pi, saddr->v4, sport,
4365 			   daddr->v4, dport, INPLOOKUP_WILDCARD |
4366 			   INPLOOKUP_RLOCKPCB, NULL, m);
4367 			if (inp == NULL)
4368 				return (-1);
4369 		}
4370 		break;
4371 #endif /* INET */
4372 #ifdef INET6
4373 	case AF_INET6:
4374 		inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport, &daddr->v6,
4375 		    dport, INPLOOKUP_RLOCKPCB, NULL, m);
4376 		if (inp == NULL) {
4377 			inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport,
4378 			    &daddr->v6, dport, INPLOOKUP_WILDCARD |
4379 			    INPLOOKUP_RLOCKPCB, NULL, m);
4380 			if (inp == NULL)
4381 				return (-1);
4382 		}
4383 		break;
4384 #endif /* INET6 */
4385 
4386 	default:
4387 		return (-1);
4388 	}
4389 	INP_RLOCK_ASSERT(inp);
4390 	pd->lookup.uid = inp->inp_cred->cr_uid;
4391 	pd->lookup.gid = inp->inp_cred->cr_groups[0];
4392 	INP_RUNLOCK(inp);
4393 
4394 	return (1);
4395 }
4396 
4397 u_int8_t
4398 pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
4399 {
4400 	int		 hlen;
4401 	u_int8_t	 hdr[60];
4402 	u_int8_t	*opt, optlen;
4403 	u_int8_t	 wscale = 0;
4404 
4405 	hlen = th_off << 2;		/* hlen <= sizeof(hdr) */
4406 	if (hlen <= sizeof(struct tcphdr))
4407 		return (0);
4408 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
4409 		return (0);
4410 	opt = hdr + sizeof(struct tcphdr);
4411 	hlen -= sizeof(struct tcphdr);
4412 	while (hlen >= 3) {
4413 		switch (*opt) {
4414 		case TCPOPT_EOL:
4415 		case TCPOPT_NOP:
4416 			++opt;
4417 			--hlen;
4418 			break;
4419 		case TCPOPT_WINDOW:
4420 			wscale = opt[2];
4421 			if (wscale > TCP_MAX_WINSHIFT)
4422 				wscale = TCP_MAX_WINSHIFT;
4423 			wscale |= PF_WSCALE_FLAG;
4424 			/* FALLTHROUGH */
4425 		default:
4426 			optlen = opt[1];
4427 			if (optlen < 2)
4428 				optlen = 2;
4429 			hlen -= optlen;
4430 			opt += optlen;
4431 			break;
4432 		}
4433 	}
4434 	return (wscale);
4435 }
4436 
4437 u_int16_t
4438 pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
4439 {
4440 	int		 hlen;
4441 	u_int8_t	 hdr[60];
4442 	u_int8_t	*opt, optlen;
4443 	u_int16_t	 mss = V_tcp_mssdflt;
4444 
4445 	hlen = th_off << 2;	/* hlen <= sizeof(hdr) */
4446 	if (hlen <= sizeof(struct tcphdr))
4447 		return (0);
4448 	if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af))
4449 		return (0);
4450 	opt = hdr + sizeof(struct tcphdr);
4451 	hlen -= sizeof(struct tcphdr);
4452 	while (hlen >= TCPOLEN_MAXSEG) {
4453 		switch (*opt) {
4454 		case TCPOPT_EOL:
4455 		case TCPOPT_NOP:
4456 			++opt;
4457 			--hlen;
4458 			break;
4459 		case TCPOPT_MAXSEG:
4460 			bcopy((caddr_t)(opt + 2), (caddr_t)&mss, 2);
4461 			NTOHS(mss);
4462 			/* FALLTHROUGH */
4463 		default:
4464 			optlen = opt[1];
4465 			if (optlen < 2)
4466 				optlen = 2;
4467 			hlen -= optlen;
4468 			opt += optlen;
4469 			break;
4470 		}
4471 	}
4472 	return (mss);
4473 }
4474 
4475 static u_int16_t
4476 pf_calc_mss(struct pf_addr *addr, sa_family_t af, int rtableid, u_int16_t offer)
4477 {
4478 	struct nhop_object *nh;
4479 #ifdef INET6
4480 	struct in6_addr		dst6;
4481 	uint32_t		scopeid;
4482 #endif /* INET6 */
4483 	int			 hlen = 0;
4484 	uint16_t		 mss = 0;
4485 
4486 	NET_EPOCH_ASSERT();
4487 
4488 	switch (af) {
4489 #ifdef INET
4490 	case AF_INET:
4491 		hlen = sizeof(struct ip);
4492 		nh = fib4_lookup(rtableid, addr->v4, 0, 0, 0);
4493 		if (nh != NULL)
4494 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
4495 		break;
4496 #endif /* INET */
4497 #ifdef INET6
4498 	case AF_INET6:
4499 		hlen = sizeof(struct ip6_hdr);
4500 		in6_splitscope(&addr->v6, &dst6, &scopeid);
4501 		nh = fib6_lookup(rtableid, &dst6, scopeid, 0, 0);
4502 		if (nh != NULL)
4503 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
4504 		break;
4505 #endif /* INET6 */
4506 	}
4507 
4508 	mss = max(V_tcp_mssdflt, mss);
4509 	mss = min(mss, offer);
4510 	mss = max(mss, 64);		/* sanity - at least max opt space */
4511 	return (mss);
4512 }
4513 
4514 static u_int32_t
4515 pf_tcp_iss(struct pf_pdesc *pd)
4516 {
4517 	MD5_CTX ctx;
4518 	u_int32_t digest[4];
4519 
4520 	if (V_pf_tcp_secret_init == 0) {
4521 		arc4random_buf(&V_pf_tcp_secret, sizeof(V_pf_tcp_secret));
4522 		MD5Init(&V_pf_tcp_secret_ctx);
4523 		MD5Update(&V_pf_tcp_secret_ctx, V_pf_tcp_secret,
4524 		    sizeof(V_pf_tcp_secret));
4525 		V_pf_tcp_secret_init = 1;
4526 	}
4527 
4528 	ctx = V_pf_tcp_secret_ctx;
4529 
4530 	MD5Update(&ctx, (char *)&pd->hdr.tcp.th_sport, sizeof(u_short));
4531 	MD5Update(&ctx, (char *)&pd->hdr.tcp.th_dport, sizeof(u_short));
4532 	if (pd->af == AF_INET6) {
4533 		MD5Update(&ctx, (char *)&pd->src->v6, sizeof(struct in6_addr));
4534 		MD5Update(&ctx, (char *)&pd->dst->v6, sizeof(struct in6_addr));
4535 	} else {
4536 		MD5Update(&ctx, (char *)&pd->src->v4, sizeof(struct in_addr));
4537 		MD5Update(&ctx, (char *)&pd->dst->v4, sizeof(struct in_addr));
4538 	}
4539 	MD5Final((u_char *)digest, &ctx);
4540 	V_pf_tcp_iss_off += 4096;
4541 #define	ISN_RANDOM_INCREMENT (4096 - 1)
4542 	return (digest[0] + (arc4random() & ISN_RANDOM_INCREMENT) +
4543 	    V_pf_tcp_iss_off);
4544 #undef	ISN_RANDOM_INCREMENT
4545 }
4546 
4547 static bool
4548 pf_match_eth_addr(const uint8_t *a, const struct pf_keth_rule_addr *r)
4549 {
4550 	bool match = true;
4551 
4552 	/* Always matches if not set */
4553 	if (! r->isset)
4554 		return (!r->neg);
4555 
4556 	for (int i = 0; i < ETHER_ADDR_LEN; i++) {
4557 		if ((a[i] & r->mask[i]) != (r->addr[i] & r->mask[i])) {
4558 			match = false;
4559 			break;
4560 		}
4561 	}
4562 
4563 	return (match ^ r->neg);
4564 }
4565 
4566 static int
4567 pf_match_eth_tag(struct mbuf *m, struct pf_keth_rule *r, int *tag, int mtag)
4568 {
4569 	if (*tag == -1)
4570 		*tag = mtag;
4571 
4572 	return ((!r->match_tag_not && r->match_tag == *tag) ||
4573 	    (r->match_tag_not && r->match_tag != *tag));
4574 }
4575 
4576 static void
4577 pf_bridge_to(struct ifnet *ifp, struct mbuf *m)
4578 {
4579 	/* If we don't have the interface drop the packet. */
4580 	if (ifp == NULL) {
4581 		m_freem(m);
4582 		return;
4583 	}
4584 
4585 	switch (ifp->if_type) {
4586 	case IFT_ETHER:
4587 	case IFT_XETHER:
4588 	case IFT_L2VLAN:
4589 	case IFT_BRIDGE:
4590 	case IFT_IEEE8023ADLAG:
4591 		break;
4592 	default:
4593 		m_freem(m);
4594 		return;
4595 	}
4596 
4597 	ifp->if_transmit(ifp, m);
4598 }
4599 
4600 static int
4601 pf_test_eth_rule(int dir, struct pfi_kkif *kif, struct mbuf **m0)
4602 {
4603 #ifdef INET
4604 	struct ip ip;
4605 #endif
4606 #ifdef INET6
4607 	struct ip6_hdr ip6;
4608 #endif
4609 	struct mbuf *m = *m0;
4610 	struct ether_header *e;
4611 	struct pf_keth_rule *r, *rm, *a = NULL;
4612 	struct pf_keth_ruleset *ruleset = NULL;
4613 	struct pf_mtag *mtag;
4614 	struct pf_keth_ruleq *rules;
4615 	struct pf_addr *src = NULL, *dst = NULL;
4616 	struct pfi_kkif *bridge_to;
4617 	sa_family_t af = 0;
4618 	uint16_t proto;
4619 	int asd = 0, match = 0;
4620 	int tag = -1;
4621 	uint8_t action;
4622 	struct pf_keth_anchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
4623 
4624 	MPASS(kif->pfik_ifp->if_vnet == curvnet);
4625 	NET_EPOCH_ASSERT();
4626 
4627 	PF_RULES_RLOCK_TRACKER;
4628 
4629 	SDT_PROBE3(pf, eth, test_rule, entry, dir, kif->pfik_ifp, m);
4630 
4631 	mtag = pf_find_mtag(m);
4632 	if (mtag != NULL && mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
4633 		/* Dummynet re-injects packets after they've
4634 		 * completed their delay. We've already
4635 		 * processed them, so pass unconditionally. */
4636 
4637 		/* But only once. We may see the packet multiple times (e.g.
4638 		 * PFIL_IN/PFIL_OUT). */
4639 		pf_dummynet_flag_remove(m, mtag);
4640 
4641 		return (PF_PASS);
4642 	}
4643 
4644 	ruleset = V_pf_keth;
4645 	rules = ck_pr_load_ptr(&ruleset->active.rules);
4646 	r = TAILQ_FIRST(rules);
4647 	rm = NULL;
4648 
4649 	if (__predict_false(m->m_len < sizeof(struct ether_header)) &&
4650 	    (m = *m0 = m_pullup(*m0, sizeof(struct ether_header))) == NULL) {
4651 		DPFPRINTF(PF_DEBUG_URGENT,
4652 		    ("pf_test_eth_rule: m_len < sizeof(struct ether_header)"
4653 		     ", pullup failed\n"));
4654 		return (PF_DROP);
4655 	}
4656 	e = mtod(m, struct ether_header *);
4657 	proto = ntohs(e->ether_type);
4658 
4659 	switch (proto) {
4660 #ifdef INET
4661 	case ETHERTYPE_IP: {
4662 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
4663 		    sizeof(ip)))
4664 			return (PF_DROP);
4665 
4666 		af = AF_INET;
4667 		m_copydata(m, sizeof(struct ether_header), sizeof(ip),
4668 		    (caddr_t)&ip);
4669 		src = (struct pf_addr *)&ip.ip_src;
4670 		dst = (struct pf_addr *)&ip.ip_dst;
4671 		break;
4672 	}
4673 #endif /* INET */
4674 #ifdef INET6
4675 	case ETHERTYPE_IPV6: {
4676 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
4677 		    sizeof(ip6)))
4678 			return (PF_DROP);
4679 
4680 		af = AF_INET6;
4681 		m_copydata(m, sizeof(struct ether_header), sizeof(ip6),
4682 		    (caddr_t)&ip6);
4683 		src = (struct pf_addr *)&ip6.ip6_src;
4684 		dst = (struct pf_addr *)&ip6.ip6_dst;
4685 		break;
4686 	}
4687 #endif /* INET6 */
4688 	}
4689 
4690 	PF_RULES_RLOCK();
4691 
4692 	while (r != NULL) {
4693 		counter_u64_add(r->evaluations, 1);
4694 		SDT_PROBE2(pf, eth, test_rule, test, r->nr, r);
4695 
4696 		if (pfi_kkif_match(r->kif, kif) == r->ifnot) {
4697 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4698 			    "kif");
4699 			r = r->skip[PFE_SKIP_IFP].ptr;
4700 		}
4701 		else if (r->direction && r->direction != dir) {
4702 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4703 			    "dir");
4704 			r = r->skip[PFE_SKIP_DIR].ptr;
4705 		}
4706 		else if (r->proto && r->proto != proto) {
4707 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4708 			    "proto");
4709 			r = r->skip[PFE_SKIP_PROTO].ptr;
4710 		}
4711 		else if (! pf_match_eth_addr(e->ether_shost, &r->src)) {
4712 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4713 			    "src");
4714 			r = r->skip[PFE_SKIP_SRC_ADDR].ptr;
4715 		}
4716 		else if (! pf_match_eth_addr(e->ether_dhost, &r->dst)) {
4717 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4718 			    "dst");
4719 			r = r->skip[PFE_SKIP_DST_ADDR].ptr;
4720 		}
4721 		else if (src != NULL && PF_MISMATCHAW(&r->ipsrc.addr, src, af,
4722 		    r->ipsrc.neg, kif, M_GETFIB(m))) {
4723 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4724 			    "ip_src");
4725 			r = r->skip[PFE_SKIP_SRC_IP_ADDR].ptr;
4726 		}
4727 		else if (dst != NULL && PF_MISMATCHAW(&r->ipdst.addr, dst, af,
4728 		    r->ipdst.neg, kif, M_GETFIB(m))) {
4729 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4730 			    "ip_dst");
4731 			r = r->skip[PFE_SKIP_DST_IP_ADDR].ptr;
4732 		}
4733 		else if (r->match_tag && !pf_match_eth_tag(m, r, &tag,
4734 		    mtag ? mtag->tag : 0)) {
4735 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
4736 			    "match_tag");
4737 			r = TAILQ_NEXT(r, entries);
4738 		}
4739 		else {
4740 			if (r->tag)
4741 				tag = r->tag;
4742 			if (r->anchor == NULL) {
4743 				/* Rule matches */
4744 				rm = r;
4745 
4746 				SDT_PROBE2(pf, eth, test_rule, match, r->nr, r);
4747 
4748 				if (r->quick)
4749 					break;
4750 
4751 				r = TAILQ_NEXT(r, entries);
4752 			} else {
4753 				pf_step_into_keth_anchor(anchor_stack, &asd,
4754 				    &ruleset, &r, &a, &match);
4755 			}
4756 		}
4757 		if (r == NULL && pf_step_out_of_keth_anchor(anchor_stack, &asd,
4758 		    &ruleset, &r, &a, &match))
4759 			break;
4760 	}
4761 
4762 	r = rm;
4763 
4764 	SDT_PROBE2(pf, eth, test_rule, final_match, (r != NULL ? r->nr : -1), r);
4765 
4766 	/* Default to pass. */
4767 	if (r == NULL) {
4768 		PF_RULES_RUNLOCK();
4769 		return (PF_PASS);
4770 	}
4771 
4772 	/* Execute action. */
4773 	counter_u64_add(r->packets[dir == PF_OUT], 1);
4774 	counter_u64_add(r->bytes[dir == PF_OUT], m_length(m, NULL));
4775 	pf_update_timestamp(r);
4776 
4777 	/* Shortcut. Don't tag if we're just going to drop anyway. */
4778 	if (r->action == PF_DROP) {
4779 		PF_RULES_RUNLOCK();
4780 		return (PF_DROP);
4781 	}
4782 
4783 	if (tag > 0) {
4784 		if (mtag == NULL)
4785 			mtag = pf_get_mtag(m);
4786 		if (mtag == NULL) {
4787 			PF_RULES_RUNLOCK();
4788 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4789 			return (PF_DROP);
4790 		}
4791 		mtag->tag = tag;
4792 	}
4793 
4794 	if (r->qid != 0) {
4795 		if (mtag == NULL)
4796 			mtag = pf_get_mtag(m);
4797 		if (mtag == NULL) {
4798 			PF_RULES_RUNLOCK();
4799 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4800 			return (PF_DROP);
4801 		}
4802 		mtag->qid = r->qid;
4803 	}
4804 
4805 	action = r->action;
4806 	bridge_to = r->bridge_to;
4807 
4808 	/* Dummynet */
4809 	if (r->dnpipe) {
4810 		struct ip_fw_args dnflow;
4811 
4812 		/* Drop packet if dummynet is not loaded. */
4813 		if (ip_dn_io_ptr == NULL) {
4814 			PF_RULES_RUNLOCK();
4815 			m_freem(m);
4816 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4817 			return (PF_DROP);
4818 		}
4819 		if (mtag == NULL)
4820 			mtag = pf_get_mtag(m);
4821 		if (mtag == NULL) {
4822 			PF_RULES_RUNLOCK();
4823 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
4824 			return (PF_DROP);
4825 		}
4826 
4827 		bzero(&dnflow, sizeof(dnflow));
4828 
4829 		/* We don't have port numbers here, so we set 0.  That means
4830 		 * that we'll be somewhat limited in distinguishing flows (i.e.
4831 		 * only based on IP addresses, not based on port numbers), but
4832 		 * it's better than nothing. */
4833 		dnflow.f_id.dst_port = 0;
4834 		dnflow.f_id.src_port = 0;
4835 		dnflow.f_id.proto = 0;
4836 
4837 		dnflow.rule.info = r->dnpipe;
4838 		dnflow.rule.info |= IPFW_IS_DUMMYNET;
4839 		if (r->dnflags & PFRULE_DN_IS_PIPE)
4840 			dnflow.rule.info |= IPFW_IS_PIPE;
4841 
4842 		dnflow.f_id.extra = dnflow.rule.info;
4843 
4844 		dnflow.flags = dir == PF_IN ? IPFW_ARGS_IN : IPFW_ARGS_OUT;
4845 		dnflow.flags |= IPFW_ARGS_ETHER;
4846 		dnflow.ifp = kif->pfik_ifp;
4847 
4848 		switch (af) {
4849 		case AF_INET:
4850 			dnflow.f_id.addr_type = 4;
4851 			dnflow.f_id.src_ip = src->v4.s_addr;
4852 			dnflow.f_id.dst_ip = dst->v4.s_addr;
4853 			break;
4854 		case AF_INET6:
4855 			dnflow.flags |= IPFW_ARGS_IP6;
4856 			dnflow.f_id.addr_type = 6;
4857 			dnflow.f_id.src_ip6 = src->v6;
4858 			dnflow.f_id.dst_ip6 = dst->v6;
4859 			break;
4860 		}
4861 
4862 		PF_RULES_RUNLOCK();
4863 
4864 		mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
4865 		ip_dn_io_ptr(m0, &dnflow);
4866 		if (*m0 != NULL)
4867 			pf_dummynet_flag_remove(m, mtag);
4868 	} else {
4869 		PF_RULES_RUNLOCK();
4870 	}
4871 
4872 	if (action == PF_PASS && bridge_to) {
4873 		pf_bridge_to(bridge_to->pfik_ifp, *m0);
4874 		*m0 = NULL; /* We've eaten the packet. */
4875 	}
4876 
4877 	return (action);
4878 }
4879 
4880 static int
4881 pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, struct pfi_kkif *kif,
4882     struct mbuf *m, int off, struct pf_pdesc *pd, struct pf_krule **am,
4883     struct pf_kruleset **rsm, struct inpcb *inp, int hdrlen)
4884 {
4885 	struct pf_krule		*nr = NULL;
4886 	struct pf_addr		* const saddr = pd->src;
4887 	struct pf_addr		* const daddr = pd->dst;
4888 	sa_family_t		 af = pd->af;
4889 	struct pf_krule		*r, *a = NULL;
4890 	struct pf_kruleset	*ruleset = NULL;
4891 	struct pf_krule_slist	 match_rules;
4892 	struct pf_krule_item	*ri;
4893 	struct pf_ksrc_node	*nsn = NULL;
4894 	struct tcphdr		*th = &pd->hdr.tcp;
4895 	struct pf_state_key	*sk = NULL, *nk = NULL;
4896 	u_short			 reason, transerror;
4897 	int			 rewrite = 0;
4898 	int			 tag = -1;
4899 	int			 asd = 0;
4900 	int			 match = 0;
4901 	int			 state_icmp = 0, icmp_dir, multi;
4902 	u_int16_t		 sport = 0, dport = 0, virtual_type, virtual_id;
4903 	u_int16_t		 bproto_sum = 0, bip_sum = 0;
4904 	u_int8_t		 icmptype = 0, icmpcode = 0;
4905 	struct pf_kanchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
4906 	struct pf_udp_mapping	*udp_mapping = NULL;
4907 
4908 	PF_RULES_RASSERT();
4909 
4910 	SLIST_INIT(&match_rules);
4911 
4912 	if (inp != NULL) {
4913 		INP_LOCK_ASSERT(inp);
4914 		pd->lookup.uid = inp->inp_cred->cr_uid;
4915 		pd->lookup.gid = inp->inp_cred->cr_groups[0];
4916 		pd->lookup.done = 1;
4917 	}
4918 
4919 	switch (pd->proto) {
4920 	case IPPROTO_TCP:
4921 		sport = th->th_sport;
4922 		dport = th->th_dport;
4923 		break;
4924 	case IPPROTO_UDP:
4925 		sport = pd->hdr.udp.uh_sport;
4926 		dport = pd->hdr.udp.uh_dport;
4927 		break;
4928 	case IPPROTO_SCTP:
4929 		sport = pd->hdr.sctp.src_port;
4930 		dport = pd->hdr.sctp.dest_port;
4931 		break;
4932 #ifdef INET
4933 	case IPPROTO_ICMP:
4934 		MPASS(af == AF_INET);
4935 		icmptype = pd->hdr.icmp.icmp_type;
4936 		icmpcode = pd->hdr.icmp.icmp_code;
4937 		state_icmp = pf_icmp_mapping(pd, icmptype,
4938 		    &icmp_dir, &multi, &virtual_id, &virtual_type);
4939 		if (icmp_dir == PF_IN) {
4940 			sport = virtual_id;
4941 			dport = virtual_type;
4942 		} else {
4943 			sport = virtual_type;
4944 			dport = virtual_id;
4945 		}
4946 		break;
4947 #endif /* INET */
4948 #ifdef INET6
4949 	case IPPROTO_ICMPV6:
4950 		MPASS(af == AF_INET6);
4951 		icmptype = pd->hdr.icmp6.icmp6_type;
4952 		icmpcode = pd->hdr.icmp6.icmp6_code;
4953 		state_icmp = pf_icmp_mapping(pd, icmptype,
4954 		    &icmp_dir, &multi, &virtual_id, &virtual_type);
4955 		if (icmp_dir == PF_IN) {
4956 			sport = virtual_id;
4957 			dport = virtual_type;
4958 		} else {
4959 			sport = virtual_type;
4960 			dport = virtual_id;
4961 		}
4962 
4963 		break;
4964 #endif /* INET6 */
4965 	default:
4966 		sport = dport = 0;
4967 		break;
4968 	}
4969 
4970 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
4971 
4972 	/* check packet for BINAT/NAT/RDR */
4973 	transerror = pf_get_translation(pd, m, off, kif, &nsn, &sk,
4974 	    &nk, saddr, daddr, sport, dport, anchor_stack, &nr, &udp_mapping);
4975 	switch (transerror) {
4976 	default:
4977 		/* A translation error occurred. */
4978 		REASON_SET(&reason, transerror);
4979 		goto cleanup;
4980 	case PFRES_MAX:
4981 		/* No match. */
4982 		break;
4983 	case PFRES_MATCH:
4984 		KASSERT(sk != NULL, ("%s: null sk", __func__));
4985 		KASSERT(nk != NULL, ("%s: null nk", __func__));
4986 
4987 		if (nr->log) {
4988 			PFLOG_PACKET(kif, m, PF_PASS, PFRES_MATCH, nr, a,
4989 			    ruleset, pd, 1);
4990 		}
4991 
4992 		if (pd->ip_sum)
4993 			bip_sum = *pd->ip_sum;
4994 
4995 		switch (pd->proto) {
4996 		case IPPROTO_TCP:
4997 			bproto_sum = th->th_sum;
4998 			pd->proto_sum = &th->th_sum;
4999 
5000 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
5001 			    nk->port[pd->sidx] != sport) {
5002 				pf_change_ap(m, saddr, &th->th_sport, pd->ip_sum,
5003 				    &th->th_sum, &nk->addr[pd->sidx],
5004 				    nk->port[pd->sidx], 0, af);
5005 				pd->sport = &th->th_sport;
5006 				sport = th->th_sport;
5007 			}
5008 
5009 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
5010 			    nk->port[pd->didx] != dport) {
5011 				pf_change_ap(m, daddr, &th->th_dport, pd->ip_sum,
5012 				    &th->th_sum, &nk->addr[pd->didx],
5013 				    nk->port[pd->didx], 0, af);
5014 				dport = th->th_dport;
5015 				pd->dport = &th->th_dport;
5016 			}
5017 			rewrite++;
5018 			break;
5019 		case IPPROTO_UDP:
5020 			bproto_sum = pd->hdr.udp.uh_sum;
5021 			pd->proto_sum = &pd->hdr.udp.uh_sum;
5022 
5023 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
5024 			    nk->port[pd->sidx] != sport) {
5025 				pf_change_ap(m, saddr, &pd->hdr.udp.uh_sport,
5026 				    pd->ip_sum, &pd->hdr.udp.uh_sum,
5027 				    &nk->addr[pd->sidx],
5028 				    nk->port[pd->sidx], 1, af);
5029 				sport = pd->hdr.udp.uh_sport;
5030 				pd->sport = &pd->hdr.udp.uh_sport;
5031 			}
5032 
5033 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
5034 			    nk->port[pd->didx] != dport) {
5035 				pf_change_ap(m, daddr, &pd->hdr.udp.uh_dport,
5036 				    pd->ip_sum, &pd->hdr.udp.uh_sum,
5037 				    &nk->addr[pd->didx],
5038 				    nk->port[pd->didx], 1, af);
5039 				dport = pd->hdr.udp.uh_dport;
5040 				pd->dport = &pd->hdr.udp.uh_dport;
5041 			}
5042 			rewrite++;
5043 			break;
5044 		case IPPROTO_SCTP: {
5045 			uint16_t checksum = 0;
5046 
5047 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], af) ||
5048 			    nk->port[pd->sidx] != sport) {
5049 				pf_change_ap(m, saddr, &pd->hdr.sctp.src_port,
5050 				    pd->ip_sum, &checksum,
5051 				    &nk->addr[pd->sidx],
5052 				    nk->port[pd->sidx], 1, af);
5053 			}
5054 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], af) ||
5055 			    nk->port[pd->didx] != dport) {
5056 				pf_change_ap(m, daddr, &pd->hdr.sctp.dest_port,
5057 				    pd->ip_sum, &checksum,
5058 				    &nk->addr[pd->didx],
5059 				    nk->port[pd->didx], 1, af);
5060 			}
5061 			break;
5062 		}
5063 #ifdef INET
5064 		case IPPROTO_ICMP:
5065 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET))
5066 				pf_change_a(&saddr->v4.s_addr, pd->ip_sum,
5067 				    nk->addr[pd->sidx].v4.s_addr, 0);
5068 
5069 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET))
5070 				pf_change_a(&daddr->v4.s_addr, pd->ip_sum,
5071 				    nk->addr[pd->didx].v4.s_addr, 0);
5072 
5073 			if (virtual_type == htons(ICMP_ECHO) &&
5074 			     nk->port[pd->sidx] != pd->hdr.icmp.icmp_id) {
5075 				pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
5076 				    pd->hdr.icmp.icmp_cksum, sport,
5077 				    nk->port[pd->sidx], 0);
5078 				pd->hdr.icmp.icmp_id = nk->port[pd->sidx];
5079 				pd->sport = &pd->hdr.icmp.icmp_id;
5080 			}
5081 			m_copyback(m, off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
5082 			break;
5083 #endif /* INET */
5084 #ifdef INET6
5085 		case IPPROTO_ICMPV6:
5086 			if (PF_ANEQ(saddr, &nk->addr[pd->sidx], AF_INET6))
5087 				pf_change_a6(saddr, &pd->hdr.icmp6.icmp6_cksum,
5088 				    &nk->addr[pd->sidx], 0);
5089 
5090 			if (PF_ANEQ(daddr, &nk->addr[pd->didx], AF_INET6))
5091 				pf_change_a6(daddr, &pd->hdr.icmp6.icmp6_cksum,
5092 				    &nk->addr[pd->didx], 0);
5093 			rewrite++;
5094 			break;
5095 #endif /* INET */
5096 		default:
5097 			switch (af) {
5098 #ifdef INET
5099 			case AF_INET:
5100 				if (PF_ANEQ(saddr,
5101 				    &nk->addr[pd->sidx], AF_INET))
5102 					pf_change_a(&saddr->v4.s_addr,
5103 					    pd->ip_sum,
5104 					    nk->addr[pd->sidx].v4.s_addr, 0);
5105 
5106 				if (PF_ANEQ(daddr,
5107 				    &nk->addr[pd->didx], AF_INET))
5108 					pf_change_a(&daddr->v4.s_addr,
5109 					    pd->ip_sum,
5110 					    nk->addr[pd->didx].v4.s_addr, 0);
5111 				break;
5112 #endif /* INET */
5113 #ifdef INET6
5114 			case AF_INET6:
5115 				if (PF_ANEQ(saddr,
5116 				    &nk->addr[pd->sidx], AF_INET6))
5117 					PF_ACPY(saddr, &nk->addr[pd->sidx], af);
5118 
5119 				if (PF_ANEQ(daddr,
5120 				    &nk->addr[pd->didx], AF_INET6))
5121 					PF_ACPY(daddr, &nk->addr[pd->didx], af);
5122 				break;
5123 #endif /* INET */
5124 			}
5125 			break;
5126 		}
5127 		if (nr->natpass)
5128 			r = NULL;
5129 		pd->nat_rule = nr;
5130 	}
5131 
5132 	while (r != NULL) {
5133 		pf_counter_u64_add(&r->evaluations, 1);
5134 		if (pfi_kkif_match(r->kif, kif) == r->ifnot)
5135 			r = r->skip[PF_SKIP_IFP].ptr;
5136 		else if (r->direction && r->direction != pd->dir)
5137 			r = r->skip[PF_SKIP_DIR].ptr;
5138 		else if (r->af && r->af != af)
5139 			r = r->skip[PF_SKIP_AF].ptr;
5140 		else if (r->proto && r->proto != pd->proto)
5141 			r = r->skip[PF_SKIP_PROTO].ptr;
5142 		else if (PF_MISMATCHAW(&r->src.addr, saddr, af,
5143 		    r->src.neg, kif, M_GETFIB(m)))
5144 			r = r->skip[PF_SKIP_SRC_ADDR].ptr;
5145 		/* tcp/udp only. port_op always 0 in other cases */
5146 		else if (r->src.port_op && !pf_match_port(r->src.port_op,
5147 		    r->src.port[0], r->src.port[1], sport))
5148 			r = r->skip[PF_SKIP_SRC_PORT].ptr;
5149 		else if (PF_MISMATCHAW(&r->dst.addr, daddr, af,
5150 		    r->dst.neg, NULL, M_GETFIB(m)))
5151 			r = r->skip[PF_SKIP_DST_ADDR].ptr;
5152 		/* tcp/udp only. port_op always 0 in other cases */
5153 		else if (r->dst.port_op && !pf_match_port(r->dst.port_op,
5154 		    r->dst.port[0], r->dst.port[1], dport))
5155 			r = r->skip[PF_SKIP_DST_PORT].ptr;
5156 		/* icmp only. type always 0 in other cases */
5157 		else if (r->type && r->type != icmptype + 1)
5158 			r = TAILQ_NEXT(r, entries);
5159 		/* icmp only. type always 0 in other cases */
5160 		else if (r->code && r->code != icmpcode + 1)
5161 			r = TAILQ_NEXT(r, entries);
5162 		else if (r->tos && !(r->tos == pd->tos))
5163 			r = TAILQ_NEXT(r, entries);
5164 		else if (r->rule_flag & PFRULE_FRAGMENT)
5165 			r = TAILQ_NEXT(r, entries);
5166 		else if (pd->proto == IPPROTO_TCP &&
5167 		    (r->flagset & th->th_flags) != r->flags)
5168 			r = TAILQ_NEXT(r, entries);
5169 		/* tcp/udp only. uid.op always 0 in other cases */
5170 		else if (r->uid.op && (pd->lookup.done || (pd->lookup.done =
5171 		    pf_socket_lookup(pd, m), 1)) &&
5172 		    !pf_match_uid(r->uid.op, r->uid.uid[0], r->uid.uid[1],
5173 		    pd->lookup.uid))
5174 			r = TAILQ_NEXT(r, entries);
5175 		/* tcp/udp only. gid.op always 0 in other cases */
5176 		else if (r->gid.op && (pd->lookup.done || (pd->lookup.done =
5177 		    pf_socket_lookup(pd, m), 1)) &&
5178 		    !pf_match_gid(r->gid.op, r->gid.gid[0], r->gid.gid[1],
5179 		    pd->lookup.gid))
5180 			r = TAILQ_NEXT(r, entries);
5181 		else if (r->prio &&
5182 		    !pf_match_ieee8021q_pcp(r->prio, m))
5183 			r = TAILQ_NEXT(r, entries);
5184 		else if (r->prob &&
5185 		    r->prob <= arc4random())
5186 			r = TAILQ_NEXT(r, entries);
5187 		else if (r->match_tag && !pf_match_tag(m, r, &tag,
5188 		    pd->pf_mtag ? pd->pf_mtag->tag : 0))
5189 			r = TAILQ_NEXT(r, entries);
5190 		else if (r->rcv_kif && !pf_match_rcvif(m, r))
5191 			r = TAILQ_NEXT(r, entries);
5192 		else if (r->os_fingerprint != PF_OSFP_ANY &&
5193 		    (pd->proto != IPPROTO_TCP || !pf_osfp_match(
5194 		    pf_osfp_fingerprint(pd, m, off, th),
5195 		    r->os_fingerprint)))
5196 			r = TAILQ_NEXT(r, entries);
5197 		else {
5198 			if (r->tag)
5199 				tag = r->tag;
5200 			if (r->anchor == NULL) {
5201 				if (r->action == PF_MATCH) {
5202 					ri = malloc(sizeof(struct pf_krule_item), M_PF_RULE_ITEM, M_NOWAIT | M_ZERO);
5203 					if (ri == NULL) {
5204 						REASON_SET(&reason, PFRES_MEMORY);
5205 						goto cleanup;
5206 					}
5207 					ri->r = r;
5208 					SLIST_INSERT_HEAD(&match_rules, ri, entry);
5209 					pf_counter_u64_critical_enter();
5210 					pf_counter_u64_add_protected(&r->packets[pd->dir == PF_OUT], 1);
5211 					pf_counter_u64_add_protected(&r->bytes[pd->dir == PF_OUT], pd->tot_len);
5212 					pf_counter_u64_critical_exit();
5213 					pf_rule_to_actions(r, &pd->act);
5214 					if (r->log || pd->act.log & PF_LOG_MATCHES)
5215 						PFLOG_PACKET(kif, m,
5216 						    r->action, PFRES_MATCH, r,
5217 						    a, ruleset, pd, 1);
5218 				} else {
5219 					match = 1;
5220 					*rm = r;
5221 					*am = a;
5222 					*rsm = ruleset;
5223 					if (pd->act.log & PF_LOG_MATCHES)
5224 						PFLOG_PACKET(kif, m,
5225 						    r->action, PFRES_MATCH, r,
5226 						    a, ruleset, pd, 1);
5227 				}
5228 				if ((*rm)->quick)
5229 					break;
5230 				r = TAILQ_NEXT(r, entries);
5231 			} else
5232 				pf_step_into_anchor(anchor_stack, &asd,
5233 				    &ruleset, PF_RULESET_FILTER, &r, &a,
5234 				    &match);
5235 		}
5236 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
5237 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
5238 			break;
5239 	}
5240 	r = *rm;
5241 	a = *am;
5242 	ruleset = *rsm;
5243 
5244 	REASON_SET(&reason, PFRES_MATCH);
5245 
5246 	/* apply actions for last matching pass/block rule */
5247 	pf_rule_to_actions(r, &pd->act);
5248 
5249 	if (r->log || pd->act.log & PF_LOG_MATCHES) {
5250 		if (rewrite)
5251 			m_copyback(m, off, hdrlen, pd->hdr.any);
5252 		PFLOG_PACKET(kif, m, r->action, reason, r, a, ruleset, pd, 1);
5253 	}
5254 
5255 	if ((r->action == PF_DROP) &&
5256 	    ((r->rule_flag & PFRULE_RETURNRST) ||
5257 	    (r->rule_flag & PFRULE_RETURNICMP) ||
5258 	    (r->rule_flag & PFRULE_RETURN))) {
5259 		pf_return(r, nr, pd, sk, off, m, th, kif, bproto_sum,
5260 		    bip_sum, hdrlen, &reason, r->rtableid);
5261 	}
5262 
5263 	if (r->action == PF_DROP)
5264 		goto cleanup;
5265 
5266 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
5267 		REASON_SET(&reason, PFRES_MEMORY);
5268 		goto cleanup;
5269 	}
5270 	if (pd->act.rtableid >= 0)
5271 		M_SETFIB(m, pd->act.rtableid);
5272 
5273 	if (!state_icmp && (r->keep_state || nr != NULL ||
5274 	    (pd->flags & PFDESC_TCP_NORM))) {
5275 		int action;
5276 		action = pf_create_state(r, nr, a, pd, nsn, nk, sk, m, off,
5277 		    sport, dport, &rewrite, kif, sm, tag, bproto_sum, bip_sum,
5278 		    hdrlen, &match_rules, udp_mapping);
5279 		if (action != PF_PASS) {
5280 			pf_udp_mapping_release(udp_mapping);
5281 			if (action == PF_DROP &&
5282 			    (r->rule_flag & PFRULE_RETURN))
5283 				pf_return(r, nr, pd, sk, off, m, th, kif,
5284 				    bproto_sum, bip_sum, hdrlen, &reason,
5285 				    pd->act.rtableid);
5286 			return (action);
5287 		}
5288 	} else {
5289 		while ((ri = SLIST_FIRST(&match_rules))) {
5290 			SLIST_REMOVE_HEAD(&match_rules, entry);
5291 			free(ri, M_PF_RULE_ITEM);
5292 		}
5293 
5294 		uma_zfree(V_pf_state_key_z, sk);
5295 		uma_zfree(V_pf_state_key_z, nk);
5296 		pf_udp_mapping_release(udp_mapping);
5297 	}
5298 
5299 	/* copy back packet headers if we performed NAT operations */
5300 	if (rewrite)
5301 		m_copyback(m, off, hdrlen, pd->hdr.any);
5302 
5303 	if (*sm != NULL && !((*sm)->state_flags & PFSTATE_NOSYNC) &&
5304 	    pd->dir == PF_OUT &&
5305 	    V_pfsync_defer_ptr != NULL && V_pfsync_defer_ptr(*sm, m))
5306 		/*
5307 		 * We want the state created, but we dont
5308 		 * want to send this in case a partner
5309 		 * firewall has to know about it to allow
5310 		 * replies through it.
5311 		 */
5312 		return (PF_DEFER);
5313 
5314 	return (PF_PASS);
5315 
5316 cleanup:
5317 	while ((ri = SLIST_FIRST(&match_rules))) {
5318 		SLIST_REMOVE_HEAD(&match_rules, entry);
5319 		free(ri, M_PF_RULE_ITEM);
5320 	}
5321 
5322 	uma_zfree(V_pf_state_key_z, sk);
5323 	uma_zfree(V_pf_state_key_z, nk);
5324 	pf_udp_mapping_release(udp_mapping);
5325 
5326 	return (PF_DROP);
5327 }
5328 
5329 static int
5330 pf_create_state(struct pf_krule *r, struct pf_krule *nr, struct pf_krule *a,
5331     struct pf_pdesc *pd, struct pf_ksrc_node *nsn, struct pf_state_key *nk,
5332     struct pf_state_key *sk, struct mbuf *m, int off, u_int16_t sport,
5333     u_int16_t dport, int *rewrite, struct pfi_kkif *kif, struct pf_kstate **sm,
5334     int tag, u_int16_t bproto_sum, u_int16_t bip_sum, int hdrlen,
5335     struct pf_krule_slist *match_rules, struct pf_udp_mapping *udp_mapping)
5336 {
5337 	struct pf_kstate	*s = NULL;
5338 	struct pf_ksrc_node	*sn = NULL;
5339 	struct tcphdr		*th = &pd->hdr.tcp;
5340 	u_int16_t		 mss = V_tcp_mssdflt;
5341 	u_short			 reason, sn_reason;
5342 	struct pf_krule_item	*ri;
5343 
5344 	/* check maximums */
5345 	if (r->max_states &&
5346 	    (counter_u64_fetch(r->states_cur) >= r->max_states)) {
5347 		counter_u64_add(V_pf_status.lcounters[LCNT_STATES], 1);
5348 		REASON_SET(&reason, PFRES_MAXSTATES);
5349 		goto csfailed;
5350 	}
5351 	/* src node for filter rule */
5352 	if ((r->rule_flag & PFRULE_SRCTRACK ||
5353 	    r->rpool.opts & PF_POOL_STICKYADDR) &&
5354 	    (sn_reason = pf_insert_src_node(&sn, r, pd->src, pd->af)) != 0) {
5355 		REASON_SET(&reason, sn_reason);
5356 		goto csfailed;
5357 	}
5358 	/* src node for translation rule */
5359 	if (nr != NULL && (nr->rpool.opts & PF_POOL_STICKYADDR) &&
5360 	    (sn_reason = pf_insert_src_node(&nsn, nr, &sk->addr[pd->sidx],
5361 	    pd->af)) != 0 ) {
5362 		REASON_SET(&reason, sn_reason);
5363 		goto csfailed;
5364 	}
5365 	s = pf_alloc_state(M_NOWAIT);
5366 	if (s == NULL) {
5367 		REASON_SET(&reason, PFRES_MEMORY);
5368 		goto csfailed;
5369 	}
5370 	s->rule.ptr = r;
5371 	s->nat_rule.ptr = nr;
5372 	s->anchor.ptr = a;
5373 	bcopy(match_rules, &s->match_rules, sizeof(s->match_rules));
5374 	memcpy(&s->act, &pd->act, sizeof(struct pf_rule_actions));
5375 
5376 	STATE_INC_COUNTERS(s);
5377 	if (r->allow_opts)
5378 		s->state_flags |= PFSTATE_ALLOWOPTS;
5379 	if (r->rule_flag & PFRULE_STATESLOPPY)
5380 		s->state_flags |= PFSTATE_SLOPPY;
5381 	if (pd->flags & PFDESC_TCP_NORM) /* Set by old-style scrub rules */
5382 		s->state_flags |= PFSTATE_SCRUB_TCP;
5383 	if ((r->rule_flag & PFRULE_PFLOW) ||
5384 	    (nr != NULL && nr->rule_flag & PFRULE_PFLOW))
5385 		s->state_flags |= PFSTATE_PFLOW;
5386 
5387 	s->act.log = pd->act.log & PF_LOG_ALL;
5388 	s->sync_state = PFSYNC_S_NONE;
5389 	s->state_flags |= pd->act.flags; /* Only needed for pfsync and state export */
5390 
5391 	if (nr != NULL)
5392 		s->act.log |= nr->log & PF_LOG_ALL;
5393 	switch (pd->proto) {
5394 	case IPPROTO_TCP:
5395 		s->src.seqlo = ntohl(th->th_seq);
5396 		s->src.seqhi = s->src.seqlo + pd->p_len + 1;
5397 		if ((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN &&
5398 		    r->keep_state == PF_STATE_MODULATE) {
5399 			/* Generate sequence number modulator */
5400 			if ((s->src.seqdiff = pf_tcp_iss(pd) - s->src.seqlo) ==
5401 			    0)
5402 				s->src.seqdiff = 1;
5403 			pf_change_proto_a(m, &th->th_seq, &th->th_sum,
5404 			    htonl(s->src.seqlo + s->src.seqdiff), 0);
5405 			*rewrite = 1;
5406 		} else
5407 			s->src.seqdiff = 0;
5408 		if (th->th_flags & TH_SYN) {
5409 			s->src.seqhi++;
5410 			s->src.wscale = pf_get_wscale(m, off,
5411 			    th->th_off, pd->af);
5412 		}
5413 		s->src.max_win = MAX(ntohs(th->th_win), 1);
5414 		if (s->src.wscale & PF_WSCALE_MASK) {
5415 			/* Remove scale factor from initial window */
5416 			int win = s->src.max_win;
5417 			win += 1 << (s->src.wscale & PF_WSCALE_MASK);
5418 			s->src.max_win = (win - 1) >>
5419 			    (s->src.wscale & PF_WSCALE_MASK);
5420 		}
5421 		if (th->th_flags & TH_FIN)
5422 			s->src.seqhi++;
5423 		s->dst.seqhi = 1;
5424 		s->dst.max_win = 1;
5425 		pf_set_protostate(s, PF_PEER_SRC, TCPS_SYN_SENT);
5426 		pf_set_protostate(s, PF_PEER_DST, TCPS_CLOSED);
5427 		s->timeout = PFTM_TCP_FIRST_PACKET;
5428 		atomic_add_32(&V_pf_status.states_halfopen, 1);
5429 		break;
5430 	case IPPROTO_UDP:
5431 		pf_set_protostate(s, PF_PEER_SRC, PFUDPS_SINGLE);
5432 		pf_set_protostate(s, PF_PEER_DST, PFUDPS_NO_TRAFFIC);
5433 		s->timeout = PFTM_UDP_FIRST_PACKET;
5434 		break;
5435 	case IPPROTO_SCTP:
5436 		pf_set_protostate(s, PF_PEER_SRC, SCTP_COOKIE_WAIT);
5437 		pf_set_protostate(s, PF_PEER_DST, SCTP_CLOSED);
5438 		s->timeout = PFTM_SCTP_FIRST_PACKET;
5439 		break;
5440 	case IPPROTO_ICMP:
5441 #ifdef INET6
5442 	case IPPROTO_ICMPV6:
5443 #endif
5444 		s->timeout = PFTM_ICMP_FIRST_PACKET;
5445 		break;
5446 	default:
5447 		pf_set_protostate(s, PF_PEER_SRC, PFOTHERS_SINGLE);
5448 		pf_set_protostate(s, PF_PEER_DST, PFOTHERS_NO_TRAFFIC);
5449 		s->timeout = PFTM_OTHER_FIRST_PACKET;
5450 	}
5451 
5452 	if (r->rt) {
5453 		/* pf_map_addr increases the reason counters */
5454 		if ((reason = pf_map_addr(pd->af, r, pd->src, &s->rt_addr,
5455 		    &s->rt_kif, NULL, &sn)) != 0)
5456 			goto csfailed;
5457 		s->rt = r->rt;
5458 	}
5459 
5460 	s->creation = s->expire = pf_get_uptime();
5461 
5462 	if (sn != NULL)
5463 		s->src_node = sn;
5464 	if (nsn != NULL) {
5465 		/* XXX We only modify one side for now. */
5466 		PF_ACPY(&nsn->raddr, &nk->addr[1], pd->af);
5467 		s->nat_src_node = nsn;
5468 	}
5469 	if (pd->proto == IPPROTO_TCP) {
5470 		if (s->state_flags & PFSTATE_SCRUB_TCP &&
5471 		    pf_normalize_tcp_init(m, off, pd, th, &s->src, &s->dst)) {
5472 			REASON_SET(&reason, PFRES_MEMORY);
5473 			goto csfailed;
5474 		}
5475 		if (s->state_flags & PFSTATE_SCRUB_TCP && s->src.scrub &&
5476 		    pf_normalize_tcp_stateful(m, off, pd, &reason, th, s,
5477 		    &s->src, &s->dst, rewrite)) {
5478 			/* This really shouldn't happen!!! */
5479 			DPFPRINTF(PF_DEBUG_URGENT,
5480 			    ("pf_normalize_tcp_stateful failed on first "
5481 			     "pkt\n"));
5482 			goto csfailed;
5483 		}
5484 	} else if (pd->proto == IPPROTO_SCTP) {
5485 		if (pf_normalize_sctp_init(m, off, pd, &s->src, &s->dst))
5486 			goto csfailed;
5487 		if (! (pd->sctp_flags & (PFDESC_SCTP_INIT | PFDESC_SCTP_ADD_IP)))
5488 			goto csfailed;
5489 	}
5490 	s->direction = pd->dir;
5491 
5492 	/*
5493 	 * sk/nk could already been setup by pf_get_translation().
5494 	 */
5495 	if (nr == NULL) {
5496 		KASSERT((sk == NULL && nk == NULL), ("%s: nr %p sk %p, nk %p",
5497 		    __func__, nr, sk, nk));
5498 		sk = pf_state_key_setup(pd, m, off, pd->src, pd->dst, sport, dport);
5499 		if (sk == NULL)
5500 			goto csfailed;
5501 		nk = sk;
5502 	} else
5503 		KASSERT((sk != NULL && nk != NULL), ("%s: nr %p sk %p, nk %p",
5504 		    __func__, nr, sk, nk));
5505 
5506 	/* Swap sk/nk for PF_OUT. */
5507 	if (pf_state_insert(BOUND_IFACE(s, kif), kif,
5508 	    (pd->dir == PF_IN) ? sk : nk,
5509 	    (pd->dir == PF_IN) ? nk : sk, s)) {
5510 		REASON_SET(&reason, PFRES_STATEINS);
5511 		goto drop;
5512 	} else
5513 		*sm = s;
5514 
5515 	if (tag > 0)
5516 		s->tag = tag;
5517 	if (pd->proto == IPPROTO_TCP && (th->th_flags & (TH_SYN|TH_ACK)) ==
5518 	    TH_SYN && r->keep_state == PF_STATE_SYNPROXY) {
5519 		pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_SRC);
5520 		/* undo NAT changes, if they have taken place */
5521 		if (nr != NULL) {
5522 			struct pf_state_key *skt = s->key[PF_SK_WIRE];
5523 			if (pd->dir == PF_OUT)
5524 				skt = s->key[PF_SK_STACK];
5525 			PF_ACPY(pd->src, &skt->addr[pd->sidx], pd->af);
5526 			PF_ACPY(pd->dst, &skt->addr[pd->didx], pd->af);
5527 			if (pd->sport)
5528 				*pd->sport = skt->port[pd->sidx];
5529 			if (pd->dport)
5530 				*pd->dport = skt->port[pd->didx];
5531 			if (pd->proto_sum)
5532 				*pd->proto_sum = bproto_sum;
5533 			if (pd->ip_sum)
5534 				*pd->ip_sum = bip_sum;
5535 			m_copyback(m, off, hdrlen, pd->hdr.any);
5536 		}
5537 		s->src.seqhi = htonl(arc4random());
5538 		/* Find mss option */
5539 		int rtid = M_GETFIB(m);
5540 		mss = pf_get_mss(m, off, th->th_off, pd->af);
5541 		mss = pf_calc_mss(pd->src, pd->af, rtid, mss);
5542 		mss = pf_calc_mss(pd->dst, pd->af, rtid, mss);
5543 		s->src.mss = mss;
5544 		pf_send_tcp(r, pd->af, pd->dst, pd->src, th->th_dport,
5545 		    th->th_sport, s->src.seqhi, ntohl(th->th_seq) + 1,
5546 		    TH_SYN|TH_ACK, 0, s->src.mss, 0, true, 0, 0,
5547 		    pd->act.rtableid);
5548 		REASON_SET(&reason, PFRES_SYNPROXY);
5549 		return (PF_SYNPROXY_DROP);
5550 	}
5551 
5552 	s->udp_mapping = udp_mapping;
5553 
5554 	return (PF_PASS);
5555 
5556 csfailed:
5557 	while ((ri = SLIST_FIRST(match_rules))) {
5558 		SLIST_REMOVE_HEAD(match_rules, entry);
5559 		free(ri, M_PF_RULE_ITEM);
5560 	}
5561 
5562 	uma_zfree(V_pf_state_key_z, sk);
5563 	uma_zfree(V_pf_state_key_z, nk);
5564 
5565 	if (sn != NULL) {
5566 		PF_SRC_NODE_LOCK(sn);
5567 		if (--sn->states == 0 && sn->expire == 0) {
5568 			pf_unlink_src_node(sn);
5569 			uma_zfree(V_pf_sources_z, sn);
5570 			counter_u64_add(
5571 			    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
5572 		}
5573 		PF_SRC_NODE_UNLOCK(sn);
5574 	}
5575 
5576 	if (nsn != sn && nsn != NULL) {
5577 		PF_SRC_NODE_LOCK(nsn);
5578 		if (--nsn->states == 0 && nsn->expire == 0) {
5579 			pf_unlink_src_node(nsn);
5580 			uma_zfree(V_pf_sources_z, nsn);
5581 			counter_u64_add(
5582 			    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
5583 		}
5584 		PF_SRC_NODE_UNLOCK(nsn);
5585 	}
5586 
5587 drop:
5588 	if (s != NULL) {
5589 		pf_src_tree_remove_state(s);
5590 		s->timeout = PFTM_UNLINKED;
5591 		STATE_DEC_COUNTERS(s);
5592 		pf_free_state(s);
5593 	}
5594 
5595 	return (PF_DROP);
5596 }
5597 
5598 static int
5599 pf_test_fragment(struct pf_krule **rm, struct pfi_kkif *kif,
5600     struct mbuf *m, struct pf_pdesc *pd, struct pf_krule **am,
5601     struct pf_kruleset **rsm)
5602 {
5603 	struct pf_krule		*r, *a = NULL;
5604 	struct pf_kruleset	*ruleset = NULL;
5605 	struct pf_krule_slist	 match_rules;
5606 	struct pf_krule_item	*ri;
5607 	sa_family_t		 af = pd->af;
5608 	u_short			 reason;
5609 	int			 tag = -1;
5610 	int			 asd = 0;
5611 	int			 match = 0;
5612 	struct pf_kanchor_stackframe	anchor_stack[PF_ANCHOR_STACKSIZE];
5613 
5614 	PF_RULES_RASSERT();
5615 
5616 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_FILTER].active.ptr);
5617 	SLIST_INIT(&match_rules);
5618 	while (r != NULL) {
5619 		pf_counter_u64_add(&r->evaluations, 1);
5620 		if (pfi_kkif_match(r->kif, kif) == r->ifnot)
5621 			r = r->skip[PF_SKIP_IFP].ptr;
5622 		else if (r->direction && r->direction != pd->dir)
5623 			r = r->skip[PF_SKIP_DIR].ptr;
5624 		else if (r->af && r->af != af)
5625 			r = r->skip[PF_SKIP_AF].ptr;
5626 		else if (r->proto && r->proto != pd->proto)
5627 			r = r->skip[PF_SKIP_PROTO].ptr;
5628 		else if (PF_MISMATCHAW(&r->src.addr, pd->src, af,
5629 		    r->src.neg, kif, M_GETFIB(m)))
5630 			r = r->skip[PF_SKIP_SRC_ADDR].ptr;
5631 		else if (PF_MISMATCHAW(&r->dst.addr, pd->dst, af,
5632 		    r->dst.neg, NULL, M_GETFIB(m)))
5633 			r = r->skip[PF_SKIP_DST_ADDR].ptr;
5634 		else if (r->tos && !(r->tos == pd->tos))
5635 			r = TAILQ_NEXT(r, entries);
5636 		else if (r->os_fingerprint != PF_OSFP_ANY)
5637 			r = TAILQ_NEXT(r, entries);
5638 		else if (pd->proto == IPPROTO_UDP &&
5639 		    (r->src.port_op || r->dst.port_op))
5640 			r = TAILQ_NEXT(r, entries);
5641 		else if (pd->proto == IPPROTO_TCP &&
5642 		    (r->src.port_op || r->dst.port_op || r->flagset))
5643 			r = TAILQ_NEXT(r, entries);
5644 		else if ((pd->proto == IPPROTO_ICMP ||
5645 		    pd->proto == IPPROTO_ICMPV6) &&
5646 		    (r->type || r->code))
5647 			r = TAILQ_NEXT(r, entries);
5648 		else if (r->prio &&
5649 		    !pf_match_ieee8021q_pcp(r->prio, m))
5650 			r = TAILQ_NEXT(r, entries);
5651 		else if (r->prob && r->prob <=
5652 		    (arc4random() % (UINT_MAX - 1) + 1))
5653 			r = TAILQ_NEXT(r, entries);
5654 		else if (r->match_tag && !pf_match_tag(m, r, &tag,
5655 		    pd->pf_mtag ? pd->pf_mtag->tag : 0))
5656 			r = TAILQ_NEXT(r, entries);
5657 		else {
5658 			if (r->anchor == NULL) {
5659 				if (r->action == PF_MATCH) {
5660 					ri = malloc(sizeof(struct pf_krule_item), M_PF_RULE_ITEM, M_NOWAIT | M_ZERO);
5661 					if (ri == NULL) {
5662 						REASON_SET(&reason, PFRES_MEMORY);
5663 						goto cleanup;
5664 					}
5665 					ri->r = r;
5666 					SLIST_INSERT_HEAD(&match_rules, ri, entry);
5667 					pf_counter_u64_critical_enter();
5668 					pf_counter_u64_add_protected(&r->packets[pd->dir == PF_OUT], 1);
5669 					pf_counter_u64_add_protected(&r->bytes[pd->dir == PF_OUT], pd->tot_len);
5670 					pf_counter_u64_critical_exit();
5671 					pf_rule_to_actions(r, &pd->act);
5672 					if (r->log)
5673 						PFLOG_PACKET(kif, m,
5674 						    r->action, PFRES_MATCH, r,
5675 						    a, ruleset, pd, 1);
5676 				} else {
5677 					match = 1;
5678 					*rm = r;
5679 					*am = a;
5680 					*rsm = ruleset;
5681 				}
5682 				if ((*rm)->quick)
5683 					break;
5684 				r = TAILQ_NEXT(r, entries);
5685 			} else
5686 				pf_step_into_anchor(anchor_stack, &asd,
5687 				    &ruleset, PF_RULESET_FILTER, &r, &a,
5688 				    &match);
5689 		}
5690 		if (r == NULL && pf_step_out_of_anchor(anchor_stack, &asd,
5691 		    &ruleset, PF_RULESET_FILTER, &r, &a, &match))
5692 			break;
5693 	}
5694 	r = *rm;
5695 	a = *am;
5696 	ruleset = *rsm;
5697 
5698 	REASON_SET(&reason, PFRES_MATCH);
5699 
5700 	/* apply actions for last matching pass/block rule */
5701 	pf_rule_to_actions(r, &pd->act);
5702 
5703 	if (r->log)
5704 		PFLOG_PACKET(kif, m, r->action, reason, r, a, ruleset, pd, 1);
5705 
5706 	if (r->action != PF_PASS)
5707 		return (PF_DROP);
5708 
5709 	if (tag > 0 && pf_tag_packet(m, pd, tag)) {
5710 		REASON_SET(&reason, PFRES_MEMORY);
5711 		goto cleanup;
5712 	}
5713 
5714 	return (PF_PASS);
5715 
5716 cleanup:
5717 	while ((ri = SLIST_FIRST(&match_rules))) {
5718 		SLIST_REMOVE_HEAD(&match_rules, entry);
5719 		free(ri, M_PF_RULE_ITEM);
5720 	}
5721 
5722 	return (PF_DROP);
5723 }
5724 
5725 static int
5726 pf_tcp_track_full(struct pf_kstate **state, struct pfi_kkif *kif,
5727     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason,
5728     int *copyback)
5729 {
5730 	struct tcphdr		*th = &pd->hdr.tcp;
5731 	struct pf_state_peer	*src, *dst;
5732 	u_int16_t		 win = ntohs(th->th_win);
5733 	u_int32_t		 ack, end, data_end, seq, orig_seq;
5734 	u_int8_t		 sws, dws, psrc, pdst;
5735 	int			 ackskew;
5736 
5737 	if (pd->dir == (*state)->direction) {
5738 		src = &(*state)->src;
5739 		dst = &(*state)->dst;
5740 		psrc = PF_PEER_SRC;
5741 		pdst = PF_PEER_DST;
5742 	} else {
5743 		src = &(*state)->dst;
5744 		dst = &(*state)->src;
5745 		psrc = PF_PEER_DST;
5746 		pdst = PF_PEER_SRC;
5747 	}
5748 
5749 	if (src->wscale && dst->wscale && !(th->th_flags & TH_SYN)) {
5750 		sws = src->wscale & PF_WSCALE_MASK;
5751 		dws = dst->wscale & PF_WSCALE_MASK;
5752 	} else
5753 		sws = dws = 0;
5754 
5755 	/*
5756 	 * Sequence tracking algorithm from Guido van Rooij's paper:
5757 	 *   http://www.madison-gurkha.com/publications/tcp_filtering/
5758 	 *	tcp_filtering.ps
5759 	 */
5760 
5761 	orig_seq = seq = ntohl(th->th_seq);
5762 	if (src->seqlo == 0) {
5763 		/* First packet from this end. Set its state */
5764 
5765 		if (((*state)->state_flags & PFSTATE_SCRUB_TCP || dst->scrub) &&
5766 		    src->scrub == NULL) {
5767 			if (pf_normalize_tcp_init(m, off, pd, th, src, dst)) {
5768 				REASON_SET(reason, PFRES_MEMORY);
5769 				return (PF_DROP);
5770 			}
5771 		}
5772 
5773 		/* Deferred generation of sequence number modulator */
5774 		if (dst->seqdiff && !src->seqdiff) {
5775 			/* use random iss for the TCP server */
5776 			while ((src->seqdiff = arc4random() - seq) == 0)
5777 				;
5778 			ack = ntohl(th->th_ack) - dst->seqdiff;
5779 			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
5780 			    src->seqdiff), 0);
5781 			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
5782 			*copyback = 1;
5783 		} else {
5784 			ack = ntohl(th->th_ack);
5785 		}
5786 
5787 		end = seq + pd->p_len;
5788 		if (th->th_flags & TH_SYN) {
5789 			end++;
5790 			if (dst->wscale & PF_WSCALE_FLAG) {
5791 				src->wscale = pf_get_wscale(m, off, th->th_off,
5792 				    pd->af);
5793 				if (src->wscale & PF_WSCALE_FLAG) {
5794 					/* Remove scale factor from initial
5795 					 * window */
5796 					sws = src->wscale & PF_WSCALE_MASK;
5797 					win = ((u_int32_t)win + (1 << sws) - 1)
5798 					    >> sws;
5799 					dws = dst->wscale & PF_WSCALE_MASK;
5800 				} else {
5801 					/* fixup other window */
5802 					dst->max_win = MIN(TCP_MAXWIN,
5803 					    (u_int32_t)dst->max_win <<
5804 					    (dst->wscale & PF_WSCALE_MASK));
5805 					/* in case of a retrans SYN|ACK */
5806 					dst->wscale = 0;
5807 				}
5808 			}
5809 		}
5810 		data_end = end;
5811 		if (th->th_flags & TH_FIN)
5812 			end++;
5813 
5814 		src->seqlo = seq;
5815 		if (src->state < TCPS_SYN_SENT)
5816 			pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5817 
5818 		/*
5819 		 * May need to slide the window (seqhi may have been set by
5820 		 * the crappy stack check or if we picked up the connection
5821 		 * after establishment)
5822 		 */
5823 		if (src->seqhi == 1 ||
5824 		    SEQ_GEQ(end + MAX(1, dst->max_win << dws), src->seqhi))
5825 			src->seqhi = end + MAX(1, dst->max_win << dws);
5826 		if (win > src->max_win)
5827 			src->max_win = win;
5828 
5829 	} else {
5830 		ack = ntohl(th->th_ack) - dst->seqdiff;
5831 		if (src->seqdiff) {
5832 			/* Modulate sequence numbers */
5833 			pf_change_proto_a(m, &th->th_seq, &th->th_sum, htonl(seq +
5834 			    src->seqdiff), 0);
5835 			pf_change_proto_a(m, &th->th_ack, &th->th_sum, htonl(ack), 0);
5836 			*copyback = 1;
5837 		}
5838 		end = seq + pd->p_len;
5839 		if (th->th_flags & TH_SYN)
5840 			end++;
5841 		data_end = end;
5842 		if (th->th_flags & TH_FIN)
5843 			end++;
5844 	}
5845 
5846 	if ((th->th_flags & TH_ACK) == 0) {
5847 		/* Let it pass through the ack skew check */
5848 		ack = dst->seqlo;
5849 	} else if ((ack == 0 &&
5850 	    (th->th_flags & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) ||
5851 	    /* broken tcp stacks do not set ack */
5852 	    (dst->state < TCPS_SYN_SENT)) {
5853 		/*
5854 		 * Many stacks (ours included) will set the ACK number in an
5855 		 * FIN|ACK if the SYN times out -- no sequence to ACK.
5856 		 */
5857 		ack = dst->seqlo;
5858 	}
5859 
5860 	if (seq == end) {
5861 		/* Ease sequencing restrictions on no data packets */
5862 		seq = src->seqlo;
5863 		data_end = end = seq;
5864 	}
5865 
5866 	ackskew = dst->seqlo - ack;
5867 
5868 	/*
5869 	 * Need to demodulate the sequence numbers in any TCP SACK options
5870 	 * (Selective ACK). We could optionally validate the SACK values
5871 	 * against the current ACK window, either forwards or backwards, but
5872 	 * I'm not confident that SACK has been implemented properly
5873 	 * everywhere. It wouldn't surprise me if several stacks accidentally
5874 	 * SACK too far backwards of previously ACKed data. There really aren't
5875 	 * any security implications of bad SACKing unless the target stack
5876 	 * doesn't validate the option length correctly. Someone trying to
5877 	 * spoof into a TCP connection won't bother blindly sending SACK
5878 	 * options anyway.
5879 	 */
5880 	if (dst->seqdiff && (th->th_off << 2) > sizeof(struct tcphdr)) {
5881 		if (pf_modulate_sack(m, off, pd, th, dst))
5882 			*copyback = 1;
5883 	}
5884 
5885 #define	MAXACKWINDOW (0xffff + 1500)	/* 1500 is an arbitrary fudge factor */
5886 	if (SEQ_GEQ(src->seqhi, data_end) &&
5887 	    /* Last octet inside other's window space */
5888 	    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) &&
5889 	    /* Retrans: not more than one window back */
5890 	    (ackskew >= -MAXACKWINDOW) &&
5891 	    /* Acking not more than one reassembled fragment backwards */
5892 	    (ackskew <= (MAXACKWINDOW << sws)) &&
5893 	    /* Acking not more than one window forward */
5894 	    ((th->th_flags & TH_RST) == 0 || orig_seq == src->seqlo ||
5895 	    (orig_seq == src->seqlo + 1) || (orig_seq + 1 == src->seqlo))) {
5896 	    /* Require an exact/+1 sequence match on resets when possible */
5897 
5898 		if (dst->scrub || src->scrub) {
5899 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
5900 			    *state, src, dst, copyback))
5901 				return (PF_DROP);
5902 		}
5903 
5904 		/* update max window */
5905 		if (src->max_win < win)
5906 			src->max_win = win;
5907 		/* synchronize sequencing */
5908 		if (SEQ_GT(end, src->seqlo))
5909 			src->seqlo = end;
5910 		/* slide the window of what the other end can send */
5911 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
5912 			dst->seqhi = ack + MAX((win << sws), 1);
5913 
5914 		/* update states */
5915 		if (th->th_flags & TH_SYN)
5916 			if (src->state < TCPS_SYN_SENT)
5917 				pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
5918 		if (th->th_flags & TH_FIN)
5919 			if (src->state < TCPS_CLOSING)
5920 				pf_set_protostate(*state, psrc, TCPS_CLOSING);
5921 		if (th->th_flags & TH_ACK) {
5922 			if (dst->state == TCPS_SYN_SENT) {
5923 				pf_set_protostate(*state, pdst,
5924 				    TCPS_ESTABLISHED);
5925 				if (src->state == TCPS_ESTABLISHED &&
5926 				    (*state)->src_node != NULL &&
5927 				    pf_src_connlimit(state)) {
5928 					REASON_SET(reason, PFRES_SRCLIMIT);
5929 					return (PF_DROP);
5930 				}
5931 			} else if (dst->state == TCPS_CLOSING)
5932 				pf_set_protostate(*state, pdst,
5933 				    TCPS_FIN_WAIT_2);
5934 		}
5935 		if (th->th_flags & TH_RST)
5936 			pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
5937 
5938 		/* update expire time */
5939 		(*state)->expire = pf_get_uptime();
5940 		if (src->state >= TCPS_FIN_WAIT_2 &&
5941 		    dst->state >= TCPS_FIN_WAIT_2)
5942 			(*state)->timeout = PFTM_TCP_CLOSED;
5943 		else if (src->state >= TCPS_CLOSING &&
5944 		    dst->state >= TCPS_CLOSING)
5945 			(*state)->timeout = PFTM_TCP_FIN_WAIT;
5946 		else if (src->state < TCPS_ESTABLISHED ||
5947 		    dst->state < TCPS_ESTABLISHED)
5948 			(*state)->timeout = PFTM_TCP_OPENING;
5949 		else if (src->state >= TCPS_CLOSING ||
5950 		    dst->state >= TCPS_CLOSING)
5951 			(*state)->timeout = PFTM_TCP_CLOSING;
5952 		else
5953 			(*state)->timeout = PFTM_TCP_ESTABLISHED;
5954 
5955 		/* Fall through to PASS packet */
5956 
5957 	} else if ((dst->state < TCPS_SYN_SENT ||
5958 		dst->state >= TCPS_FIN_WAIT_2 ||
5959 		src->state >= TCPS_FIN_WAIT_2) &&
5960 	    SEQ_GEQ(src->seqhi + MAXACKWINDOW, data_end) &&
5961 	    /* Within a window forward of the originating packet */
5962 	    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW)) {
5963 	    /* Within a window backward of the originating packet */
5964 
5965 		/*
5966 		 * This currently handles three situations:
5967 		 *  1) Stupid stacks will shotgun SYNs before their peer
5968 		 *     replies.
5969 		 *  2) When PF catches an already established stream (the
5970 		 *     firewall rebooted, the state table was flushed, routes
5971 		 *     changed...)
5972 		 *  3) Packets get funky immediately after the connection
5973 		 *     closes (this should catch Solaris spurious ACK|FINs
5974 		 *     that web servers like to spew after a close)
5975 		 *
5976 		 * This must be a little more careful than the above code
5977 		 * since packet floods will also be caught here. We don't
5978 		 * update the TTL here to mitigate the damage of a packet
5979 		 * flood and so the same code can handle awkward establishment
5980 		 * and a loosened connection close.
5981 		 * In the establishment case, a correct peer response will
5982 		 * validate the connection, go through the normal state code
5983 		 * and keep updating the state TTL.
5984 		 */
5985 
5986 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
5987 			printf("pf: loose state match: ");
5988 			pf_print_state(*state);
5989 			pf_print_flags(th->th_flags);
5990 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
5991 			    "pkts=%llu:%llu dir=%s,%s\n", seq, orig_seq, ack,
5992 			    pd->p_len, ackskew, (unsigned long long)(*state)->packets[0],
5993 			    (unsigned long long)(*state)->packets[1],
5994 			    pd->dir == PF_IN ? "in" : "out",
5995 			    pd->dir == (*state)->direction ? "fwd" : "rev");
5996 		}
5997 
5998 		if (dst->scrub || src->scrub) {
5999 			if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
6000 			    *state, src, dst, copyback))
6001 				return (PF_DROP);
6002 		}
6003 
6004 		/* update max window */
6005 		if (src->max_win < win)
6006 			src->max_win = win;
6007 		/* synchronize sequencing */
6008 		if (SEQ_GT(end, src->seqlo))
6009 			src->seqlo = end;
6010 		/* slide the window of what the other end can send */
6011 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
6012 			dst->seqhi = ack + MAX((win << sws), 1);
6013 
6014 		/*
6015 		 * Cannot set dst->seqhi here since this could be a shotgunned
6016 		 * SYN and not an already established connection.
6017 		 */
6018 
6019 		if (th->th_flags & TH_FIN)
6020 			if (src->state < TCPS_CLOSING)
6021 				pf_set_protostate(*state, psrc, TCPS_CLOSING);
6022 		if (th->th_flags & TH_RST)
6023 			pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
6024 
6025 		/* Fall through to PASS packet */
6026 
6027 	} else {
6028 		if ((*state)->dst.state == TCPS_SYN_SENT &&
6029 		    (*state)->src.state == TCPS_SYN_SENT) {
6030 			/* Send RST for state mismatches during handshake */
6031 			if (!(th->th_flags & TH_RST))
6032 				pf_send_tcp((*state)->rule.ptr, pd->af,
6033 				    pd->dst, pd->src, th->th_dport,
6034 				    th->th_sport, ntohl(th->th_ack), 0,
6035 				    TH_RST, 0, 0,
6036 				    (*state)->rule.ptr->return_ttl, true, 0, 0,
6037 				    (*state)->act.rtableid);
6038 			src->seqlo = 0;
6039 			src->seqhi = 1;
6040 			src->max_win = 1;
6041 		} else if (V_pf_status.debug >= PF_DEBUG_MISC) {
6042 			printf("pf: BAD state: ");
6043 			pf_print_state(*state);
6044 			pf_print_flags(th->th_flags);
6045 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
6046 			    "pkts=%llu:%llu dir=%s,%s\n",
6047 			    seq, orig_seq, ack, pd->p_len, ackskew,
6048 			    (unsigned long long)(*state)->packets[0],
6049 			    (unsigned long long)(*state)->packets[1],
6050 			    pd->dir == PF_IN ? "in" : "out",
6051 			    pd->dir == (*state)->direction ? "fwd" : "rev");
6052 			printf("pf: State failure on: %c %c %c %c | %c %c\n",
6053 			    SEQ_GEQ(src->seqhi, data_end) ? ' ' : '1',
6054 			    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) ?
6055 			    ' ': '2',
6056 			    (ackskew >= -MAXACKWINDOW) ? ' ' : '3',
6057 			    (ackskew <= (MAXACKWINDOW << sws)) ? ' ' : '4',
6058 			    SEQ_GEQ(src->seqhi + MAXACKWINDOW, data_end) ?' ' :'5',
6059 			    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW) ?' ' :'6');
6060 		}
6061 		REASON_SET(reason, PFRES_BADSTATE);
6062 		return (PF_DROP);
6063 	}
6064 
6065 	return (PF_PASS);
6066 }
6067 
6068 static int
6069 pf_tcp_track_sloppy(struct pf_kstate **state, struct pf_pdesc *pd, u_short *reason)
6070 {
6071 	struct tcphdr		*th = &pd->hdr.tcp;
6072 	struct pf_state_peer	*src, *dst;
6073 	u_int8_t		 psrc, pdst;
6074 
6075 	if (pd->dir == (*state)->direction) {
6076 		src = &(*state)->src;
6077 		dst = &(*state)->dst;
6078 		psrc = PF_PEER_SRC;
6079 		pdst = PF_PEER_DST;
6080 	} else {
6081 		src = &(*state)->dst;
6082 		dst = &(*state)->src;
6083 		psrc = PF_PEER_DST;
6084 		pdst = PF_PEER_SRC;
6085 	}
6086 
6087 	if (th->th_flags & TH_SYN)
6088 		if (src->state < TCPS_SYN_SENT)
6089 			pf_set_protostate(*state, psrc, TCPS_SYN_SENT);
6090 	if (th->th_flags & TH_FIN)
6091 		if (src->state < TCPS_CLOSING)
6092 			pf_set_protostate(*state, psrc, TCPS_CLOSING);
6093 	if (th->th_flags & TH_ACK) {
6094 		if (dst->state == TCPS_SYN_SENT) {
6095 			pf_set_protostate(*state, pdst, TCPS_ESTABLISHED);
6096 			if (src->state == TCPS_ESTABLISHED &&
6097 			    (*state)->src_node != NULL &&
6098 			    pf_src_connlimit(state)) {
6099 				REASON_SET(reason, PFRES_SRCLIMIT);
6100 				return (PF_DROP);
6101 			}
6102 		} else if (dst->state == TCPS_CLOSING) {
6103 			pf_set_protostate(*state, pdst, TCPS_FIN_WAIT_2);
6104 		} else if (src->state == TCPS_SYN_SENT &&
6105 		    dst->state < TCPS_SYN_SENT) {
6106 			/*
6107 			 * Handle a special sloppy case where we only see one
6108 			 * half of the connection. If there is a ACK after
6109 			 * the initial SYN without ever seeing a packet from
6110 			 * the destination, set the connection to established.
6111 			 */
6112 			pf_set_protostate(*state, PF_PEER_BOTH,
6113 			    TCPS_ESTABLISHED);
6114 			dst->state = src->state = TCPS_ESTABLISHED;
6115 			if ((*state)->src_node != NULL &&
6116 			    pf_src_connlimit(state)) {
6117 				REASON_SET(reason, PFRES_SRCLIMIT);
6118 				return (PF_DROP);
6119 			}
6120 		} else if (src->state == TCPS_CLOSING &&
6121 		    dst->state == TCPS_ESTABLISHED &&
6122 		    dst->seqlo == 0) {
6123 			/*
6124 			 * Handle the closing of half connections where we
6125 			 * don't see the full bidirectional FIN/ACK+ACK
6126 			 * handshake.
6127 			 */
6128 			pf_set_protostate(*state, pdst, TCPS_CLOSING);
6129 		}
6130 	}
6131 	if (th->th_flags & TH_RST)
6132 		pf_set_protostate(*state, PF_PEER_BOTH, TCPS_TIME_WAIT);
6133 
6134 	/* update expire time */
6135 	(*state)->expire = pf_get_uptime();
6136 	if (src->state >= TCPS_FIN_WAIT_2 &&
6137 	    dst->state >= TCPS_FIN_WAIT_2)
6138 		(*state)->timeout = PFTM_TCP_CLOSED;
6139 	else if (src->state >= TCPS_CLOSING &&
6140 	    dst->state >= TCPS_CLOSING)
6141 		(*state)->timeout = PFTM_TCP_FIN_WAIT;
6142 	else if (src->state < TCPS_ESTABLISHED ||
6143 	    dst->state < TCPS_ESTABLISHED)
6144 		(*state)->timeout = PFTM_TCP_OPENING;
6145 	else if (src->state >= TCPS_CLOSING ||
6146 	    dst->state >= TCPS_CLOSING)
6147 		(*state)->timeout = PFTM_TCP_CLOSING;
6148 	else
6149 		(*state)->timeout = PFTM_TCP_ESTABLISHED;
6150 
6151 	return (PF_PASS);
6152 }
6153 
6154 static int
6155 pf_synproxy(struct pf_pdesc *pd, struct pf_kstate **state, u_short *reason)
6156 {
6157 	struct pf_state_key	*sk = (*state)->key[pd->didx];
6158 	struct tcphdr		*th = &pd->hdr.tcp;
6159 
6160 	if ((*state)->src.state == PF_TCPS_PROXY_SRC) {
6161 		if (pd->dir != (*state)->direction) {
6162 			REASON_SET(reason, PFRES_SYNPROXY);
6163 			return (PF_SYNPROXY_DROP);
6164 		}
6165 		if (th->th_flags & TH_SYN) {
6166 			if (ntohl(th->th_seq) != (*state)->src.seqlo) {
6167 				REASON_SET(reason, PFRES_SYNPROXY);
6168 				return (PF_DROP);
6169 			}
6170 			pf_send_tcp((*state)->rule.ptr, pd->af, pd->dst,
6171 			    pd->src, th->th_dport, th->th_sport,
6172 			    (*state)->src.seqhi, ntohl(th->th_seq) + 1,
6173 			    TH_SYN|TH_ACK, 0, (*state)->src.mss, 0, true, 0, 0,
6174 			    (*state)->act.rtableid);
6175 			REASON_SET(reason, PFRES_SYNPROXY);
6176 			return (PF_SYNPROXY_DROP);
6177 		} else if ((th->th_flags & (TH_ACK|TH_RST|TH_FIN)) != TH_ACK ||
6178 		    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
6179 		    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
6180 			REASON_SET(reason, PFRES_SYNPROXY);
6181 			return (PF_DROP);
6182 		} else if ((*state)->src_node != NULL &&
6183 		    pf_src_connlimit(state)) {
6184 			REASON_SET(reason, PFRES_SRCLIMIT);
6185 			return (PF_DROP);
6186 		} else
6187 			pf_set_protostate(*state, PF_PEER_SRC,
6188 			    PF_TCPS_PROXY_DST);
6189 	}
6190 	if ((*state)->src.state == PF_TCPS_PROXY_DST) {
6191 		if (pd->dir == (*state)->direction) {
6192 			if (((th->th_flags & (TH_SYN|TH_ACK)) != TH_ACK) ||
6193 			    (ntohl(th->th_ack) != (*state)->src.seqhi + 1) ||
6194 			    (ntohl(th->th_seq) != (*state)->src.seqlo + 1)) {
6195 				REASON_SET(reason, PFRES_SYNPROXY);
6196 				return (PF_DROP);
6197 			}
6198 			(*state)->src.max_win = MAX(ntohs(th->th_win), 1);
6199 			if ((*state)->dst.seqhi == 1)
6200 				(*state)->dst.seqhi = htonl(arc4random());
6201 			pf_send_tcp((*state)->rule.ptr, pd->af,
6202 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
6203 			    sk->port[pd->sidx], sk->port[pd->didx],
6204 			    (*state)->dst.seqhi, 0, TH_SYN, 0,
6205 			    (*state)->src.mss, 0, false, (*state)->tag, 0,
6206 			    (*state)->act.rtableid);
6207 			REASON_SET(reason, PFRES_SYNPROXY);
6208 			return (PF_SYNPROXY_DROP);
6209 		} else if (((th->th_flags & (TH_SYN|TH_ACK)) !=
6210 		    (TH_SYN|TH_ACK)) ||
6211 		    (ntohl(th->th_ack) != (*state)->dst.seqhi + 1)) {
6212 			REASON_SET(reason, PFRES_SYNPROXY);
6213 			return (PF_DROP);
6214 		} else {
6215 			(*state)->dst.max_win = MAX(ntohs(th->th_win), 1);
6216 			(*state)->dst.seqlo = ntohl(th->th_seq);
6217 			pf_send_tcp((*state)->rule.ptr, pd->af, pd->dst,
6218 			    pd->src, th->th_dport, th->th_sport,
6219 			    ntohl(th->th_ack), ntohl(th->th_seq) + 1,
6220 			    TH_ACK, (*state)->src.max_win, 0, 0, false,
6221 			    (*state)->tag, 0, (*state)->act.rtableid);
6222 			pf_send_tcp((*state)->rule.ptr, pd->af,
6223 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
6224 			    sk->port[pd->sidx], sk->port[pd->didx],
6225 			    (*state)->src.seqhi + 1, (*state)->src.seqlo + 1,
6226 			    TH_ACK, (*state)->dst.max_win, 0, 0, true, 0, 0,
6227 			    (*state)->act.rtableid);
6228 			(*state)->src.seqdiff = (*state)->dst.seqhi -
6229 			    (*state)->src.seqlo;
6230 			(*state)->dst.seqdiff = (*state)->src.seqhi -
6231 			    (*state)->dst.seqlo;
6232 			(*state)->src.seqhi = (*state)->src.seqlo +
6233 			    (*state)->dst.max_win;
6234 			(*state)->dst.seqhi = (*state)->dst.seqlo +
6235 			    (*state)->src.max_win;
6236 			(*state)->src.wscale = (*state)->dst.wscale = 0;
6237 			pf_set_protostate(*state, PF_PEER_BOTH,
6238 			    TCPS_ESTABLISHED);
6239 			REASON_SET(reason, PFRES_SYNPROXY);
6240 			return (PF_SYNPROXY_DROP);
6241 		}
6242 	}
6243 
6244 	return (PF_PASS);
6245 }
6246 
6247 static int
6248 pf_test_state_tcp(struct pf_kstate **state, struct pfi_kkif *kif,
6249     struct mbuf *m, int off, struct pf_pdesc *pd,
6250     u_short *reason)
6251 {
6252 	struct pf_state_key_cmp	 key;
6253 	struct tcphdr		*th = &pd->hdr.tcp;
6254 	int			 copyback = 0;
6255 	int			 action;
6256 	struct pf_state_peer	*src, *dst;
6257 
6258 	bzero(&key, sizeof(key));
6259 	key.af = pd->af;
6260 	key.proto = IPPROTO_TCP;
6261 	if (pd->dir == PF_IN)	{	/* wire side, straight */
6262 		PF_ACPY(&key.addr[0], pd->src, key.af);
6263 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6264 		key.port[0] = th->th_sport;
6265 		key.port[1] = th->th_dport;
6266 	} else {			/* stack side, reverse */
6267 		PF_ACPY(&key.addr[1], pd->src, key.af);
6268 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6269 		key.port[1] = th->th_sport;
6270 		key.port[0] = th->th_dport;
6271 	}
6272 
6273 	STATE_LOOKUP(kif, &key, *state, pd);
6274 
6275 	if (pd->dir == (*state)->direction) {
6276 		src = &(*state)->src;
6277 		dst = &(*state)->dst;
6278 	} else {
6279 		src = &(*state)->dst;
6280 		dst = &(*state)->src;
6281 	}
6282 
6283 	if ((action = pf_synproxy(pd, state, reason)) != PF_PASS)
6284 		return (action);
6285 
6286 	if (dst->state >= TCPS_FIN_WAIT_2 &&
6287 	    src->state >= TCPS_FIN_WAIT_2 &&
6288 	    (((th->th_flags & (TH_SYN|TH_ACK)) == TH_SYN) ||
6289 	    ((th->th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_ACK &&
6290 	    pf_syncookie_check(pd) && pd->dir == PF_IN))) {
6291 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
6292 			printf("pf: state reuse ");
6293 			pf_print_state(*state);
6294 			pf_print_flags(th->th_flags);
6295 			printf("\n");
6296 		}
6297 		/* XXX make sure it's the same direction ?? */
6298 		pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
6299 		pf_unlink_state(*state);
6300 		*state = NULL;
6301 		return (PF_DROP);
6302 	}
6303 
6304 	if ((*state)->state_flags & PFSTATE_SLOPPY) {
6305 		if (pf_tcp_track_sloppy(state, pd, reason) == PF_DROP)
6306 			return (PF_DROP);
6307 	} else {
6308 		if (pf_tcp_track_full(state, kif, m, off, pd, reason,
6309 		    &copyback) == PF_DROP)
6310 			return (PF_DROP);
6311 	}
6312 
6313 	/* translate source/destination address, if necessary */
6314 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6315 		struct pf_state_key *nk = (*state)->key[pd->didx];
6316 
6317 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
6318 		    nk->port[pd->sidx] != th->th_sport)
6319 			pf_change_ap(m, pd->src, &th->th_sport,
6320 			    pd->ip_sum, &th->th_sum, &nk->addr[pd->sidx],
6321 			    nk->port[pd->sidx], 0, pd->af);
6322 
6323 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
6324 		    nk->port[pd->didx] != th->th_dport)
6325 			pf_change_ap(m, pd->dst, &th->th_dport,
6326 			    pd->ip_sum, &th->th_sum, &nk->addr[pd->didx],
6327 			    nk->port[pd->didx], 0, pd->af);
6328 		copyback = 1;
6329 	}
6330 
6331 	/* Copyback sequence modulation or stateful scrub changes if needed */
6332 	if (copyback)
6333 		m_copyback(m, off, sizeof(*th), (caddr_t)th);
6334 
6335 	return (PF_PASS);
6336 }
6337 
6338 static int
6339 pf_test_state_udp(struct pf_kstate **state, struct pfi_kkif *kif,
6340     struct mbuf *m, int off, struct pf_pdesc *pd)
6341 {
6342 	struct pf_state_peer	*src, *dst;
6343 	struct pf_state_key_cmp	 key;
6344 	struct udphdr		*uh = &pd->hdr.udp;
6345 	uint8_t			 psrc, pdst;
6346 
6347 	bzero(&key, sizeof(key));
6348 	key.af = pd->af;
6349 	key.proto = IPPROTO_UDP;
6350 	if (pd->dir == PF_IN)	{	/* wire side, straight */
6351 		PF_ACPY(&key.addr[0], pd->src, key.af);
6352 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6353 		key.port[0] = uh->uh_sport;
6354 		key.port[1] = uh->uh_dport;
6355 	} else {			/* stack side, reverse */
6356 		PF_ACPY(&key.addr[1], pd->src, key.af);
6357 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6358 		key.port[1] = uh->uh_sport;
6359 		key.port[0] = uh->uh_dport;
6360 	}
6361 
6362 	STATE_LOOKUP(kif, &key, *state, pd);
6363 
6364 	if (pd->dir == (*state)->direction) {
6365 		src = &(*state)->src;
6366 		dst = &(*state)->dst;
6367 		psrc = PF_PEER_SRC;
6368 		pdst = PF_PEER_DST;
6369 	} else {
6370 		src = &(*state)->dst;
6371 		dst = &(*state)->src;
6372 		psrc = PF_PEER_DST;
6373 		pdst = PF_PEER_SRC;
6374 	}
6375 
6376 	/* update states */
6377 	if (src->state < PFUDPS_SINGLE)
6378 		pf_set_protostate(*state, psrc, PFUDPS_SINGLE);
6379 	if (dst->state == PFUDPS_SINGLE)
6380 		pf_set_protostate(*state, pdst, PFUDPS_MULTIPLE);
6381 
6382 	/* update expire time */
6383 	(*state)->expire = pf_get_uptime();
6384 	if (src->state == PFUDPS_MULTIPLE && dst->state == PFUDPS_MULTIPLE)
6385 		(*state)->timeout = PFTM_UDP_MULTIPLE;
6386 	else
6387 		(*state)->timeout = PFTM_UDP_SINGLE;
6388 
6389 	/* translate source/destination address, if necessary */
6390 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6391 		struct pf_state_key *nk = (*state)->key[pd->didx];
6392 
6393 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
6394 		    nk->port[pd->sidx] != uh->uh_sport)
6395 			pf_change_ap(m, pd->src, &uh->uh_sport, pd->ip_sum,
6396 			    &uh->uh_sum, &nk->addr[pd->sidx],
6397 			    nk->port[pd->sidx], 1, pd->af);
6398 
6399 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
6400 		    nk->port[pd->didx] != uh->uh_dport)
6401 			pf_change_ap(m, pd->dst, &uh->uh_dport, pd->ip_sum,
6402 			    &uh->uh_sum, &nk->addr[pd->didx],
6403 			    nk->port[pd->didx], 1, pd->af);
6404 		m_copyback(m, off, sizeof(*uh), (caddr_t)uh);
6405 	}
6406 
6407 	return (PF_PASS);
6408 }
6409 
6410 static int
6411 pf_test_state_sctp(struct pf_kstate **state, struct pfi_kkif *kif,
6412     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason)
6413 {
6414 	struct pf_state_key_cmp	 key;
6415 	struct pf_state_peer	*src, *dst;
6416 	struct sctphdr		*sh = &pd->hdr.sctp;
6417 	u_int8_t		 psrc; //, pdst;
6418 
6419 	bzero(&key, sizeof(key));
6420 	key.af = pd->af;
6421 	key.proto = IPPROTO_SCTP;
6422 	if (pd->dir == PF_IN)	{	/* wire side, straight */
6423 		PF_ACPY(&key.addr[0], pd->src, key.af);
6424 		PF_ACPY(&key.addr[1], pd->dst, key.af);
6425 		key.port[0] = sh->src_port;
6426 		key.port[1] = sh->dest_port;
6427 	} else {			/* stack side, reverse */
6428 		PF_ACPY(&key.addr[1], pd->src, key.af);
6429 		PF_ACPY(&key.addr[0], pd->dst, key.af);
6430 		key.port[1] = sh->src_port;
6431 		key.port[0] = sh->dest_port;
6432 	}
6433 
6434 	STATE_LOOKUP(kif, &key, *state, pd);
6435 
6436 	if (pd->dir == (*state)->direction) {
6437 		src = &(*state)->src;
6438 		dst = &(*state)->dst;
6439 		psrc = PF_PEER_SRC;
6440 	} else {
6441 		src = &(*state)->dst;
6442 		dst = &(*state)->src;
6443 		psrc = PF_PEER_DST;
6444 	}
6445 
6446 	if ((src->state >= SCTP_SHUTDOWN_SENT || src->state == SCTP_CLOSED) &&
6447 	    (dst->state >= SCTP_SHUTDOWN_SENT || dst->state == SCTP_CLOSED) &&
6448 	    pd->sctp_flags & PFDESC_SCTP_INIT) {
6449 		pf_set_protostate(*state, PF_PEER_BOTH, SCTP_CLOSED);
6450 		pf_unlink_state(*state);
6451 		*state = NULL;
6452 		return (PF_DROP);
6453 	}
6454 
6455 	/* Track state. */
6456 	if (pd->sctp_flags & PFDESC_SCTP_INIT) {
6457 		if (src->state < SCTP_COOKIE_WAIT) {
6458 			pf_set_protostate(*state, psrc, SCTP_COOKIE_WAIT);
6459 			(*state)->timeout = PFTM_SCTP_OPENING;
6460 		}
6461 	}
6462 	if (pd->sctp_flags & PFDESC_SCTP_INIT_ACK) {
6463 		MPASS(dst->scrub != NULL);
6464 		if (dst->scrub->pfss_v_tag == 0)
6465 			dst->scrub->pfss_v_tag = pd->sctp_initiate_tag;
6466 	}
6467 
6468 	if (pd->sctp_flags & (PFDESC_SCTP_COOKIE | PFDESC_SCTP_HEARTBEAT_ACK)) {
6469 		if (src->state < SCTP_ESTABLISHED) {
6470 			pf_set_protostate(*state, psrc, SCTP_ESTABLISHED);
6471 			(*state)->timeout = PFTM_SCTP_ESTABLISHED;
6472 		}
6473 	}
6474 	if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN | PFDESC_SCTP_ABORT |
6475 	    PFDESC_SCTP_SHUTDOWN_COMPLETE)) {
6476 		if (src->state < SCTP_SHUTDOWN_PENDING) {
6477 			pf_set_protostate(*state, psrc, SCTP_SHUTDOWN_PENDING);
6478 			(*state)->timeout = PFTM_SCTP_CLOSING;
6479 		}
6480 	}
6481 	if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN_COMPLETE)) {
6482 		pf_set_protostate(*state, psrc, SCTP_CLOSED);
6483 		(*state)->timeout = PFTM_SCTP_CLOSED;
6484 	}
6485 
6486 	if (src->scrub != NULL) {
6487 		if (src->scrub->pfss_v_tag == 0) {
6488 			src->scrub->pfss_v_tag = pd->hdr.sctp.v_tag;
6489 		} else  if (src->scrub->pfss_v_tag != pd->hdr.sctp.v_tag)
6490 			return (PF_DROP);
6491 	}
6492 
6493 	(*state)->expire = pf_get_uptime();
6494 
6495 	/* translate source/destination address, if necessary */
6496 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
6497 		uint16_t checksum = 0;
6498 		struct pf_state_key *nk = (*state)->key[pd->didx];
6499 
6500 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af) ||
6501 		    nk->port[pd->sidx] != pd->hdr.sctp.src_port) {
6502 			pf_change_ap(m, pd->src, &pd->hdr.sctp.src_port,
6503 			    pd->ip_sum, &checksum, &nk->addr[pd->sidx],
6504 			    nk->port[pd->sidx], 1, pd->af);
6505 		}
6506 
6507 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af) ||
6508 		    nk->port[pd->didx] != pd->hdr.sctp.dest_port) {
6509 			pf_change_ap(m, pd->dst, &pd->hdr.sctp.dest_port,
6510 			    pd->ip_sum, &checksum, &nk->addr[pd->didx],
6511 			    nk->port[pd->didx], 1, pd->af);
6512 		}
6513 	}
6514 
6515 	return (PF_PASS);
6516 }
6517 
6518 static void
6519 pf_sctp_multihome_detach_addr(const struct pf_kstate *s)
6520 {
6521 	struct pf_sctp_endpoint key;
6522 	struct pf_sctp_endpoint *ep;
6523 	struct pf_state_key *sks = s->key[PF_SK_STACK];
6524 	struct pf_sctp_source *i, *tmp;
6525 
6526 	if (sks == NULL || sks->proto != IPPROTO_SCTP || s->dst.scrub == NULL)
6527 		return;
6528 
6529 	PF_SCTP_ENDPOINTS_LOCK();
6530 
6531 	key.v_tag = s->dst.scrub->pfss_v_tag;
6532 	ep  = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6533 	if (ep != NULL) {
6534 		TAILQ_FOREACH_SAFE(i, &ep->sources, entry, tmp) {
6535 			if (pf_addr_cmp(&i->addr,
6536 			    &s->key[PF_SK_WIRE]->addr[s->direction == PF_OUT],
6537 			    s->key[PF_SK_WIRE]->af) == 0) {
6538 				SDT_PROBE3(pf, sctp, multihome, remove,
6539 				    key.v_tag, s, i);
6540 				TAILQ_REMOVE(&ep->sources, i, entry);
6541 				free(i, M_PFTEMP);
6542 				break;
6543 			}
6544 		}
6545 
6546 		if (TAILQ_EMPTY(&ep->sources)) {
6547 			RB_REMOVE(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
6548 			free(ep, M_PFTEMP);
6549 		}
6550 	}
6551 
6552 	/* Other direction. */
6553 	key.v_tag = s->src.scrub->pfss_v_tag;
6554 	ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6555 	if (ep != NULL) {
6556 		TAILQ_FOREACH_SAFE(i, &ep->sources, entry, tmp) {
6557 			if (pf_addr_cmp(&i->addr,
6558 			    &s->key[PF_SK_WIRE]->addr[s->direction == PF_IN],
6559 			    s->key[PF_SK_WIRE]->af) == 0) {
6560 				SDT_PROBE3(pf, sctp, multihome, remove,
6561 				    key.v_tag, s, i);
6562 				TAILQ_REMOVE(&ep->sources, i, entry);
6563 				free(i, M_PFTEMP);
6564 				break;
6565 			}
6566 		}
6567 
6568 		if (TAILQ_EMPTY(&ep->sources)) {
6569 			RB_REMOVE(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
6570 			free(ep, M_PFTEMP);
6571 		}
6572 	}
6573 
6574 	PF_SCTP_ENDPOINTS_UNLOCK();
6575 }
6576 
6577 static void
6578 pf_sctp_multihome_add_addr(struct pf_pdesc *pd, struct pf_addr *a, uint32_t v_tag)
6579 {
6580 	struct pf_sctp_endpoint key = {
6581 		.v_tag = v_tag,
6582 	};
6583 	struct pf_sctp_source *i;
6584 	struct pf_sctp_endpoint *ep;
6585 
6586 	PF_SCTP_ENDPOINTS_LOCK();
6587 
6588 	ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6589 	if (ep == NULL) {
6590 		ep = malloc(sizeof(struct pf_sctp_endpoint),
6591 		    M_PFTEMP, M_NOWAIT);
6592 		if (ep == NULL) {
6593 			PF_SCTP_ENDPOINTS_UNLOCK();
6594 			return;
6595 		}
6596 
6597 		ep->v_tag = v_tag;
6598 		TAILQ_INIT(&ep->sources);
6599 		RB_INSERT(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
6600 	}
6601 
6602 	/* Avoid inserting duplicates. */
6603 	TAILQ_FOREACH(i, &ep->sources, entry) {
6604 		if (pf_addr_cmp(&i->addr, a, pd->af) == 0) {
6605 			PF_SCTP_ENDPOINTS_UNLOCK();
6606 			return;
6607 		}
6608 	}
6609 
6610 	i = malloc(sizeof(*i), M_PFTEMP, M_NOWAIT);
6611 	if (i == NULL) {
6612 		PF_SCTP_ENDPOINTS_UNLOCK();
6613 		return;
6614 	}
6615 
6616 	i->af = pd->af;
6617 	memcpy(&i->addr, a, sizeof(*a));
6618 	TAILQ_INSERT_TAIL(&ep->sources, i, entry);
6619 	SDT_PROBE2(pf, sctp, multihome, add, v_tag, i);
6620 
6621 	PF_SCTP_ENDPOINTS_UNLOCK();
6622 }
6623 
6624 static void
6625 pf_sctp_multihome_delayed(struct pf_pdesc *pd, int off, struct pfi_kkif *kif,
6626     struct pf_kstate *s, int action)
6627 {
6628 	struct pf_sctp_multihome_job	*j, *tmp;
6629 	struct pf_sctp_source		*i;
6630 	int			 ret __unused;
6631 	struct pf_kstate	*sm = NULL;
6632 	struct pf_krule		*ra = NULL;
6633 	struct pf_krule		*r = &V_pf_default_rule;
6634 	struct pf_kruleset	*rs = NULL;
6635 	bool do_extra = true;
6636 
6637 	PF_RULES_RLOCK_TRACKER;
6638 
6639 again:
6640 	TAILQ_FOREACH_SAFE(j, &pd->sctp_multihome_jobs, next, tmp) {
6641 		if (s == NULL || action != PF_PASS)
6642 			goto free;
6643 
6644 		/* Confirm we don't recurse here. */
6645 		MPASS(! (pd->sctp_flags & PFDESC_SCTP_ADD_IP));
6646 
6647 		switch (j->op) {
6648 		case  SCTP_ADD_IP_ADDRESS: {
6649 			uint32_t v_tag = pd->sctp_initiate_tag;
6650 
6651 			if (v_tag == 0) {
6652 				if (s->direction == pd->dir)
6653 					v_tag = s->src.scrub->pfss_v_tag;
6654 				else
6655 					v_tag = s->dst.scrub->pfss_v_tag;
6656 			}
6657 
6658 			/*
6659 			 * Avoid duplicating states. We'll already have
6660 			 * created a state based on the source address of
6661 			 * the packet, but SCTP endpoints may also list this
6662 			 * address again in the INIT(_ACK) parameters.
6663 			 */
6664 			if (pf_addr_cmp(&j->src, pd->src, pd->af) == 0) {
6665 				break;
6666 			}
6667 
6668 			j->pd.sctp_flags |= PFDESC_SCTP_ADD_IP;
6669 			PF_RULES_RLOCK();
6670 			sm = NULL;
6671 			/*
6672 			 * New connections need to be floating, because
6673 			 * we cannot know what interfaces it will use.
6674 			 * That's why we pass V_pfi_all rather than kif.
6675 			 */
6676 			ret = pf_test_rule(&r, &sm, V_pfi_all,
6677 			    j->m, off, &j->pd, &ra, &rs, NULL,
6678 			    sizeof(j->pd.hdr.sctp));
6679 			PF_RULES_RUNLOCK();
6680 			SDT_PROBE4(pf, sctp, multihome, test, kif, r, j->m, ret);
6681 			if (ret != PF_DROP && sm != NULL) {
6682 				/* Inherit v_tag values. */
6683 				if (sm->direction == s->direction) {
6684 					sm->src.scrub->pfss_v_tag = s->src.scrub->pfss_v_tag;
6685 					sm->dst.scrub->pfss_v_tag = s->dst.scrub->pfss_v_tag;
6686 				} else {
6687 					sm->src.scrub->pfss_v_tag = s->dst.scrub->pfss_v_tag;
6688 					sm->dst.scrub->pfss_v_tag = s->src.scrub->pfss_v_tag;
6689 				}
6690 				PF_STATE_UNLOCK(sm);
6691 			} else {
6692 				/* If we try duplicate inserts? */
6693 				break;
6694 			}
6695 
6696 			/* Only add the address if we've actually allowed the state. */
6697 			pf_sctp_multihome_add_addr(pd, &j->src, v_tag);
6698 
6699 			if (! do_extra) {
6700 				break;
6701 			}
6702 			/*
6703 			 * We need to do this for each of our source addresses.
6704 			 * Find those based on the verification tag.
6705 			 */
6706 			struct pf_sctp_endpoint key = {
6707 				.v_tag = pd->hdr.sctp.v_tag,
6708 			};
6709 			struct pf_sctp_endpoint *ep;
6710 
6711 			PF_SCTP_ENDPOINTS_LOCK();
6712 			ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
6713 			if (ep == NULL) {
6714 				PF_SCTP_ENDPOINTS_UNLOCK();
6715 				break;
6716 			}
6717 			MPASS(ep != NULL);
6718 
6719 			TAILQ_FOREACH(i, &ep->sources, entry) {
6720 				struct pf_sctp_multihome_job *nj;
6721 
6722 				/* SCTP can intermingle IPv4 and IPv6. */
6723 				if (i->af != pd->af)
6724 					continue;
6725 
6726 				nj = malloc(sizeof(*nj), M_PFTEMP, M_NOWAIT | M_ZERO);
6727 				if (! nj) {
6728 					continue;
6729 				}
6730 				memcpy(&nj->pd, &j->pd, sizeof(j->pd));
6731 				memcpy(&nj->src, &j->src, sizeof(nj->src));
6732 				nj->pd.src = &nj->src;
6733 				// New destination address!
6734 				memcpy(&nj->dst, &i->addr, sizeof(nj->dst));
6735 				nj->pd.dst = &nj->dst;
6736 				nj->m = j->m;
6737 				nj->op = j->op;
6738 
6739 				TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, nj, next);
6740 			}
6741 			PF_SCTP_ENDPOINTS_UNLOCK();
6742 
6743 			break;
6744 		}
6745 		case SCTP_DEL_IP_ADDRESS: {
6746 			struct pf_state_key_cmp key;
6747 			uint8_t psrc;
6748 
6749 			bzero(&key, sizeof(key));
6750 			key.af = j->pd.af;
6751 			key.proto = IPPROTO_SCTP;
6752 			if (j->pd.dir == PF_IN)	{	/* wire side, straight */
6753 				PF_ACPY(&key.addr[0], j->pd.src, key.af);
6754 				PF_ACPY(&key.addr[1], j->pd.dst, key.af);
6755 				key.port[0] = j->pd.hdr.sctp.src_port;
6756 				key.port[1] = j->pd.hdr.sctp.dest_port;
6757 			} else {			/* stack side, reverse */
6758 				PF_ACPY(&key.addr[1], j->pd.src, key.af);
6759 				PF_ACPY(&key.addr[0], j->pd.dst, key.af);
6760 				key.port[1] = j->pd.hdr.sctp.src_port;
6761 				key.port[0] = j->pd.hdr.sctp.dest_port;
6762 			}
6763 
6764 			sm = pf_find_state(kif, &key, j->pd.dir);
6765 			if (sm != NULL) {
6766 				PF_STATE_LOCK_ASSERT(sm);
6767 				if (j->pd.dir == sm->direction) {
6768 					psrc = PF_PEER_SRC;
6769 				} else {
6770 					psrc = PF_PEER_DST;
6771 				}
6772 				pf_set_protostate(sm, psrc, SCTP_SHUTDOWN_PENDING);
6773 				sm->timeout = PFTM_SCTP_CLOSING;
6774 				PF_STATE_UNLOCK(sm);
6775 			}
6776 			break;
6777 		default:
6778 			panic("Unknown op %#x", j->op);
6779 		}
6780 	}
6781 
6782 	free:
6783 		TAILQ_REMOVE(&pd->sctp_multihome_jobs, j, next);
6784 		free(j, M_PFTEMP);
6785 	}
6786 
6787 	/* We may have inserted extra work while processing the list. */
6788 	if (! TAILQ_EMPTY(&pd->sctp_multihome_jobs)) {
6789 		do_extra = false;
6790 		goto again;
6791 	}
6792 }
6793 
6794 static int
6795 pf_multihome_scan(struct mbuf *m, int start, int len, struct pf_pdesc *pd,
6796     struct pfi_kkif *kif, int op)
6797 {
6798 	int			 off = 0;
6799 	struct pf_sctp_multihome_job	*job;
6800 
6801 	while (off < len) {
6802 		struct sctp_paramhdr h;
6803 
6804 		if (!pf_pull_hdr(m, start + off, &h, sizeof(h), NULL, NULL,
6805 		    pd->af))
6806 			return (PF_DROP);
6807 
6808 		/* Parameters are at least 4 bytes. */
6809 		if (ntohs(h.param_length) < 4)
6810 			return (PF_DROP);
6811 
6812 		switch (ntohs(h.param_type)) {
6813 		case  SCTP_IPV4_ADDRESS: {
6814 			struct in_addr t;
6815 
6816 			if (ntohs(h.param_length) !=
6817 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
6818 				return (PF_DROP);
6819 
6820 			if (!pf_pull_hdr(m, start + off + sizeof(h), &t, sizeof(t),
6821 			    NULL, NULL, pd->af))
6822 				return (PF_DROP);
6823 
6824 			if (in_nullhost(t))
6825 				t.s_addr = pd->src->v4.s_addr;
6826 
6827 			/*
6828 			 * We hold the state lock (idhash) here, which means
6829 			 * that we can't acquire the keyhash, or we'll get a
6830 			 * LOR (and potentially double-lock things too). We also
6831 			 * can't release the state lock here, so instead we'll
6832 			 * enqueue this for async handling.
6833 			 * There's a relatively small race here, in that a
6834 			 * packet using the new addresses could arrive already,
6835 			 * but that's just though luck for it.
6836 			 */
6837 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
6838 			if (! job)
6839 				return (PF_DROP);
6840 
6841 			memcpy(&job->pd, pd, sizeof(*pd));
6842 
6843 			// New source address!
6844 			memcpy(&job->src, &t, sizeof(t));
6845 			job->pd.src = &job->src;
6846 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
6847 			job->pd.dst = &job->dst;
6848 			job->m = m;
6849 			job->op = op;
6850 
6851 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
6852 			break;
6853 		}
6854 #ifdef INET6
6855 		case SCTP_IPV6_ADDRESS: {
6856 			struct in6_addr t;
6857 
6858 			if (ntohs(h.param_length) !=
6859 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
6860 				return (PF_DROP);
6861 
6862 			if (!pf_pull_hdr(m, start + off + sizeof(h), &t, sizeof(t),
6863 			    NULL, NULL, pd->af))
6864 				return (PF_DROP);
6865 			if (memcmp(&t, &pd->src->v6, sizeof(t)) == 0)
6866 				break;
6867 			if (memcmp(&t, &in6addr_any, sizeof(t)) == 0)
6868 				memcpy(&t, &pd->src->v6, sizeof(t));
6869 
6870 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
6871 			if (! job)
6872 				return (PF_DROP);
6873 
6874 			memcpy(&job->pd, pd, sizeof(*pd));
6875 			memcpy(&job->src, &t, sizeof(t));
6876 			job->pd.src = &job->src;
6877 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
6878 			job->pd.dst = &job->dst;
6879 			job->m = m;
6880 			job->op = op;
6881 
6882 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
6883 			break;
6884 		}
6885 #endif
6886 		case SCTP_ADD_IP_ADDRESS: {
6887 			int ret;
6888 			struct sctp_asconf_paramhdr ah;
6889 
6890 			if (!pf_pull_hdr(m, start + off, &ah, sizeof(ah),
6891 			    NULL, NULL, pd->af))
6892 				return (PF_DROP);
6893 
6894 			ret = pf_multihome_scan(m, start + off + sizeof(ah),
6895 			    ntohs(ah.ph.param_length) - sizeof(ah), pd, kif,
6896 			    SCTP_ADD_IP_ADDRESS);
6897 			if (ret != PF_PASS)
6898 				return (ret);
6899 			break;
6900 		}
6901 		case SCTP_DEL_IP_ADDRESS: {
6902 			int ret;
6903 			struct sctp_asconf_paramhdr ah;
6904 
6905 			if (!pf_pull_hdr(m, start + off, &ah, sizeof(ah),
6906 			    NULL, NULL, pd->af))
6907 				return (PF_DROP);
6908 			ret = pf_multihome_scan(m, start + off + sizeof(ah),
6909 			    ntohs(ah.ph.param_length) - sizeof(ah), pd, kif,
6910 			    SCTP_DEL_IP_ADDRESS);
6911 			if (ret != PF_PASS)
6912 				return (ret);
6913 			break;
6914 		}
6915 		default:
6916 			break;
6917 		}
6918 
6919 		off += roundup(ntohs(h.param_length), 4);
6920 	}
6921 
6922 	return (PF_PASS);
6923 }
6924 int
6925 pf_multihome_scan_init(struct mbuf *m, int start, int len, struct pf_pdesc *pd,
6926     struct pfi_kkif *kif)
6927 {
6928 	start += sizeof(struct sctp_init_chunk);
6929 	len -= sizeof(struct sctp_init_chunk);
6930 
6931 	return (pf_multihome_scan(m, start, len, pd, kif, SCTP_ADD_IP_ADDRESS));
6932 }
6933 
6934 int
6935 pf_multihome_scan_asconf(struct mbuf *m, int start, int len,
6936     struct pf_pdesc *pd, struct pfi_kkif *kif)
6937 {
6938 	start += sizeof(struct sctp_asconf_chunk);
6939 	len -= sizeof(struct sctp_asconf_chunk);
6940 
6941 	return (pf_multihome_scan(m, start, len, pd, kif, SCTP_ADD_IP_ADDRESS));
6942 }
6943 
6944 int
6945 pf_icmp_state_lookup(struct pf_state_key_cmp *key, struct pf_pdesc *pd,
6946     struct pf_kstate **state, struct mbuf *m, int off, int direction,
6947     struct pfi_kkif *kif, u_int16_t icmpid, u_int16_t type, int icmp_dir,
6948     int *iidx, int multi, int inner)
6949 {
6950 	key->af = pd->af;
6951 	key->proto = pd->proto;
6952 	if (icmp_dir == PF_IN) {
6953 		*iidx = pd->sidx;
6954 		key->port[pd->sidx] = icmpid;
6955 		key->port[pd->didx] = type;
6956 	} else {
6957 		*iidx = pd->didx;
6958 		key->port[pd->sidx] = type;
6959 		key->port[pd->didx] = icmpid;
6960 	}
6961 	if (pf_state_key_addr_setup(pd, m, off, key, pd->sidx, pd->src,
6962 	    pd->didx, pd->dst, multi))
6963 		return (PF_DROP);
6964 
6965 	STATE_LOOKUP(kif, key, *state, pd);
6966 
6967 	if ((*state)->state_flags & PFSTATE_SLOPPY)
6968 		return (-1);
6969 
6970 	/* Is this ICMP message flowing in right direction? */
6971 	if ((*state)->rule.ptr->type &&
6972 	    (((!inner && (*state)->direction == direction) ||
6973 	    (inner && (*state)->direction != direction)) ?
6974 	    PF_IN : PF_OUT) != icmp_dir) {
6975 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
6976 			printf("pf: icmp type %d in wrong direction (%d): ",
6977 			    ntohs(type), icmp_dir);
6978 			pf_print_state(*state);
6979 			printf("\n");
6980 		}
6981 		PF_STATE_UNLOCK(*state);
6982 		*state = NULL;
6983 		return (PF_DROP);
6984 	}
6985 	return (-1);
6986 }
6987 
6988 static int
6989 pf_test_state_icmp(struct pf_kstate **state, struct pfi_kkif *kif,
6990     struct mbuf *m, int off, struct pf_pdesc *pd, u_short *reason)
6991 {
6992 	struct pf_addr  *saddr = pd->src, *daddr = pd->dst;
6993 	u_int16_t	*icmpsum, virtual_id, virtual_type;
6994 	u_int8_t	 icmptype, icmpcode;
6995 	int		 icmp_dir, iidx, ret, multi;
6996 	struct pf_state_key_cmp key;
6997 #ifdef INET
6998 	u_int16_t	 icmpid;
6999 #endif
7000 
7001 	MPASS(*state == NULL);
7002 
7003 	bzero(&key, sizeof(key));
7004 	switch (pd->proto) {
7005 #ifdef INET
7006 	case IPPROTO_ICMP:
7007 		icmptype = pd->hdr.icmp.icmp_type;
7008 		icmpcode = pd->hdr.icmp.icmp_code;
7009 		icmpid = pd->hdr.icmp.icmp_id;
7010 		icmpsum = &pd->hdr.icmp.icmp_cksum;
7011 		break;
7012 #endif /* INET */
7013 #ifdef INET6
7014 	case IPPROTO_ICMPV6:
7015 		icmptype = pd->hdr.icmp6.icmp6_type;
7016 		icmpcode = pd->hdr.icmp6.icmp6_code;
7017 #ifdef INET
7018 		icmpid = pd->hdr.icmp6.icmp6_id;
7019 #endif
7020 		icmpsum = &pd->hdr.icmp6.icmp6_cksum;
7021 		break;
7022 #endif /* INET6 */
7023 	}
7024 
7025 	if (pf_icmp_mapping(pd, icmptype, &icmp_dir, &multi,
7026 	    &virtual_id, &virtual_type) == 0) {
7027 		/*
7028 		 * ICMP query/reply message not related to a TCP/UDP packet.
7029 		 * Search for an ICMP state.
7030 		 */
7031 		ret = pf_icmp_state_lookup(&key, pd, state, m, off, pd->dir,
7032 		    kif, virtual_id, virtual_type, icmp_dir, &iidx,
7033 		    PF_ICMP_MULTI_NONE, 0);
7034 		if (ret >= 0) {
7035 			MPASS(*state == NULL);
7036 			if (ret == PF_DROP && pd->af == AF_INET6 &&
7037 			    icmp_dir == PF_OUT) {
7038 				ret = pf_icmp_state_lookup(&key, pd, state, m, off,
7039 				    pd->dir, kif, virtual_id, virtual_type,
7040 				    icmp_dir, &iidx, multi, 0);
7041 				if (ret >= 0) {
7042 					MPASS(*state == NULL);
7043 					return (ret);
7044 				}
7045 			} else
7046 				return (ret);
7047 		}
7048 
7049 		(*state)->expire = pf_get_uptime();
7050 		(*state)->timeout = PFTM_ICMP_ERROR_REPLY;
7051 
7052 		/* translate source/destination address, if necessary */
7053 		if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
7054 			struct pf_state_key *nk = (*state)->key[pd->didx];
7055 
7056 			switch (pd->af) {
7057 #ifdef INET
7058 			case AF_INET:
7059 				if (PF_ANEQ(pd->src,
7060 				    &nk->addr[pd->sidx], AF_INET))
7061 					pf_change_a(&saddr->v4.s_addr,
7062 					    pd->ip_sum,
7063 					    nk->addr[pd->sidx].v4.s_addr, 0);
7064 
7065 				if (PF_ANEQ(pd->dst, &nk->addr[pd->didx],
7066 				    AF_INET))
7067 					pf_change_a(&daddr->v4.s_addr,
7068 					    pd->ip_sum,
7069 					    nk->addr[pd->didx].v4.s_addr, 0);
7070 
7071 				if (nk->port[iidx] !=
7072 				    pd->hdr.icmp.icmp_id) {
7073 					pd->hdr.icmp.icmp_cksum =
7074 					    pf_cksum_fixup(
7075 					    pd->hdr.icmp.icmp_cksum, icmpid,
7076 					    nk->port[iidx], 0);
7077 					pd->hdr.icmp.icmp_id =
7078 					    nk->port[iidx];
7079 				}
7080 
7081 				m_copyback(m, off, ICMP_MINLEN,
7082 				    (caddr_t )&pd->hdr.icmp);
7083 				break;
7084 #endif /* INET */
7085 #ifdef INET6
7086 			case AF_INET6:
7087 				if (PF_ANEQ(pd->src,
7088 				    &nk->addr[pd->sidx], AF_INET6))
7089 					pf_change_a6(saddr,
7090 					    &pd->hdr.icmp6.icmp6_cksum,
7091 					    &nk->addr[pd->sidx], 0);
7092 
7093 				if (PF_ANEQ(pd->dst,
7094 				    &nk->addr[pd->didx], AF_INET6))
7095 					pf_change_a6(daddr,
7096 					    &pd->hdr.icmp6.icmp6_cksum,
7097 					    &nk->addr[pd->didx], 0);
7098 
7099 				m_copyback(m, off, sizeof(struct icmp6_hdr),
7100 				    (caddr_t )&pd->hdr.icmp6);
7101 				break;
7102 #endif /* INET6 */
7103 			}
7104 		}
7105 		return (PF_PASS);
7106 
7107 	} else {
7108 		/*
7109 		 * ICMP error message in response to a TCP/UDP packet.
7110 		 * Extract the inner TCP/UDP header and search for that state.
7111 		 */
7112 
7113 		struct pf_pdesc	pd2;
7114 		bzero(&pd2, sizeof pd2);
7115 #ifdef INET
7116 		struct ip	h2;
7117 #endif /* INET */
7118 #ifdef INET6
7119 		struct ip6_hdr	h2_6;
7120 		int		terminal = 0;
7121 #endif /* INET6 */
7122 		int		ipoff2 = 0;
7123 		int		off2 = 0;
7124 
7125 		pd2.af = pd->af;
7126 		pd2.dir = pd->dir;
7127 		/* Payload packet is from the opposite direction. */
7128 		pd2.sidx = (pd->dir == PF_IN) ? 1 : 0;
7129 		pd2.didx = (pd->dir == PF_IN) ? 0 : 1;
7130 		switch (pd->af) {
7131 #ifdef INET
7132 		case AF_INET:
7133 			/* offset of h2 in mbuf chain */
7134 			ipoff2 = off + ICMP_MINLEN;
7135 
7136 			if (!pf_pull_hdr(m, ipoff2, &h2, sizeof(h2),
7137 			    NULL, reason, pd2.af)) {
7138 				DPFPRINTF(PF_DEBUG_MISC,
7139 				    ("pf: ICMP error message too short "
7140 				    "(ip)\n"));
7141 				return (PF_DROP);
7142 			}
7143 			/*
7144 			 * ICMP error messages don't refer to non-first
7145 			 * fragments
7146 			 */
7147 			if (h2.ip_off & htons(IP_OFFMASK)) {
7148 				REASON_SET(reason, PFRES_FRAG);
7149 				return (PF_DROP);
7150 			}
7151 
7152 			/* offset of protocol header that follows h2 */
7153 			off2 = ipoff2 + (h2.ip_hl << 2);
7154 
7155 			pd2.proto = h2.ip_p;
7156 			pd2.src = (struct pf_addr *)&h2.ip_src;
7157 			pd2.dst = (struct pf_addr *)&h2.ip_dst;
7158 			pd2.ip_sum = &h2.ip_sum;
7159 			break;
7160 #endif /* INET */
7161 #ifdef INET6
7162 		case AF_INET6:
7163 			ipoff2 = off + sizeof(struct icmp6_hdr);
7164 
7165 			if (!pf_pull_hdr(m, ipoff2, &h2_6, sizeof(h2_6),
7166 			    NULL, reason, pd2.af)) {
7167 				DPFPRINTF(PF_DEBUG_MISC,
7168 				    ("pf: ICMP error message too short "
7169 				    "(ip6)\n"));
7170 				return (PF_DROP);
7171 			}
7172 			pd2.proto = h2_6.ip6_nxt;
7173 			pd2.src = (struct pf_addr *)&h2_6.ip6_src;
7174 			pd2.dst = (struct pf_addr *)&h2_6.ip6_dst;
7175 			pd2.ip_sum = NULL;
7176 			off2 = ipoff2 + sizeof(h2_6);
7177 			do {
7178 				switch (pd2.proto) {
7179 				case IPPROTO_FRAGMENT:
7180 					/*
7181 					 * ICMPv6 error messages for
7182 					 * non-first fragments
7183 					 */
7184 					REASON_SET(reason, PFRES_FRAG);
7185 					return (PF_DROP);
7186 				case IPPROTO_AH:
7187 				case IPPROTO_HOPOPTS:
7188 				case IPPROTO_ROUTING:
7189 				case IPPROTO_DSTOPTS: {
7190 					/* get next header and header length */
7191 					struct ip6_ext opt6;
7192 
7193 					if (!pf_pull_hdr(m, off2, &opt6,
7194 					    sizeof(opt6), NULL, reason,
7195 					    pd2.af)) {
7196 						DPFPRINTF(PF_DEBUG_MISC,
7197 						    ("pf: ICMPv6 short opt\n"));
7198 						return (PF_DROP);
7199 					}
7200 					if (pd2.proto == IPPROTO_AH)
7201 						off2 += (opt6.ip6e_len + 2) * 4;
7202 					else
7203 						off2 += (opt6.ip6e_len + 1) * 8;
7204 					pd2.proto = opt6.ip6e_nxt;
7205 					/* goto the next header */
7206 					break;
7207 				}
7208 				default:
7209 					terminal++;
7210 					break;
7211 				}
7212 			} while (!terminal);
7213 			break;
7214 #endif /* INET6 */
7215 		}
7216 
7217 		if (PF_ANEQ(pd->dst, pd2.src, pd->af)) {
7218 			if (V_pf_status.debug >= PF_DEBUG_MISC) {
7219 				printf("pf: BAD ICMP %d:%d outer dst: ",
7220 				    icmptype, icmpcode);
7221 				pf_print_host(pd->src, 0, pd->af);
7222 				printf(" -> ");
7223 				pf_print_host(pd->dst, 0, pd->af);
7224 				printf(" inner src: ");
7225 				pf_print_host(pd2.src, 0, pd2.af);
7226 				printf(" -> ");
7227 				pf_print_host(pd2.dst, 0, pd2.af);
7228 				printf("\n");
7229 			}
7230 			REASON_SET(reason, PFRES_BADSTATE);
7231 			return (PF_DROP);
7232 		}
7233 
7234 		switch (pd2.proto) {
7235 		case IPPROTO_TCP: {
7236 			struct tcphdr		 th;
7237 			u_int32_t		 seq;
7238 			struct pf_state_peer	*src, *dst;
7239 			u_int8_t		 dws;
7240 			int			 copyback = 0;
7241 
7242 			/*
7243 			 * Only the first 8 bytes of the TCP header can be
7244 			 * expected. Don't access any TCP header fields after
7245 			 * th_seq, an ackskew test is not possible.
7246 			 */
7247 			if (!pf_pull_hdr(m, off2, &th, 8, NULL, reason,
7248 			    pd2.af)) {
7249 				DPFPRINTF(PF_DEBUG_MISC,
7250 				    ("pf: ICMP error message too short "
7251 				    "(tcp)\n"));
7252 				return (PF_DROP);
7253 			}
7254 
7255 			key.af = pd2.af;
7256 			key.proto = IPPROTO_TCP;
7257 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
7258 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
7259 			key.port[pd2.sidx] = th.th_sport;
7260 			key.port[pd2.didx] = th.th_dport;
7261 
7262 			STATE_LOOKUP(kif, &key, *state, pd);
7263 
7264 			if (pd->dir == (*state)->direction) {
7265 				src = &(*state)->dst;
7266 				dst = &(*state)->src;
7267 			} else {
7268 				src = &(*state)->src;
7269 				dst = &(*state)->dst;
7270 			}
7271 
7272 			if (src->wscale && dst->wscale)
7273 				dws = dst->wscale & PF_WSCALE_MASK;
7274 			else
7275 				dws = 0;
7276 
7277 			/* Demodulate sequence number */
7278 			seq = ntohl(th.th_seq) - src->seqdiff;
7279 			if (src->seqdiff) {
7280 				pf_change_a(&th.th_seq, icmpsum,
7281 				    htonl(seq), 0);
7282 				copyback = 1;
7283 			}
7284 
7285 			if (!((*state)->state_flags & PFSTATE_SLOPPY) &&
7286 			    (!SEQ_GEQ(src->seqhi, seq) ||
7287 			    !SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)))) {
7288 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
7289 					printf("pf: BAD ICMP %d:%d ",
7290 					    icmptype, icmpcode);
7291 					pf_print_host(pd->src, 0, pd->af);
7292 					printf(" -> ");
7293 					pf_print_host(pd->dst, 0, pd->af);
7294 					printf(" state: ");
7295 					pf_print_state(*state);
7296 					printf(" seq=%u\n", seq);
7297 				}
7298 				REASON_SET(reason, PFRES_BADSTATE);
7299 				return (PF_DROP);
7300 			} else {
7301 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
7302 					printf("pf: OK ICMP %d:%d ",
7303 					    icmptype, icmpcode);
7304 					pf_print_host(pd->src, 0, pd->af);
7305 					printf(" -> ");
7306 					pf_print_host(pd->dst, 0, pd->af);
7307 					printf(" state: ");
7308 					pf_print_state(*state);
7309 					printf(" seq=%u\n", seq);
7310 				}
7311 			}
7312 
7313 			/* translate source/destination address, if necessary */
7314 			if ((*state)->key[PF_SK_WIRE] !=
7315 			    (*state)->key[PF_SK_STACK]) {
7316 				struct pf_state_key *nk =
7317 				    (*state)->key[pd->didx];
7318 
7319 				if (PF_ANEQ(pd2.src,
7320 				    &nk->addr[pd2.sidx], pd2.af) ||
7321 				    nk->port[pd2.sidx] != th.th_sport)
7322 					pf_change_icmp(pd2.src, &th.th_sport,
7323 					    daddr, &nk->addr[pd2.sidx],
7324 					    nk->port[pd2.sidx], NULL,
7325 					    pd2.ip_sum, icmpsum,
7326 					    pd->ip_sum, 0, pd2.af);
7327 
7328 				if (PF_ANEQ(pd2.dst,
7329 				    &nk->addr[pd2.didx], pd2.af) ||
7330 				    nk->port[pd2.didx] != th.th_dport)
7331 					pf_change_icmp(pd2.dst, &th.th_dport,
7332 					    saddr, &nk->addr[pd2.didx],
7333 					    nk->port[pd2.didx], NULL,
7334 					    pd2.ip_sum, icmpsum,
7335 					    pd->ip_sum, 0, pd2.af);
7336 				copyback = 1;
7337 			}
7338 
7339 			if (copyback) {
7340 				switch (pd2.af) {
7341 #ifdef INET
7342 				case AF_INET:
7343 					m_copyback(m, off, ICMP_MINLEN,
7344 					    (caddr_t )&pd->hdr.icmp);
7345 					m_copyback(m, ipoff2, sizeof(h2),
7346 					    (caddr_t )&h2);
7347 					break;
7348 #endif /* INET */
7349 #ifdef INET6
7350 				case AF_INET6:
7351 					m_copyback(m, off,
7352 					    sizeof(struct icmp6_hdr),
7353 					    (caddr_t )&pd->hdr.icmp6);
7354 					m_copyback(m, ipoff2, sizeof(h2_6),
7355 					    (caddr_t )&h2_6);
7356 					break;
7357 #endif /* INET6 */
7358 				}
7359 				m_copyback(m, off2, 8, (caddr_t)&th);
7360 			}
7361 
7362 			return (PF_PASS);
7363 			break;
7364 		}
7365 		case IPPROTO_UDP: {
7366 			struct udphdr		uh;
7367 
7368 			if (!pf_pull_hdr(m, off2, &uh, sizeof(uh),
7369 			    NULL, reason, pd2.af)) {
7370 				DPFPRINTF(PF_DEBUG_MISC,
7371 				    ("pf: ICMP error message too short "
7372 				    "(udp)\n"));
7373 				return (PF_DROP);
7374 			}
7375 
7376 			key.af = pd2.af;
7377 			key.proto = IPPROTO_UDP;
7378 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
7379 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
7380 			key.port[pd2.sidx] = uh.uh_sport;
7381 			key.port[pd2.didx] = uh.uh_dport;
7382 
7383 			STATE_LOOKUP(kif, &key, *state, pd);
7384 
7385 			/* translate source/destination address, if necessary */
7386 			if ((*state)->key[PF_SK_WIRE] !=
7387 			    (*state)->key[PF_SK_STACK]) {
7388 				struct pf_state_key *nk =
7389 				    (*state)->key[pd->didx];
7390 
7391 				if (PF_ANEQ(pd2.src,
7392 				    &nk->addr[pd2.sidx], pd2.af) ||
7393 				    nk->port[pd2.sidx] != uh.uh_sport)
7394 					pf_change_icmp(pd2.src, &uh.uh_sport,
7395 					    daddr, &nk->addr[pd2.sidx],
7396 					    nk->port[pd2.sidx], &uh.uh_sum,
7397 					    pd2.ip_sum, icmpsum,
7398 					    pd->ip_sum, 1, pd2.af);
7399 
7400 				if (PF_ANEQ(pd2.dst,
7401 				    &nk->addr[pd2.didx], pd2.af) ||
7402 				    nk->port[pd2.didx] != uh.uh_dport)
7403 					pf_change_icmp(pd2.dst, &uh.uh_dport,
7404 					    saddr, &nk->addr[pd2.didx],
7405 					    nk->port[pd2.didx], &uh.uh_sum,
7406 					    pd2.ip_sum, icmpsum,
7407 					    pd->ip_sum, 1, pd2.af);
7408 
7409 				switch (pd2.af) {
7410 #ifdef INET
7411 				case AF_INET:
7412 					m_copyback(m, off, ICMP_MINLEN,
7413 					    (caddr_t )&pd->hdr.icmp);
7414 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
7415 					break;
7416 #endif /* INET */
7417 #ifdef INET6
7418 				case AF_INET6:
7419 					m_copyback(m, off,
7420 					    sizeof(struct icmp6_hdr),
7421 					    (caddr_t )&pd->hdr.icmp6);
7422 					m_copyback(m, ipoff2, sizeof(h2_6),
7423 					    (caddr_t )&h2_6);
7424 					break;
7425 #endif /* INET6 */
7426 				}
7427 				m_copyback(m, off2, sizeof(uh), (caddr_t)&uh);
7428 			}
7429 			return (PF_PASS);
7430 			break;
7431 		}
7432 #ifdef INET
7433 		case IPPROTO_ICMP: {
7434 			struct icmp	*iih = &pd2.hdr.icmp;
7435 
7436 			if (!pf_pull_hdr(m, off2, iih, ICMP_MINLEN,
7437 			    NULL, reason, pd2.af)) {
7438 				DPFPRINTF(PF_DEBUG_MISC,
7439 				    ("pf: ICMP error message too short i"
7440 				    "(icmp)\n"));
7441 				return (PF_DROP);
7442 			}
7443 
7444 			icmpid = iih->icmp_id;
7445 			pf_icmp_mapping(&pd2, iih->icmp_type,
7446 			    &icmp_dir, &multi, &virtual_id, &virtual_type);
7447 
7448 			ret = pf_icmp_state_lookup(&key, &pd2, state, m, off,
7449 			    pd2.dir, kif, virtual_id, virtual_type,
7450 			    icmp_dir, &iidx, PF_ICMP_MULTI_NONE, 1);
7451 			if (ret >= 0) {
7452 				MPASS(*state == NULL);
7453 				return (ret);
7454 			}
7455 
7456 			/* translate source/destination address, if necessary */
7457 			if ((*state)->key[PF_SK_WIRE] !=
7458 			    (*state)->key[PF_SK_STACK]) {
7459 				struct pf_state_key *nk =
7460 				    (*state)->key[pd->didx];
7461 
7462 				if (PF_ANEQ(pd2.src,
7463 				    &nk->addr[pd2.sidx], pd2.af) ||
7464 				    (virtual_type == htons(ICMP_ECHO) &&
7465 				    nk->port[iidx] != iih->icmp_id))
7466 					pf_change_icmp(pd2.src,
7467 					    (virtual_type == htons(ICMP_ECHO)) ?
7468 					    &iih->icmp_id : NULL,
7469 					    daddr, &nk->addr[pd2.sidx],
7470 					    (virtual_type == htons(ICMP_ECHO)) ?
7471 					    nk->port[iidx] : 0, NULL,
7472 					    pd2.ip_sum, icmpsum,
7473 					    pd->ip_sum, 0, AF_INET);
7474 
7475 				if (PF_ANEQ(pd2.dst,
7476 				    &nk->addr[pd2.didx], pd2.af))
7477 					pf_change_icmp(pd2.dst, NULL, NULL,
7478 					    &nk->addr[pd2.didx], 0, NULL,
7479 					    pd2.ip_sum, icmpsum, pd->ip_sum, 0,
7480 					    AF_INET);
7481 
7482 				m_copyback(m, off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
7483 				m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
7484 				m_copyback(m, off2, ICMP_MINLEN, (caddr_t)iih);
7485 			}
7486 			return (PF_PASS);
7487 			break;
7488 		}
7489 #endif /* INET */
7490 #ifdef INET6
7491 		case IPPROTO_ICMPV6: {
7492 			struct icmp6_hdr	*iih = &pd2.hdr.icmp6;
7493 
7494 			if (!pf_pull_hdr(m, off2, iih,
7495 			    sizeof(struct icmp6_hdr), NULL, reason, pd2.af)) {
7496 				DPFPRINTF(PF_DEBUG_MISC,
7497 				    ("pf: ICMP error message too short "
7498 				    "(icmp6)\n"));
7499 				return (PF_DROP);
7500 			}
7501 
7502 			pf_icmp_mapping(&pd2, iih->icmp6_type,
7503 			    &icmp_dir, &multi, &virtual_id, &virtual_type);
7504 
7505 			ret = pf_icmp_state_lookup(&key, &pd2, state, m, off,
7506 			    pd->dir, kif, virtual_id, virtual_type,
7507 			    icmp_dir, &iidx, PF_ICMP_MULTI_NONE, 1);
7508 			if (ret >= 0) {
7509 				MPASS(*state == NULL);
7510 				if (ret == PF_DROP && pd2.af == AF_INET6 &&
7511 				    icmp_dir == PF_OUT) {
7512 					ret = pf_icmp_state_lookup(&key, &pd2,
7513 					    state, m, off, pd->dir, kif,
7514 					    virtual_id, virtual_type,
7515 					    icmp_dir, &iidx, multi, 1);
7516 					if (ret >= 0) {
7517 						MPASS(*state == NULL);
7518 						return (ret);
7519 					}
7520 				} else
7521 					return (ret);
7522 			}
7523 
7524 			/* translate source/destination address, if necessary */
7525 			if ((*state)->key[PF_SK_WIRE] !=
7526 			    (*state)->key[PF_SK_STACK]) {
7527 				struct pf_state_key *nk =
7528 				    (*state)->key[pd->didx];
7529 
7530 				if (PF_ANEQ(pd2.src,
7531 				    &nk->addr[pd2.sidx], pd2.af) ||
7532 				    ((virtual_type == htons(ICMP6_ECHO_REQUEST)) &&
7533 				    nk->port[pd2.sidx] != iih->icmp6_id))
7534 					pf_change_icmp(pd2.src,
7535 					    (virtual_type == htons(ICMP6_ECHO_REQUEST))
7536 					    ? &iih->icmp6_id : NULL,
7537 					    daddr, &nk->addr[pd2.sidx],
7538 					    (virtual_type == htons(ICMP6_ECHO_REQUEST))
7539 					    ? nk->port[iidx] : 0, NULL,
7540 					    pd2.ip_sum, icmpsum,
7541 					    pd->ip_sum, 0, AF_INET6);
7542 
7543 				if (PF_ANEQ(pd2.dst,
7544 				    &nk->addr[pd2.didx], pd2.af))
7545 					pf_change_icmp(pd2.dst, NULL, NULL,
7546 					    &nk->addr[pd2.didx], 0, NULL,
7547 					    pd2.ip_sum, icmpsum,
7548 					    pd->ip_sum, 0, AF_INET6);
7549 
7550 				m_copyback(m, off, sizeof(struct icmp6_hdr),
7551 				    (caddr_t)&pd->hdr.icmp6);
7552 				m_copyback(m, ipoff2, sizeof(h2_6), (caddr_t)&h2_6);
7553 				m_copyback(m, off2, sizeof(struct icmp6_hdr),
7554 				    (caddr_t)iih);
7555 			}
7556 			return (PF_PASS);
7557 			break;
7558 		}
7559 #endif /* INET6 */
7560 		default: {
7561 			key.af = pd2.af;
7562 			key.proto = pd2.proto;
7563 			PF_ACPY(&key.addr[pd2.sidx], pd2.src, key.af);
7564 			PF_ACPY(&key.addr[pd2.didx], pd2.dst, key.af);
7565 			key.port[0] = key.port[1] = 0;
7566 
7567 			STATE_LOOKUP(kif, &key, *state, pd);
7568 
7569 			/* translate source/destination address, if necessary */
7570 			if ((*state)->key[PF_SK_WIRE] !=
7571 			    (*state)->key[PF_SK_STACK]) {
7572 				struct pf_state_key *nk =
7573 				    (*state)->key[pd->didx];
7574 
7575 				if (PF_ANEQ(pd2.src,
7576 				    &nk->addr[pd2.sidx], pd2.af))
7577 					pf_change_icmp(pd2.src, NULL, daddr,
7578 					    &nk->addr[pd2.sidx], 0, NULL,
7579 					    pd2.ip_sum, icmpsum,
7580 					    pd->ip_sum, 0, pd2.af);
7581 
7582 				if (PF_ANEQ(pd2.dst,
7583 				    &nk->addr[pd2.didx], pd2.af))
7584 					pf_change_icmp(pd2.dst, NULL, saddr,
7585 					    &nk->addr[pd2.didx], 0, NULL,
7586 					    pd2.ip_sum, icmpsum,
7587 					    pd->ip_sum, 0, pd2.af);
7588 
7589 				switch (pd2.af) {
7590 #ifdef INET
7591 				case AF_INET:
7592 					m_copyback(m, off, ICMP_MINLEN,
7593 					    (caddr_t)&pd->hdr.icmp);
7594 					m_copyback(m, ipoff2, sizeof(h2), (caddr_t)&h2);
7595 					break;
7596 #endif /* INET */
7597 #ifdef INET6
7598 				case AF_INET6:
7599 					m_copyback(m, off,
7600 					    sizeof(struct icmp6_hdr),
7601 					    (caddr_t )&pd->hdr.icmp6);
7602 					m_copyback(m, ipoff2, sizeof(h2_6),
7603 					    (caddr_t )&h2_6);
7604 					break;
7605 #endif /* INET6 */
7606 				}
7607 			}
7608 			return (PF_PASS);
7609 			break;
7610 		}
7611 		}
7612 	}
7613 }
7614 
7615 static int
7616 pf_test_state_other(struct pf_kstate **state, struct pfi_kkif *kif,
7617     struct mbuf *m, struct pf_pdesc *pd)
7618 {
7619 	struct pf_state_peer	*src, *dst;
7620 	struct pf_state_key_cmp	 key;
7621 	uint8_t			 psrc, pdst;
7622 
7623 	bzero(&key, sizeof(key));
7624 	key.af = pd->af;
7625 	key.proto = pd->proto;
7626 	if (pd->dir == PF_IN)	{
7627 		PF_ACPY(&key.addr[0], pd->src, key.af);
7628 		PF_ACPY(&key.addr[1], pd->dst, key.af);
7629 		key.port[0] = key.port[1] = 0;
7630 	} else {
7631 		PF_ACPY(&key.addr[1], pd->src, key.af);
7632 		PF_ACPY(&key.addr[0], pd->dst, key.af);
7633 		key.port[1] = key.port[0] = 0;
7634 	}
7635 
7636 	STATE_LOOKUP(kif, &key, *state, pd);
7637 
7638 	if (pd->dir == (*state)->direction) {
7639 		src = &(*state)->src;
7640 		dst = &(*state)->dst;
7641 		psrc = PF_PEER_SRC;
7642 		pdst = PF_PEER_DST;
7643 	} else {
7644 		src = &(*state)->dst;
7645 		dst = &(*state)->src;
7646 		psrc = PF_PEER_DST;
7647 		pdst = PF_PEER_SRC;
7648 	}
7649 
7650 	/* update states */
7651 	if (src->state < PFOTHERS_SINGLE)
7652 		pf_set_protostate(*state, psrc, PFOTHERS_SINGLE);
7653 	if (dst->state == PFOTHERS_SINGLE)
7654 		pf_set_protostate(*state, pdst, PFOTHERS_MULTIPLE);
7655 
7656 	/* update expire time */
7657 	(*state)->expire = pf_get_uptime();
7658 	if (src->state == PFOTHERS_MULTIPLE && dst->state == PFOTHERS_MULTIPLE)
7659 		(*state)->timeout = PFTM_OTHER_MULTIPLE;
7660 	else
7661 		(*state)->timeout = PFTM_OTHER_SINGLE;
7662 
7663 	/* translate source/destination address, if necessary */
7664 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
7665 		struct pf_state_key *nk = (*state)->key[pd->didx];
7666 
7667 		KASSERT(nk, ("%s: nk is null", __func__));
7668 		KASSERT(pd, ("%s: pd is null", __func__));
7669 		KASSERT(pd->src, ("%s: pd->src is null", __func__));
7670 		KASSERT(pd->dst, ("%s: pd->dst is null", __func__));
7671 		switch (pd->af) {
7672 #ifdef INET
7673 		case AF_INET:
7674 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET))
7675 				pf_change_a(&pd->src->v4.s_addr,
7676 				    pd->ip_sum,
7677 				    nk->addr[pd->sidx].v4.s_addr,
7678 				    0);
7679 
7680 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET))
7681 				pf_change_a(&pd->dst->v4.s_addr,
7682 				    pd->ip_sum,
7683 				    nk->addr[pd->didx].v4.s_addr,
7684 				    0);
7685 
7686 			break;
7687 #endif /* INET */
7688 #ifdef INET6
7689 		case AF_INET6:
7690 			if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], AF_INET6))
7691 				PF_ACPY(pd->src, &nk->addr[pd->sidx], pd->af);
7692 
7693 			if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], AF_INET6))
7694 				PF_ACPY(pd->dst, &nk->addr[pd->didx], pd->af);
7695 #endif /* INET6 */
7696 		}
7697 	}
7698 	return (PF_PASS);
7699 }
7700 
7701 /*
7702  * ipoff and off are measured from the start of the mbuf chain.
7703  * h must be at "ipoff" on the mbuf chain.
7704  */
7705 void *
7706 pf_pull_hdr(const struct mbuf *m, int off, void *p, int len,
7707     u_short *actionp, u_short *reasonp, sa_family_t af)
7708 {
7709 	switch (af) {
7710 #ifdef INET
7711 	case AF_INET: {
7712 		const struct ip	*h = mtod(m, struct ip *);
7713 		u_int16_t	 fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3;
7714 
7715 		if (fragoff) {
7716 			if (fragoff >= len)
7717 				ACTION_SET(actionp, PF_PASS);
7718 			else {
7719 				ACTION_SET(actionp, PF_DROP);
7720 				REASON_SET(reasonp, PFRES_FRAG);
7721 			}
7722 			return (NULL);
7723 		}
7724 		if (m->m_pkthdr.len < off + len ||
7725 		    ntohs(h->ip_len) < off + len) {
7726 			ACTION_SET(actionp, PF_DROP);
7727 			REASON_SET(reasonp, PFRES_SHORT);
7728 			return (NULL);
7729 		}
7730 		break;
7731 	}
7732 #endif /* INET */
7733 #ifdef INET6
7734 	case AF_INET6: {
7735 		const struct ip6_hdr	*h = mtod(m, struct ip6_hdr *);
7736 
7737 		if (m->m_pkthdr.len < off + len ||
7738 		    (ntohs(h->ip6_plen) + sizeof(struct ip6_hdr)) <
7739 		    (unsigned)(off + len)) {
7740 			ACTION_SET(actionp, PF_DROP);
7741 			REASON_SET(reasonp, PFRES_SHORT);
7742 			return (NULL);
7743 		}
7744 		break;
7745 	}
7746 #endif /* INET6 */
7747 	}
7748 	m_copydata(m, off, len, p);
7749 	return (p);
7750 }
7751 
7752 int
7753 pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kkif *kif,
7754     int rtableid)
7755 {
7756 	struct ifnet		*ifp;
7757 
7758 	/*
7759 	 * Skip check for addresses with embedded interface scope,
7760 	 * as they would always match anyway.
7761 	 */
7762 	if (af == AF_INET6 && IN6_IS_SCOPE_EMBED(&addr->v6))
7763 		return (1);
7764 
7765 	if (af != AF_INET && af != AF_INET6)
7766 		return (0);
7767 
7768 	if (kif == V_pfi_all)
7769 		return (1);
7770 
7771 	/* Skip checks for ipsec interfaces */
7772 	if (kif != NULL && kif->pfik_ifp->if_type == IFT_ENC)
7773 		return (1);
7774 
7775 	ifp = (kif != NULL) ? kif->pfik_ifp : NULL;
7776 
7777 	switch (af) {
7778 #ifdef INET6
7779 	case AF_INET6:
7780 		return (fib6_check_urpf(rtableid, &addr->v6, 0, NHR_NONE,
7781 		    ifp));
7782 #endif
7783 #ifdef INET
7784 	case AF_INET:
7785 		return (fib4_check_urpf(rtableid, addr->v4, 0, NHR_NONE,
7786 		    ifp));
7787 #endif
7788 	}
7789 
7790 	return (0);
7791 }
7792 
7793 #ifdef INET
7794 static void
7795 pf_route(struct mbuf **m, struct pf_krule *r, struct ifnet *oifp,
7796     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
7797 {
7798 	struct mbuf		*m0, *m1, *md;
7799 	struct sockaddr_in	dst;
7800 	struct ip		*ip;
7801 	struct pfi_kkif		*nkif = NULL;
7802 	struct ifnet		*ifp = NULL;
7803 	struct pf_addr		 naddr;
7804 	struct pf_ksrc_node	*sn = NULL;
7805 	int			 error = 0;
7806 	uint16_t		 ip_len, ip_off;
7807 	uint16_t		 tmp;
7808 	int			 r_rt, r_dir;
7809 
7810 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
7811 
7812 	if (s) {
7813 		r_rt = s->rt;
7814 		r_dir = s->direction;
7815 	} else {
7816 		r_rt = r->rt;
7817 		r_dir = r->direction;
7818 	}
7819 
7820 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
7821 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
7822 	    __func__));
7823 
7824 	if ((pd->pf_mtag == NULL &&
7825 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
7826 	    pd->pf_mtag->routed++ > 3) {
7827 		m0 = *m;
7828 		*m = NULL;
7829 		goto bad_locked;
7830 	}
7831 
7832 	if (r_rt == PF_DUPTO) {
7833 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
7834 			if (s == NULL) {
7835 				ifp = r->rpool.cur->kif ?
7836 				    r->rpool.cur->kif->pfik_ifp : NULL;
7837 			} else {
7838 				ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7839 				/* If pfsync'd */
7840 				if (ifp == NULL && r->rpool.cur != NULL)
7841 					ifp = r->rpool.cur->kif ?
7842 					    r->rpool.cur->kif->pfik_ifp : NULL;
7843 				PF_STATE_UNLOCK(s);
7844 			}
7845 			if (ifp == oifp) {
7846 				/* When the 2nd interface is not skipped */
7847 				return;
7848 			} else {
7849 				m0 = *m;
7850 				*m = NULL;
7851 				goto bad;
7852 			}
7853 		} else {
7854 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
7855 			if (((m0 = m_dup(*m, M_NOWAIT)) == NULL)) {
7856 				if (s)
7857 					PF_STATE_UNLOCK(s);
7858 				return;
7859 			}
7860 		}
7861 	} else {
7862 		if ((r_rt == PF_REPLYTO) == (r_dir == pd->dir)) {
7863 			pf_dummynet(pd, s, r, m);
7864 			if (s)
7865 				PF_STATE_UNLOCK(s);
7866 			return;
7867 		}
7868 		m0 = *m;
7869 	}
7870 
7871 	ip = mtod(m0, struct ip *);
7872 
7873 	bzero(&dst, sizeof(dst));
7874 	dst.sin_family = AF_INET;
7875 	dst.sin_len = sizeof(dst);
7876 	dst.sin_addr = ip->ip_dst;
7877 
7878 	bzero(&naddr, sizeof(naddr));
7879 
7880 	if (s == NULL) {
7881 		if (TAILQ_EMPTY(&r->rpool.list)) {
7882 			DPFPRINTF(PF_DEBUG_URGENT,
7883 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
7884 			goto bad_locked;
7885 		}
7886 		pf_map_addr(AF_INET, r, (struct pf_addr *)&ip->ip_src,
7887 		    &naddr, &nkif, NULL, &sn);
7888 		if (!PF_AZERO(&naddr, AF_INET))
7889 			dst.sin_addr.s_addr = naddr.v4.s_addr;
7890 		ifp = nkif ? nkif->pfik_ifp : NULL;
7891 	} else {
7892 		struct pfi_kkif *kif;
7893 
7894 		if (!PF_AZERO(&s->rt_addr, AF_INET))
7895 			dst.sin_addr.s_addr =
7896 			    s->rt_addr.v4.s_addr;
7897 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
7898 		kif = s->rt_kif;
7899 		/* If pfsync'd */
7900 		if (ifp == NULL && r->rpool.cur != NULL) {
7901 			ifp = r->rpool.cur->kif ?
7902 			    r->rpool.cur->kif->pfik_ifp : NULL;
7903 			kif = r->rpool.cur->kif;
7904 		}
7905 		if (ifp != NULL && kif != NULL &&
7906 		    r->rule_flag & PFRULE_IFBOUND &&
7907 		    r->rt == PF_REPLYTO &&
7908 		    s->kif == V_pfi_all) {
7909 			s->kif = kif;
7910 			s->orig_kif = oifp->if_pf_kif;
7911 		}
7912 
7913 		PF_STATE_UNLOCK(s);
7914 	}
7915 
7916 	if (ifp == NULL)
7917 		goto bad;
7918 
7919 	if (pd->dir == PF_IN) {
7920 		if (pf_test(AF_INET, PF_OUT, PFIL_FWD, ifp, &m0, inp,
7921 		    &pd->act) != PF_PASS)
7922 			goto bad;
7923 		else if (m0 == NULL)
7924 			goto done;
7925 		if (m0->m_len < sizeof(struct ip)) {
7926 			DPFPRINTF(PF_DEBUG_URGENT,
7927 			    ("%s: m0->m_len < sizeof(struct ip)\n", __func__));
7928 			goto bad;
7929 		}
7930 		ip = mtod(m0, struct ip *);
7931 	}
7932 
7933 	if (ifp->if_flags & IFF_LOOPBACK)
7934 		m0->m_flags |= M_SKIP_FIREWALL;
7935 
7936 	ip_len = ntohs(ip->ip_len);
7937 	ip_off = ntohs(ip->ip_off);
7938 
7939 	/* Copied from FreeBSD 10.0-CURRENT ip_output. */
7940 	m0->m_pkthdr.csum_flags |= CSUM_IP;
7941 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~ifp->if_hwassist) {
7942 		in_delayed_cksum(m0);
7943 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
7944 	}
7945 	if (m0->m_pkthdr.csum_flags & CSUM_SCTP & ~ifp->if_hwassist) {
7946 		pf_sctp_checksum(m0, (uint32_t)(ip->ip_hl << 2));
7947 		m0->m_pkthdr.csum_flags &= ~CSUM_SCTP;
7948 	}
7949 
7950 	if (pd->dir == PF_IN) {
7951 		/*
7952 		 * Make sure dummynet gets the correct direction, in case it needs to
7953 		 * re-inject later.
7954 		 */
7955 		pd->dir = PF_OUT;
7956 
7957 		/*
7958 		 * The following processing is actually the rest of the inbound processing, even
7959 		 * though we've marked it as outbound (so we don't look through dummynet) and it
7960 		 * happens after the outbound processing (pf_test(PF_OUT) above).
7961 		 * Swap the dummynet pipe numbers, because it's going to come to the wrong
7962 		 * conclusion about what direction it's processing, and we can't fix it or it
7963 		 * will re-inject incorrectly. Swapping the pipe numbers means that its incorrect
7964 		 * decision will pick the right pipe, and everything will mostly work as expected.
7965 		 */
7966 		tmp = pd->act.dnrpipe;
7967 		pd->act.dnrpipe = pd->act.dnpipe;
7968 		pd->act.dnpipe = tmp;
7969 	}
7970 
7971 	/*
7972 	 * If small enough for interface, or the interface will take
7973 	 * care of the fragmentation for us, we can just send directly.
7974 	 */
7975 	if (ip_len <= ifp->if_mtu ||
7976 	    (m0->m_pkthdr.csum_flags & ifp->if_hwassist & CSUM_TSO) != 0) {
7977 		ip->ip_sum = 0;
7978 		if (m0->m_pkthdr.csum_flags & CSUM_IP & ~ifp->if_hwassist) {
7979 			ip->ip_sum = in_cksum(m0, ip->ip_hl << 2);
7980 			m0->m_pkthdr.csum_flags &= ~CSUM_IP;
7981 		}
7982 		m_clrprotoflags(m0);	/* Avoid confusing lower layers. */
7983 
7984 		md = m0;
7985 		error = pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
7986 		if (md != NULL)
7987 			error = (*ifp->if_output)(ifp, md, sintosa(&dst), NULL);
7988 		goto done;
7989 	}
7990 
7991 	/* Balk when DF bit is set or the interface didn't support TSO. */
7992 	if ((ip_off & IP_DF) || (m0->m_pkthdr.csum_flags & CSUM_TSO)) {
7993 		error = EMSGSIZE;
7994 		KMOD_IPSTAT_INC(ips_cantfrag);
7995 		if (r_rt != PF_DUPTO) {
7996 			if (s && pd->nat_rule != NULL)
7997 				PACKET_UNDO_NAT(m0, pd,
7998 				    (ip->ip_hl << 2) + (ip_off & IP_OFFMASK),
7999 				    s);
8000 
8001 			icmp_error(m0, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG, 0,
8002 			    ifp->if_mtu);
8003 			goto done;
8004 		} else
8005 			goto bad;
8006 	}
8007 
8008 	error = ip_fragment(ip, &m0, ifp->if_mtu, ifp->if_hwassist);
8009 	if (error)
8010 		goto bad;
8011 
8012 	for (; m0; m0 = m1) {
8013 		m1 = m0->m_nextpkt;
8014 		m0->m_nextpkt = NULL;
8015 		if (error == 0) {
8016 			m_clrprotoflags(m0);
8017 			md = m0;
8018 			pd->pf_mtag = pf_find_mtag(md);
8019 			error = pf_dummynet_route(pd, s, r, ifp,
8020 			    sintosa(&dst), &md);
8021 			if (md != NULL)
8022 				error = (*ifp->if_output)(ifp, md,
8023 				    sintosa(&dst), NULL);
8024 		} else
8025 			m_freem(m0);
8026 	}
8027 
8028 	if (error == 0)
8029 		KMOD_IPSTAT_INC(ips_fragmented);
8030 
8031 done:
8032 	if (r_rt != PF_DUPTO)
8033 		*m = NULL;
8034 	return;
8035 
8036 bad_locked:
8037 	if (s)
8038 		PF_STATE_UNLOCK(s);
8039 bad:
8040 	m_freem(m0);
8041 	goto done;
8042 }
8043 #endif /* INET */
8044 
8045 #ifdef INET6
8046 static void
8047 pf_route6(struct mbuf **m, struct pf_krule *r, struct ifnet *oifp,
8048     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
8049 {
8050 	struct mbuf		*m0, *md;
8051 	struct sockaddr_in6	dst;
8052 	struct ip6_hdr		*ip6;
8053 	struct pfi_kkif		*nkif = NULL;
8054 	struct ifnet		*ifp = NULL;
8055 	struct pf_addr		 naddr;
8056 	struct pf_ksrc_node	*sn = NULL;
8057 	int			 r_rt, r_dir;
8058 
8059 	KASSERT(m && *m && r && oifp, ("%s: invalid parameters", __func__));
8060 
8061 	if (s) {
8062 		r_rt = s->rt;
8063 		r_dir = s->direction;
8064 	} else {
8065 		r_rt = r->rt;
8066 		r_dir = r->direction;
8067 	}
8068 
8069 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
8070 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
8071 	    __func__));
8072 
8073 	if ((pd->pf_mtag == NULL &&
8074 	    ((pd->pf_mtag = pf_get_mtag(*m)) == NULL)) ||
8075 	    pd->pf_mtag->routed++ > 3) {
8076 		m0 = *m;
8077 		*m = NULL;
8078 		goto bad_locked;
8079 	}
8080 
8081 	if (r_rt == PF_DUPTO) {
8082 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
8083 			if (s == NULL) {
8084 				ifp = r->rpool.cur->kif ?
8085 				    r->rpool.cur->kif->pfik_ifp : NULL;
8086 			} else {
8087 				ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
8088 				/* If pfsync'd */
8089 				if (ifp == NULL && r->rpool.cur != NULL)
8090 					ifp = r->rpool.cur->kif ?
8091 					    r->rpool.cur->kif->pfik_ifp : NULL;
8092 				PF_STATE_UNLOCK(s);
8093 			}
8094 			if (ifp == oifp) {
8095 				/* When the 2nd interface is not skipped */
8096 				return;
8097 			} else {
8098 				m0 = *m;
8099 				*m = NULL;
8100 				goto bad;
8101 			}
8102 		} else {
8103 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
8104 			if (((m0 = m_dup(*m, M_NOWAIT)) == NULL)) {
8105 				if (s)
8106 					PF_STATE_UNLOCK(s);
8107 				return;
8108 			}
8109 		}
8110 	} else {
8111 		if ((r_rt == PF_REPLYTO) == (r_dir == pd->dir)) {
8112 			pf_dummynet(pd, s, r, m);
8113 			if (s)
8114 				PF_STATE_UNLOCK(s);
8115 			return;
8116 		}
8117 		m0 = *m;
8118 	}
8119 
8120 	ip6 = mtod(m0, struct ip6_hdr *);
8121 
8122 	bzero(&dst, sizeof(dst));
8123 	dst.sin6_family = AF_INET6;
8124 	dst.sin6_len = sizeof(dst);
8125 	dst.sin6_addr = ip6->ip6_dst;
8126 
8127 	bzero(&naddr, sizeof(naddr));
8128 
8129 	if (s == NULL) {
8130 		if (TAILQ_EMPTY(&r->rpool.list)) {
8131 			DPFPRINTF(PF_DEBUG_URGENT,
8132 			    ("%s: TAILQ_EMPTY(&r->rpool.list)\n", __func__));
8133 			goto bad_locked;
8134 		}
8135 		pf_map_addr(AF_INET6, r, (struct pf_addr *)&ip6->ip6_src,
8136 		    &naddr, &nkif, NULL, &sn);
8137 		if (!PF_AZERO(&naddr, AF_INET6))
8138 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
8139 			    &naddr, AF_INET6);
8140 		ifp = nkif ? nkif->pfik_ifp : NULL;
8141 	} else {
8142 		struct pfi_kkif *kif;
8143 
8144 		if (!PF_AZERO(&s->rt_addr, AF_INET6))
8145 			PF_ACPY((struct pf_addr *)&dst.sin6_addr,
8146 			    &s->rt_addr, AF_INET6);
8147 		ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL;
8148 		kif = s->rt_kif;
8149 		/* If pfsync'd */
8150 		if (ifp == NULL && r->rpool.cur != NULL) {
8151 			ifp = r->rpool.cur->kif ?
8152 			    r->rpool.cur->kif->pfik_ifp : NULL;
8153 			kif = r->rpool.cur->kif;
8154 		}
8155 		if (ifp != NULL && kif != NULL &&
8156 		    r->rule_flag & PFRULE_IFBOUND &&
8157 		    r->rt == PF_REPLYTO &&
8158 		    s->kif == V_pfi_all) {
8159 			s->kif = kif;
8160 			s->orig_kif = oifp->if_pf_kif;
8161 		}
8162 	}
8163 
8164 	if (s)
8165 		PF_STATE_UNLOCK(s);
8166 
8167 	if (ifp == NULL)
8168 		goto bad;
8169 
8170 	if (pd->dir == PF_IN) {
8171 		if (pf_test(AF_INET6, PF_OUT, PFIL_FWD, ifp, &m0, inp,
8172 		    &pd->act) != PF_PASS)
8173 			goto bad;
8174 		else if (m0 == NULL)
8175 			goto done;
8176 		if (m0->m_len < sizeof(struct ip6_hdr)) {
8177 			DPFPRINTF(PF_DEBUG_URGENT,
8178 			    ("%s: m0->m_len < sizeof(struct ip6_hdr)\n",
8179 			    __func__));
8180 			goto bad;
8181 		}
8182 		ip6 = mtod(m0, struct ip6_hdr *);
8183 	}
8184 
8185 	if (ifp->if_flags & IFF_LOOPBACK)
8186 		m0->m_flags |= M_SKIP_FIREWALL;
8187 
8188 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 &
8189 	    ~ifp->if_hwassist) {
8190 		uint32_t plen = m0->m_pkthdr.len - sizeof(*ip6);
8191 		in6_delayed_cksum(m0, plen, sizeof(struct ip6_hdr));
8192 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
8193 	}
8194 
8195 	/*
8196 	 * If the packet is too large for the outgoing interface,
8197 	 * send back an icmp6 error.
8198 	 */
8199 	if (IN6_IS_SCOPE_EMBED(&dst.sin6_addr))
8200 		dst.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
8201 	if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu) {
8202 		md = m0;
8203 		pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
8204 		if (md != NULL)
8205 			nd6_output_ifp(ifp, ifp, md, &dst, NULL);
8206 	}
8207 	else {
8208 		in6_ifstat_inc(ifp, ifs6_in_toobig);
8209 		if (r_rt != PF_DUPTO) {
8210 			if (s && pd->nat_rule != NULL)
8211 				PACKET_UNDO_NAT(m0, pd,
8212 				    ((caddr_t)ip6 - m0->m_data) +
8213 				    sizeof(struct ip6_hdr), s);
8214 
8215 			icmp6_error(m0, ICMP6_PACKET_TOO_BIG, 0, ifp->if_mtu);
8216 		} else
8217 			goto bad;
8218 	}
8219 
8220 done:
8221 	if (r_rt != PF_DUPTO)
8222 		*m = NULL;
8223 	return;
8224 
8225 bad_locked:
8226 	if (s)
8227 		PF_STATE_UNLOCK(s);
8228 bad:
8229 	m_freem(m0);
8230 	goto done;
8231 }
8232 #endif /* INET6 */
8233 
8234 /*
8235  * FreeBSD supports cksum offloads for the following drivers.
8236  *  em(4), fxp(4), lge(4), nge(4), re(4), ti(4), txp(4), xl(4)
8237  *
8238  * CSUM_DATA_VALID | CSUM_PSEUDO_HDR :
8239  *  network driver performed cksum including pseudo header, need to verify
8240  *   csum_data
8241  * CSUM_DATA_VALID :
8242  *  network driver performed cksum, needs to additional pseudo header
8243  *  cksum computation with partial csum_data(i.e. lack of H/W support for
8244  *  pseudo header, for instance sk(4) and possibly gem(4))
8245  *
8246  * After validating the cksum of packet, set both flag CSUM_DATA_VALID and
8247  * CSUM_PSEUDO_HDR in order to avoid recomputation of the cksum in upper
8248  * TCP/UDP layer.
8249  * Also, set csum_data to 0xffff to force cksum validation.
8250  */
8251 static int
8252 pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t af)
8253 {
8254 	u_int16_t sum = 0;
8255 	int hw_assist = 0;
8256 	struct ip *ip;
8257 
8258 	if (off < sizeof(struct ip) || len < sizeof(struct udphdr))
8259 		return (1);
8260 	if (m->m_pkthdr.len < off + len)
8261 		return (1);
8262 
8263 	switch (p) {
8264 	case IPPROTO_TCP:
8265 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
8266 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
8267 				sum = m->m_pkthdr.csum_data;
8268 			} else {
8269 				ip = mtod(m, struct ip *);
8270 				sum = in_pseudo(ip->ip_src.s_addr,
8271 				ip->ip_dst.s_addr, htonl((u_short)len +
8272 				m->m_pkthdr.csum_data + IPPROTO_TCP));
8273 			}
8274 			sum ^= 0xffff;
8275 			++hw_assist;
8276 		}
8277 		break;
8278 	case IPPROTO_UDP:
8279 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
8280 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
8281 				sum = m->m_pkthdr.csum_data;
8282 			} else {
8283 				ip = mtod(m, struct ip *);
8284 				sum = in_pseudo(ip->ip_src.s_addr,
8285 				ip->ip_dst.s_addr, htonl((u_short)len +
8286 				m->m_pkthdr.csum_data + IPPROTO_UDP));
8287 			}
8288 			sum ^= 0xffff;
8289 			++hw_assist;
8290 		}
8291 		break;
8292 	case IPPROTO_ICMP:
8293 #ifdef INET6
8294 	case IPPROTO_ICMPV6:
8295 #endif /* INET6 */
8296 		break;
8297 	default:
8298 		return (1);
8299 	}
8300 
8301 	if (!hw_assist) {
8302 		switch (af) {
8303 		case AF_INET:
8304 			if (p == IPPROTO_ICMP) {
8305 				if (m->m_len < off)
8306 					return (1);
8307 				m->m_data += off;
8308 				m->m_len -= off;
8309 				sum = in_cksum(m, len);
8310 				m->m_data -= off;
8311 				m->m_len += off;
8312 			} else {
8313 				if (m->m_len < sizeof(struct ip))
8314 					return (1);
8315 				sum = in4_cksum(m, p, off, len);
8316 			}
8317 			break;
8318 #ifdef INET6
8319 		case AF_INET6:
8320 			if (m->m_len < sizeof(struct ip6_hdr))
8321 				return (1);
8322 			sum = in6_cksum(m, p, off, len);
8323 			break;
8324 #endif /* INET6 */
8325 		default:
8326 			return (1);
8327 		}
8328 	}
8329 	if (sum) {
8330 		switch (p) {
8331 		case IPPROTO_TCP:
8332 		    {
8333 			KMOD_TCPSTAT_INC(tcps_rcvbadsum);
8334 			break;
8335 		    }
8336 		case IPPROTO_UDP:
8337 		    {
8338 			KMOD_UDPSTAT_INC(udps_badsum);
8339 			break;
8340 		    }
8341 #ifdef INET
8342 		case IPPROTO_ICMP:
8343 		    {
8344 			KMOD_ICMPSTAT_INC(icps_checksum);
8345 			break;
8346 		    }
8347 #endif
8348 #ifdef INET6
8349 		case IPPROTO_ICMPV6:
8350 		    {
8351 			KMOD_ICMP6STAT_INC(icp6s_checksum);
8352 			break;
8353 		    }
8354 #endif /* INET6 */
8355 		}
8356 		return (1);
8357 	} else {
8358 		if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
8359 			m->m_pkthdr.csum_flags |=
8360 			    (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
8361 			m->m_pkthdr.csum_data = 0xffff;
8362 		}
8363 	}
8364 	return (0);
8365 }
8366 
8367 static bool
8368 pf_pdesc_to_dnflow(const struct pf_pdesc *pd, const struct pf_krule *r,
8369     const struct pf_kstate *s, struct ip_fw_args *dnflow)
8370 {
8371 	int dndir = r->direction;
8372 
8373 	if (s && dndir == PF_INOUT) {
8374 		dndir = s->direction;
8375 	} else if (dndir == PF_INOUT) {
8376 		/* Assume primary direction. Happens when we've set dnpipe in
8377 		 * the ethernet level code. */
8378 		dndir = pd->dir;
8379 	}
8380 
8381 	if (pd->pf_mtag->flags & PF_MTAG_FLAG_DUMMYNETED)
8382 		return (false);
8383 
8384 	memset(dnflow, 0, sizeof(*dnflow));
8385 
8386 	if (pd->dport != NULL)
8387 		dnflow->f_id.dst_port = ntohs(*pd->dport);
8388 	if (pd->sport != NULL)
8389 		dnflow->f_id.src_port = ntohs(*pd->sport);
8390 
8391 	if (pd->dir == PF_IN)
8392 		dnflow->flags |= IPFW_ARGS_IN;
8393 	else
8394 		dnflow->flags |= IPFW_ARGS_OUT;
8395 
8396 	if (pd->dir != dndir && pd->act.dnrpipe) {
8397 		dnflow->rule.info = pd->act.dnrpipe;
8398 	}
8399 	else if (pd->dir == dndir && pd->act.dnpipe) {
8400 		dnflow->rule.info = pd->act.dnpipe;
8401 	}
8402 	else {
8403 		return (false);
8404 	}
8405 
8406 	dnflow->rule.info |= IPFW_IS_DUMMYNET;
8407 	if (r->free_flags & PFRULE_DN_IS_PIPE || pd->act.flags & PFSTATE_DN_IS_PIPE)
8408 		dnflow->rule.info |= IPFW_IS_PIPE;
8409 
8410 	dnflow->f_id.proto = pd->proto;
8411 	dnflow->f_id.extra = dnflow->rule.info;
8412 	switch (pd->af) {
8413 	case AF_INET:
8414 		dnflow->f_id.addr_type = 4;
8415 		dnflow->f_id.src_ip = ntohl(pd->src->v4.s_addr);
8416 		dnflow->f_id.dst_ip = ntohl(pd->dst->v4.s_addr);
8417 		break;
8418 	case AF_INET6:
8419 		dnflow->flags |= IPFW_ARGS_IP6;
8420 		dnflow->f_id.addr_type = 6;
8421 		dnflow->f_id.src_ip6 = pd->src->v6;
8422 		dnflow->f_id.dst_ip6 = pd->dst->v6;
8423 		break;
8424 	default:
8425 		panic("Invalid AF");
8426 		break;
8427 	}
8428 
8429 	return (true);
8430 }
8431 
8432 int
8433 pf_test_eth(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
8434     struct inpcb *inp)
8435 {
8436 	struct pfi_kkif		*kif;
8437 	struct mbuf		*m = *m0;
8438 
8439 	M_ASSERTPKTHDR(m);
8440 	MPASS(ifp->if_vnet == curvnet);
8441 	NET_EPOCH_ASSERT();
8442 
8443 	if (!V_pf_status.running)
8444 		return (PF_PASS);
8445 
8446 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
8447 
8448 	if (kif == NULL) {
8449 		DPFPRINTF(PF_DEBUG_URGENT,
8450 		    ("%s: kif == NULL, if_xname %s\n", __func__, ifp->if_xname));
8451 		return (PF_DROP);
8452 	}
8453 	if (kif->pfik_flags & PFI_IFLAG_SKIP)
8454 		return (PF_PASS);
8455 
8456 	if (m->m_flags & M_SKIP_FIREWALL)
8457 		return (PF_PASS);
8458 
8459 	if (__predict_false(! M_WRITABLE(*m0))) {
8460 		m = *m0 = m_unshare(*m0, M_NOWAIT);
8461 		if (*m0 == NULL)
8462 			return (PF_DROP);
8463 	}
8464 
8465 	/* Stateless! */
8466 	return (pf_test_eth_rule(dir, kif, m0));
8467 }
8468 
8469 static __inline void
8470 pf_dummynet_flag_remove(struct mbuf *m, struct pf_mtag *pf_mtag)
8471 {
8472 	struct m_tag *mtag;
8473 
8474 	pf_mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
8475 
8476 	/* dummynet adds this tag, but pf does not need it,
8477 	 * and keeping it creates unexpected behavior,
8478 	 * e.g. in case of divert(4) usage right after dummynet. */
8479 	mtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL);
8480 	if (mtag != NULL)
8481 		m_tag_delete(m, mtag);
8482 }
8483 
8484 static int
8485 pf_dummynet(struct pf_pdesc *pd, struct pf_kstate *s,
8486     struct pf_krule *r, struct mbuf **m0)
8487 {
8488 	return (pf_dummynet_route(pd, s, r, NULL, NULL, m0));
8489 }
8490 
8491 static int
8492 pf_dummynet_route(struct pf_pdesc *pd, struct pf_kstate *s,
8493     struct pf_krule *r, struct ifnet *ifp, struct sockaddr *sa,
8494     struct mbuf **m0)
8495 {
8496 	NET_EPOCH_ASSERT();
8497 
8498 	if (pd->act.dnpipe || pd->act.dnrpipe) {
8499 		struct ip_fw_args dnflow;
8500 		if (ip_dn_io_ptr == NULL) {
8501 			m_freem(*m0);
8502 			*m0 = NULL;
8503 			return (ENOMEM);
8504 		}
8505 
8506 		if (pd->pf_mtag == NULL &&
8507 		    ((pd->pf_mtag = pf_get_mtag(*m0)) == NULL)) {
8508 			m_freem(*m0);
8509 			*m0 = NULL;
8510 			return (ENOMEM);
8511 		}
8512 
8513 		if (ifp != NULL) {
8514 			pd->pf_mtag->flags |= PF_MTAG_FLAG_ROUTE_TO;
8515 
8516 			pd->pf_mtag->if_index = ifp->if_index;
8517 			pd->pf_mtag->if_idxgen = ifp->if_idxgen;
8518 
8519 			MPASS(sa != NULL);
8520 
8521 			if (pd->af == AF_INET)
8522 				memcpy(&pd->pf_mtag->dst, sa,
8523 				    sizeof(struct sockaddr_in));
8524 			else
8525 				memcpy(&pd->pf_mtag->dst, sa,
8526 				    sizeof(struct sockaddr_in6));
8527 		}
8528 
8529 		if (s != NULL && s->nat_rule.ptr != NULL &&
8530 		    s->nat_rule.ptr->action == PF_RDR &&
8531 		    (
8532 #ifdef INET
8533 		    (pd->af == AF_INET && IN_LOOPBACK(ntohl(pd->dst->v4.s_addr))) ||
8534 #endif
8535 		    (pd->af == AF_INET6 && IN6_IS_ADDR_LOOPBACK(&pd->dst->v6)))) {
8536 			/*
8537 			 * If we're redirecting to loopback mark this packet
8538 			 * as being local. Otherwise it might get dropped
8539 			 * if dummynet re-injects.
8540 			 */
8541 			(*m0)->m_pkthdr.rcvif = V_loif;
8542 		}
8543 
8544 		if (pf_pdesc_to_dnflow(pd, r, s, &dnflow)) {
8545 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
8546 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNETED;
8547 			ip_dn_io_ptr(m0, &dnflow);
8548 			if (*m0 != NULL) {
8549 				pd->pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
8550 				pf_dummynet_flag_remove(*m0, pd->pf_mtag);
8551 			}
8552 		}
8553 	}
8554 
8555 	return (0);
8556 }
8557 
8558 static int
8559 pf_setup_pdesc(sa_family_t af, int dir, struct pf_pdesc *pd, struct mbuf *m,
8560     u_short *action, u_short *reason, struct pfi_kkif *kif, struct pf_krule **a,
8561     struct pf_krule **r, struct pf_kruleset **ruleset, int *off, int *hdrlen,
8562     struct pf_rule_actions *default_actions)
8563 {
8564 
8565 	TAILQ_INIT(&pd->sctp_multihome_jobs);
8566 	if (default_actions != NULL)
8567 		memcpy(&pd->act, default_actions, sizeof(pd->act));
8568 	pd->pf_mtag = pf_find_mtag(m);
8569 
8570 	if (pd->pf_mtag && pd->pf_mtag->dnpipe) {
8571 		pd->act.dnpipe = pd->pf_mtag->dnpipe;
8572 		pd->act.flags = pd->pf_mtag->dnflags;
8573 	}
8574 
8575 	pd->af = af;
8576 
8577 	switch (af) {
8578 #ifdef INET
8579 	case AF_INET: {
8580 		struct ip *h;
8581 
8582 		h = mtod(m, struct ip *);
8583 		*off = h->ip_hl << 2;
8584 		if (*off < (int)sizeof(*h)) {
8585 			*action = PF_DROP;
8586 			REASON_SET(reason, PFRES_SHORT);
8587 			return (-1);
8588 		}
8589 		pd->src = (struct pf_addr *)&h->ip_src;
8590 		pd->dst = (struct pf_addr *)&h->ip_dst;
8591 		pd->sport = pd->dport = NULL;
8592 		pd->ip_sum = &h->ip_sum;
8593 		pd->proto_sum = NULL;
8594 		pd->proto = h->ip_p;
8595 		pd->dir = dir;
8596 		pd->sidx = (dir == PF_IN) ? 0 : 1;
8597 		pd->didx = (dir == PF_IN) ? 1 : 0;
8598 		pd->tos = h->ip_tos;
8599 		pd->tot_len = ntohs(h->ip_len);
8600 		pd->act.rtableid = -1;
8601 
8602 		if (h->ip_hl > 5)	/* has options */
8603 			pd->badopts++;
8604 
8605 		/* fragments not reassembled handled later */
8606 		if (h->ip_off & htons(IP_MF | IP_OFFMASK))
8607 			return (0);
8608 
8609 		break;
8610 	}
8611 #endif
8612 #ifdef INET6
8613 	case AF_INET6: {
8614 		struct ip6_hdr *h;
8615 		int terminal = 0;
8616 
8617 		h = mtod(m, struct ip6_hdr *);
8618 		pd->src = (struct pf_addr *)&h->ip6_src;
8619 		pd->dst = (struct pf_addr *)&h->ip6_dst;
8620 		pd->sport = pd->dport = NULL;
8621 		pd->ip_sum = NULL;
8622 		pd->proto_sum = NULL;
8623 		pd->dir = dir;
8624 		pd->sidx = (dir == PF_IN) ? 0 : 1;
8625 		pd->didx = (dir == PF_IN) ? 1 : 0;
8626 		pd->tos = IPV6_DSCP(h);
8627 		pd->tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr);
8628 		*off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr);
8629 		pd->proto = h->ip6_nxt;
8630 		pd->act.rtableid = -1;
8631 
8632 		do {
8633 			switch (pd->proto) {
8634 			case IPPROTO_FRAGMENT:
8635 				if (kif == NULL || r == NULL) /* pflog */
8636 					*action = PF_DROP;
8637 				else
8638 					*action = pf_test_fragment(r, kif,
8639 					    m, pd, a, ruleset);
8640 				if (*action == PF_DROP)
8641 					REASON_SET(reason, PFRES_FRAG);
8642 				return (-1);
8643 			case IPPROTO_ROUTING: {
8644 				struct ip6_rthdr rthdr;
8645 
8646 				if (pd->badopts++) {
8647 					DPFPRINTF(PF_DEBUG_MISC,
8648 					    ("pf: IPv6 more than one rthdr"));
8649 					*action = PF_DROP;
8650 					REASON_SET(reason, PFRES_IPOPTIONS);
8651 					return (-1);
8652 				}
8653 				if (!pf_pull_hdr(m, *off, &rthdr, sizeof(rthdr),
8654 					NULL, reason, pd->af)) {
8655 					DPFPRINTF(PF_DEBUG_MISC,
8656 					    ("pf: IPv6 short rthdr"));
8657 					*action = PF_DROP;
8658 					REASON_SET(reason, PFRES_SHORT);
8659 					return (-1);
8660 				}
8661 				if (rthdr.ip6r_type == IPV6_RTHDR_TYPE_0) {
8662 					DPFPRINTF(PF_DEBUG_MISC,
8663 					    ("pf: IPv6 rthdr0"));
8664 					*action = PF_DROP;
8665 					REASON_SET(reason, PFRES_IPOPTIONS);
8666 					return (-1);
8667 				}
8668 				/* FALLTHROUGH */
8669 			}
8670 			case IPPROTO_AH:
8671 			case IPPROTO_HOPOPTS:
8672 			case IPPROTO_DSTOPTS: {
8673 				/* get next header and header length */
8674 				struct ip6_ext opt6;
8675 
8676 				if (!pf_pull_hdr(m, *off, &opt6, sizeof(opt6),
8677 					NULL, reason, pd->af)) {
8678 					DPFPRINTF(PF_DEBUG_MISC,
8679 					    ("pf: IPv6 short opt"));
8680 					*action = PF_DROP;
8681 					return (-1);
8682 				}
8683 				if (pd->proto == IPPROTO_AH)
8684 					*off += (opt6.ip6e_len + 2) * 4;
8685 				else
8686 					*off += (opt6.ip6e_len + 1) * 8;
8687 				pd->proto = opt6.ip6e_nxt;
8688 				/* goto the next header */
8689 				break;
8690 			}
8691 			default:
8692 				terminal++;
8693 				break;
8694 			}
8695 		} while (!terminal);
8696 
8697 		break;
8698 	}
8699 #endif
8700 	default:
8701 		panic("pf_setup_pdesc called with illegal af %u", af);
8702 	}
8703 
8704 	switch (pd->proto) {
8705 	case IPPROTO_TCP: {
8706 		struct tcphdr *th = &pd->hdr.tcp;
8707 
8708 		if (!pf_pull_hdr(m, *off, th, sizeof(*th), action,
8709 			reason, af)) {
8710 			*action = PF_DROP;
8711 			REASON_SET(reason, PFRES_SHORT);
8712 			return (-1);
8713 		}
8714 		*hdrlen = sizeof(*th);
8715 		pd->p_len = pd->tot_len - *off - (th->th_off << 2);
8716 		pd->sport = &th->th_sport;
8717 		pd->dport = &th->th_dport;
8718 		break;
8719 	}
8720 	case IPPROTO_UDP: {
8721 		struct udphdr *uh = &pd->hdr.udp;
8722 
8723 		if (!pf_pull_hdr(m, *off, uh, sizeof(*uh), action,
8724 			reason, af)) {
8725 			*action = PF_DROP;
8726 			REASON_SET(reason, PFRES_SHORT);
8727 			return (-1);
8728 		}
8729 		*hdrlen = sizeof(*uh);
8730 		if (uh->uh_dport == 0 ||
8731 		    ntohs(uh->uh_ulen) > m->m_pkthdr.len - *off ||
8732 		    ntohs(uh->uh_ulen) < sizeof(struct udphdr)) {
8733 			*action = PF_DROP;
8734 			REASON_SET(reason, PFRES_SHORT);
8735 			return (-1);
8736 		}
8737 		pd->sport = &uh->uh_sport;
8738 		pd->dport = &uh->uh_dport;
8739 		break;
8740 	}
8741 	case IPPROTO_SCTP: {
8742 		if (!pf_pull_hdr(m, *off, &pd->hdr.sctp, sizeof(pd->hdr.sctp),
8743 		    action, reason, af)) {
8744 			*action = PF_DROP;
8745 			REASON_SET(reason, PFRES_SHORT);
8746 			return (-1);
8747 		}
8748 		*hdrlen = sizeof(pd->hdr.sctp);
8749 		pd->p_len = pd->tot_len - *off;
8750 
8751 		pd->sport = &pd->hdr.sctp.src_port;
8752 		pd->dport = &pd->hdr.sctp.dest_port;
8753 		if (pd->hdr.sctp.src_port == 0 || pd->hdr.sctp.dest_port == 0) {
8754 			*action = PF_DROP;
8755 			REASON_SET(reason, PFRES_SHORT);
8756 			return (-1);
8757 		}
8758 		if (pf_scan_sctp(m, *off, pd, kif) != PF_PASS) {
8759 			*action = PF_DROP;
8760 			REASON_SET(reason, PFRES_SHORT);
8761 			return (-1);
8762 		}
8763 		break;
8764 	}
8765 	case IPPROTO_ICMP: {
8766 		if (!pf_pull_hdr(m, *off, &pd->hdr.icmp, ICMP_MINLEN,
8767 			action, reason, af)) {
8768 			*action = PF_DROP;
8769 			REASON_SET(reason, PFRES_SHORT);
8770 			return (-1);
8771 		}
8772 		*hdrlen = ICMP_MINLEN;
8773 		break;
8774 	}
8775 #ifdef INET6
8776 	case IPPROTO_ICMPV6: {
8777 		size_t icmp_hlen = sizeof(struct icmp6_hdr);
8778 
8779 		if (!pf_pull_hdr(m, *off, &pd->hdr.icmp6, icmp_hlen,
8780 			action, reason, af)) {
8781 			*action = PF_DROP;
8782 			REASON_SET(reason, PFRES_SHORT);
8783 			return (-1);
8784 		}
8785 		/* ICMP headers we look further into to match state */
8786 		switch (pd->hdr.icmp6.icmp6_type) {
8787 		case MLD_LISTENER_QUERY:
8788 		case MLD_LISTENER_REPORT:
8789 			icmp_hlen = sizeof(struct mld_hdr);
8790 			break;
8791 		case ND_NEIGHBOR_SOLICIT:
8792 		case ND_NEIGHBOR_ADVERT:
8793 			icmp_hlen = sizeof(struct nd_neighbor_solicit);
8794 			break;
8795 		}
8796 		if (icmp_hlen > sizeof(struct icmp6_hdr) &&
8797 		    !pf_pull_hdr(m, *off, &pd->hdr.icmp6, icmp_hlen,
8798 			action, reason, af)) {
8799 			*action = PF_DROP;
8800 			REASON_SET(reason, PFRES_SHORT);
8801 			return (-1);
8802 		}
8803 		*hdrlen = icmp_hlen;
8804 		break;
8805 	}
8806 #endif
8807 	}
8808 	return (0);
8809 }
8810 
8811 static void
8812 pf_counters_inc(int action, struct pf_pdesc *pd,
8813     struct pfi_kkif *kif, struct pf_kstate *s,
8814     struct pf_krule *r, struct pf_krule *a)
8815 {
8816 	struct pf_krule		*tr, *nr;
8817 	int			 dir = pd->dir;
8818 	int			 dirndx;
8819 
8820 	pf_counter_u64_critical_enter();
8821 	pf_counter_u64_add_protected(
8822 	    &kif->pfik_bytes[pd->af == AF_INET6][dir == PF_OUT][action != PF_PASS],
8823 	    pd->tot_len);
8824 	pf_counter_u64_add_protected(
8825 	    &kif->pfik_packets[pd->af == AF_INET6][dir == PF_OUT][action != PF_PASS],
8826 	    1);
8827 
8828 	if (action == PF_PASS || r->action == PF_DROP) {
8829 		dirndx = (dir == PF_OUT);
8830 		pf_counter_u64_add_protected(&r->packets[dirndx], 1);
8831 		pf_counter_u64_add_protected(&r->bytes[dirndx], pd->tot_len);
8832 		pf_update_timestamp(r);
8833 
8834 		if (a != NULL) {
8835 			pf_counter_u64_add_protected(&a->packets[dirndx], 1);
8836 			pf_counter_u64_add_protected(&a->bytes[dirndx], pd->tot_len);
8837 		}
8838 		if (s != NULL) {
8839 			struct pf_krule_item	*ri;
8840 
8841 			if (s->nat_rule.ptr != NULL) {
8842 				pf_counter_u64_add_protected(&s->nat_rule.ptr->packets[dirndx],
8843 				    1);
8844 				pf_counter_u64_add_protected(&s->nat_rule.ptr->bytes[dirndx],
8845 				    pd->tot_len);
8846 			}
8847 			if (s->src_node != NULL) {
8848 				counter_u64_add(s->src_node->packets[dirndx],
8849 				    1);
8850 				counter_u64_add(s->src_node->bytes[dirndx],
8851 				    pd->tot_len);
8852 			}
8853 			if (s->nat_src_node != NULL) {
8854 				counter_u64_add(s->nat_src_node->packets[dirndx],
8855 				    1);
8856 				counter_u64_add(s->nat_src_node->bytes[dirndx],
8857 				    pd->tot_len);
8858 			}
8859 			dirndx = (dir == s->direction) ? 0 : 1;
8860 			s->packets[dirndx]++;
8861 			s->bytes[dirndx] += pd->tot_len;
8862 
8863 			SLIST_FOREACH(ri, &s->match_rules, entry) {
8864 				pf_counter_u64_add_protected(&ri->r->packets[dirndx], 1);
8865 				pf_counter_u64_add_protected(&ri->r->bytes[dirndx], pd->tot_len);
8866 			}
8867 		}
8868 		tr = r;
8869 		nr = (s != NULL) ? s->nat_rule.ptr : pd->nat_rule;
8870 		if (nr != NULL && r == &V_pf_default_rule)
8871 			tr = nr;
8872 		if (tr->src.addr.type == PF_ADDR_TABLE)
8873 			pfr_update_stats(tr->src.addr.p.tbl,
8874 			    (s == NULL) ? pd->src :
8875 			    &s->key[(s->direction == PF_IN)]->
8876 				addr[(s->direction == PF_OUT)],
8877 			    pd->af, pd->tot_len, dir == PF_OUT,
8878 			    r->action == PF_PASS, tr->src.neg);
8879 		if (tr->dst.addr.type == PF_ADDR_TABLE)
8880 			pfr_update_stats(tr->dst.addr.p.tbl,
8881 			    (s == NULL) ? pd->dst :
8882 			    &s->key[(s->direction == PF_IN)]->
8883 				addr[(s->direction == PF_IN)],
8884 			    pd->af, pd->tot_len, dir == PF_OUT,
8885 			    r->action == PF_PASS, tr->dst.neg);
8886 	}
8887 	pf_counter_u64_critical_exit();
8888 }
8889 
8890 #if defined(INET) || defined(INET6)
8891 int
8892 pf_test(sa_family_t af, int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
8893     struct inpcb *inp, struct pf_rule_actions *default_actions)
8894 {
8895 	struct pfi_kkif		*kif;
8896 	u_short			 action, reason = 0;
8897 	struct mbuf		*m = *m0;
8898 	struct ip		*h = NULL;
8899 	struct ip6_hdr		*h6 = NULL;
8900 	struct m_tag		*mtag;
8901 	struct pf_krule		*a = NULL, *r = &V_pf_default_rule;
8902 	struct pf_kstate	*s = NULL;
8903 	struct pf_kruleset	*ruleset = NULL;
8904 	struct pf_pdesc		 pd;
8905 	int			 off, hdrlen, use_2nd_queue = 0;
8906 	uint16_t		 tag;
8907 	uint8_t			 rt;
8908 	uint8_t			 ttl;
8909 
8910 	PF_RULES_RLOCK_TRACKER;
8911 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: bad direction %d\n", __func__, dir));
8912 	M_ASSERTPKTHDR(m);
8913 
8914 	if (!V_pf_status.running)
8915 		return (PF_PASS);
8916 
8917 	PF_RULES_RLOCK();
8918 
8919 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
8920 
8921 	if (__predict_false(kif == NULL)) {
8922 		DPFPRINTF(PF_DEBUG_URGENT,
8923 		    ("pf_test: kif == NULL, if_xname %s\n", ifp->if_xname));
8924 		PF_RULES_RUNLOCK();
8925 		return (PF_DROP);
8926 	}
8927 	if (kif->pfik_flags & PFI_IFLAG_SKIP) {
8928 		PF_RULES_RUNLOCK();
8929 		return (PF_PASS);
8930 	}
8931 
8932 	if (m->m_flags & M_SKIP_FIREWALL) {
8933 		PF_RULES_RUNLOCK();
8934 		return (PF_PASS);
8935 	}
8936 
8937 	if (__predict_false(! M_WRITABLE(*m0))) {
8938 		m = *m0 = m_unshare(*m0, M_NOWAIT);
8939 		if (*m0 == NULL)
8940 			return (PF_DROP);
8941 	}
8942 
8943 	memset(&pd, 0, sizeof(pd));
8944 	pd.dir = dir;
8945 
8946 	switch (af) {
8947 #ifdef INET
8948 	case AF_INET:
8949 		if (__predict_false(m->m_len < sizeof(struct ip)) &&
8950 		    (m = *m0 = m_pullup(*m0, sizeof(struct ip))) == NULL) {
8951 			DPFPRINTF(PF_DEBUG_URGENT,
8952 			    ("pf_test: m_len < sizeof(struct ip), pullup failed\n"));
8953 			PF_RULES_RUNLOCK();
8954 			return (PF_DROP);
8955 		}
8956 
8957 		if (pf_normalize_ip(m0, kif, &reason, &pd) != PF_PASS) {
8958 			/* We do IP header normalization and packet reassembly here */
8959 			action = PF_DROP;
8960 			goto done;
8961 		}
8962 		m = *m0;	/* pf_normalize messes with m0 */
8963 		h = mtod(m, struct ip *);
8964 		ttl = h->ip_ttl;
8965 		break;
8966 #endif
8967 #ifdef INET6
8968 	case AF_INET6:
8969 		/*
8970 		 * If we end up changing IP addresses (e.g. binat) the stack may get
8971 		 * confused and fail to send the icmp6 packet too big error. Just send
8972 		 * it here, before we do any NAT.
8973 		 */
8974 		if (dir == PF_OUT && pflags & PFIL_FWD && IN6_LINKMTU(ifp) < pf_max_frag_size(m)) {
8975 			PF_RULES_RUNLOCK();
8976 			*m0 = NULL;
8977 			icmp6_error(m, ICMP6_PACKET_TOO_BIG, 0, IN6_LINKMTU(ifp));
8978 			return (PF_DROP);
8979 		}
8980 
8981 		if (__predict_false(m->m_len < sizeof(struct ip6_hdr)) &&
8982 		    (m = *m0 = m_pullup(*m0, sizeof(struct ip6_hdr))) == NULL) {
8983 			DPFPRINTF(PF_DEBUG_URGENT,
8984 			    ("pf_test6: m_len < sizeof(struct ip6_hdr)"
8985 			     ", pullup failed\n"));
8986 			PF_RULES_RUNLOCK();
8987 			return (PF_DROP);
8988 		}
8989 
8990 		/* We do IP header normalization and packet reassembly here */
8991 		if (pf_normalize_ip6(m0, kif, &reason, &pd) != PF_PASS) {
8992 			action = PF_DROP;
8993 			goto done;
8994 		}
8995 		m = *m0;	/* pf_normalize messes with m0 */
8996 		h6 = mtod(m, struct ip6_hdr *);
8997 		ttl = h6->ip6_hlim;
8998 		break;
8999 #endif
9000 	default:
9001 		panic("Unknown af %d", af);
9002 	}
9003 
9004 	if (pf_setup_pdesc(af, dir, &pd, m, &action, &reason, kif, &a, &r,
9005 		&ruleset, &off, &hdrlen, default_actions) == -1) {
9006 		if (action != PF_PASS)
9007 			pd.act.log |= PF_LOG_FORCE;
9008 		goto done;
9009 	}
9010 
9011 	if (pd.pf_mtag != NULL && (pd.pf_mtag->flags & PF_MTAG_FLAG_ROUTE_TO)) {
9012 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
9013 
9014 		ifp = ifnet_byindexgen(pd.pf_mtag->if_index,
9015 		    pd.pf_mtag->if_idxgen);
9016 		if (ifp == NULL || ifp->if_flags & IFF_DYING) {
9017 			PF_RULES_RUNLOCK();
9018 			m_freem(*m0);
9019 			*m0 = NULL;
9020 			return (PF_PASS);
9021 		}
9022 		PF_RULES_RUNLOCK();
9023 		(ifp->if_output)(ifp, m, sintosa(&pd.pf_mtag->dst), NULL);
9024 		*m0 = NULL;
9025 		return (PF_PASS);
9026 	}
9027 
9028 	if (ip_dn_io_ptr != NULL && pd.pf_mtag != NULL &&
9029 	    pd.pf_mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
9030 		/* Dummynet re-injects packets after they've
9031 		 * completed their delay. We've already
9032 		 * processed them, so pass unconditionally. */
9033 
9034 		/* But only once. We may see the packet multiple times (e.g.
9035 		 * PFIL_IN/PFIL_OUT). */
9036 		pf_dummynet_flag_remove(m, pd.pf_mtag);
9037 		PF_RULES_RUNLOCK();
9038 
9039 		return (PF_PASS);
9040 	}
9041 
9042 	if (__predict_false(ip_divert_ptr != NULL) &&
9043 	    ((mtag = m_tag_locate(m, MTAG_PF_DIVERT, 0, NULL)) != NULL)) {
9044 		struct pf_divert_mtag *dt = (struct pf_divert_mtag *)(mtag+1);
9045 		if ((dt->idir == PF_DIVERT_MTAG_DIR_IN && dir == PF_IN) ||
9046 		    (dt->idir == PF_DIVERT_MTAG_DIR_OUT && dir == PF_OUT)) {
9047 			if (pd.pf_mtag == NULL &&
9048 			    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
9049 				action = PF_DROP;
9050 				goto done;
9051 			}
9052 			pd.pf_mtag->flags |= PF_MTAG_FLAG_PACKET_LOOPED;
9053 		}
9054 		if (pd.pf_mtag && pd.pf_mtag->flags & PF_MTAG_FLAG_FASTFWD_OURS_PRESENT) {
9055 			m->m_flags |= M_FASTFWD_OURS;
9056 			pd.pf_mtag->flags &= ~PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
9057 		}
9058 		m_tag_delete(m, mtag);
9059 
9060 		mtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL);
9061 		if (mtag != NULL)
9062 			m_tag_delete(m, mtag);
9063 	}
9064 
9065 	switch (af) {
9066 #ifdef INET
9067 	case AF_INET:
9068 		/* handle fragments that didn't get reassembled by normalization */
9069 		if (h->ip_off & htons(IP_MF | IP_OFFMASK)) {
9070 			action = pf_test_fragment(&r, kif, m, &pd, &a, &ruleset);
9071 			goto done;
9072 		}
9073 		break;
9074 #endif
9075 #ifdef INET6
9076 	case AF_INET6:
9077 		/*
9078 		 * we do not support jumbogram.  if we keep going, zero ip6_plen
9079 		 * will do something bad, so drop the packet for now.
9080 		 */
9081 		if (htons(h6->ip6_plen) == 0) {
9082 			action = PF_DROP;
9083 			REASON_SET(&reason, PFRES_NORM);	/*XXX*/
9084 			goto done;
9085 		}
9086 		break;
9087 #endif
9088 	default:
9089 		panic("Unknown af %d", af);
9090 	}
9091 
9092 	switch (pd.proto) {
9093 	case IPPROTO_TCP: {
9094 		/* Respond to SYN with a syncookie. */
9095 		if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) == TH_SYN &&
9096 		    pd.dir == PF_IN && pf_synflood_check(&pd)) {
9097 			pf_syncookie_send(m, off, &pd);
9098 			action = PF_DROP;
9099 			break;
9100 		}
9101 
9102 		if ((pd.hdr.tcp.th_flags & TH_ACK) && pd.p_len == 0)
9103 			use_2nd_queue = 1;
9104 		action = pf_normalize_tcp(kif, m, 0, off, &pd);
9105 		if (action == PF_DROP)
9106 			goto done;
9107 		action = pf_test_state_tcp(&s, kif, m, off, &pd, &reason);
9108 		if (action == PF_PASS) {
9109 			if (V_pfsync_update_state_ptr != NULL)
9110 				V_pfsync_update_state_ptr(s);
9111 			r = s->rule.ptr;
9112 			a = s->anchor.ptr;
9113 		} else if (s == NULL) {
9114 			/* Validate remote SYN|ACK, re-create original SYN if
9115 			 * valid. */
9116 			if ((pd.hdr.tcp.th_flags & (TH_SYN|TH_ACK|TH_RST)) ==
9117 			    TH_ACK && pf_syncookie_validate(&pd) &&
9118 			    pd.dir == PF_IN) {
9119 				struct mbuf *msyn;
9120 
9121 				msyn = pf_syncookie_recreate_syn(ttl, off,
9122 				    &pd);
9123 				if (msyn == NULL) {
9124 					action = PF_DROP;
9125 					break;
9126 				}
9127 
9128 				action = pf_test(af, dir, pflags, ifp, &msyn, inp,
9129 				    &pd.act);
9130 				m_freem(msyn);
9131 				if (action != PF_PASS)
9132 					break;
9133 
9134 				action = pf_test_state_tcp(&s, kif, m, off,
9135 				    &pd, &reason);
9136 				if (action != PF_PASS || s == NULL) {
9137 					action = PF_DROP;
9138 					break;
9139 				}
9140 
9141 				s->src.seqhi = ntohl(pd.hdr.tcp.th_ack) - 1;
9142 				s->src.seqlo = ntohl(pd.hdr.tcp.th_seq) - 1;
9143 				pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_DST);
9144 				action = pf_synproxy(&pd, &s, &reason);
9145 				break;
9146 			} else {
9147 				action = pf_test_rule(&r, &s, kif, m, off, &pd,
9148 				    &a, &ruleset, inp, hdrlen);
9149 			}
9150 		}
9151 		break;
9152 	}
9153 
9154 	case IPPROTO_UDP: {
9155 		action = pf_test_state_udp(&s, kif, m, off, &pd);
9156 		if (action == PF_PASS) {
9157 			if (V_pfsync_update_state_ptr != NULL)
9158 				V_pfsync_update_state_ptr(s);
9159 			r = s->rule.ptr;
9160 			a = s->anchor.ptr;
9161 		} else if (s == NULL)
9162 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9163 			    &a, &ruleset, inp, hdrlen);
9164 		break;
9165 	}
9166 
9167 	case IPPROTO_SCTP: {
9168 		action = pf_normalize_sctp(dir, kif, m, 0, off, &pd);
9169 		if (action == PF_DROP)
9170 			goto done;
9171 		action = pf_test_state_sctp(&s, kif, m, off, &pd,
9172 		    &reason);
9173 		if (action == PF_PASS) {
9174 			if (V_pfsync_update_state_ptr != NULL)
9175 				V_pfsync_update_state_ptr(s);
9176 			r = s->rule.ptr;
9177 			a = s->anchor.ptr;
9178 		} else if (s == NULL) {
9179 			action = pf_test_rule(&r, &s, kif, m, off,
9180 			    &pd, &a, &ruleset, inp, hdrlen);
9181 		}
9182 		break;
9183 	}
9184 
9185 	case IPPROTO_ICMP: {
9186 		if (af != AF_INET) {
9187 			action = PF_DROP;
9188 			DPFPRINTF(PF_DEBUG_MISC,
9189 			    ("dropping IPv6 packet with ICMPv4 payload"));
9190 			goto done;
9191 		}
9192 		action = pf_test_state_icmp(&s, kif, m, off, &pd, &reason);
9193 		if (action == PF_PASS) {
9194 			if (V_pfsync_update_state_ptr != NULL)
9195 				V_pfsync_update_state_ptr(s);
9196 			r = s->rule.ptr;
9197 			a = s->anchor.ptr;
9198 		} else if (s == NULL)
9199 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9200 			    &a, &ruleset, inp, hdrlen);
9201 		break;
9202 	}
9203 
9204 	case IPPROTO_ICMPV6: {
9205 		if (af != AF_INET6) {
9206 			action = PF_DROP;
9207 			DPFPRINTF(PF_DEBUG_MISC,
9208 			    ("pf: dropping IPv4 packet with ICMPv6 payload\n"));
9209 			goto done;
9210 		}
9211 		action = pf_test_state_icmp(&s, kif, m, off, &pd, &reason);
9212 		if (action == PF_PASS) {
9213 			if (V_pfsync_update_state_ptr != NULL)
9214 				V_pfsync_update_state_ptr(s);
9215 			r = s->rule.ptr;
9216 			a = s->anchor.ptr;
9217 		} else if (s == NULL)
9218 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9219 			    &a, &ruleset, inp, hdrlen);
9220 		break;
9221 	}
9222 
9223 	default:
9224 		action = pf_test_state_other(&s, kif, m, &pd);
9225 		if (action == PF_PASS) {
9226 			if (V_pfsync_update_state_ptr != NULL)
9227 				V_pfsync_update_state_ptr(s);
9228 			r = s->rule.ptr;
9229 			a = s->anchor.ptr;
9230 		} else if (s == NULL)
9231 			action = pf_test_rule(&r, &s, kif, m, off, &pd,
9232 			    &a, &ruleset, inp, hdrlen);
9233 		break;
9234 	}
9235 
9236 done:
9237 	PF_RULES_RUNLOCK();
9238 
9239 	if (action == PF_PASS && pd.badopts &&
9240 	    !((s && s->state_flags & PFSTATE_ALLOWOPTS) || r->allow_opts)) {
9241 		action = PF_DROP;
9242 		REASON_SET(&reason, PFRES_IPOPTIONS);
9243 		pd.act.log = PF_LOG_FORCE;
9244 		DPFPRINTF(PF_DEBUG_MISC,
9245 		    ("pf: dropping packet with dangerous headers\n"));
9246 	}
9247 
9248 	if (s) {
9249 		uint8_t log = pd.act.log;
9250 		memcpy(&pd.act, &s->act, sizeof(struct pf_rule_actions));
9251 		pd.act.log |= log;
9252 		tag = s->tag;
9253 		rt = s->rt;
9254 	} else {
9255 		tag = r->tag;
9256 		rt = r->rt;
9257 	}
9258 
9259 	if (tag > 0 && pf_tag_packet(m, &pd, tag)) {
9260 		action = PF_DROP;
9261 		REASON_SET(&reason, PFRES_MEMORY);
9262 	}
9263 
9264 	pf_scrub(m, &pd);
9265 	if (pd.proto == IPPROTO_TCP && pd.act.max_mss)
9266 		pf_normalize_mss(m, off, &pd);
9267 
9268 	if (pd.act.rtableid >= 0)
9269 		M_SETFIB(m, pd.act.rtableid);
9270 
9271 	if (pd.act.flags & PFSTATE_SETPRIO) {
9272 		if (pd.tos & IPTOS_LOWDELAY)
9273 			use_2nd_queue = 1;
9274 		if (vlan_set_pcp(m, pd.act.set_prio[use_2nd_queue])) {
9275 			action = PF_DROP;
9276 			REASON_SET(&reason, PFRES_MEMORY);
9277 			pd.act.log = PF_LOG_FORCE;
9278 			DPFPRINTF(PF_DEBUG_MISC,
9279 			    ("pf: failed to allocate 802.1q mtag\n"));
9280 		}
9281 	}
9282 
9283 #ifdef ALTQ
9284 	if (action == PF_PASS && pd.act.qid) {
9285 		if (pd.pf_mtag == NULL &&
9286 		    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
9287 			action = PF_DROP;
9288 			REASON_SET(&reason, PFRES_MEMORY);
9289 		} else {
9290 			if (s != NULL)
9291 				pd.pf_mtag->qid_hash = pf_state_hash(s);
9292 			if (use_2nd_queue || (pd.tos & IPTOS_LOWDELAY))
9293 				pd.pf_mtag->qid = pd.act.pqid;
9294 			else
9295 				pd.pf_mtag->qid = pd.act.qid;
9296 			/* Add hints for ecn. */
9297 			if (af == AF_INET)
9298 				pd.pf_mtag->hdr = h;
9299 			else
9300 				pd.pf_mtag->hdr = h6;
9301 		}
9302 	}
9303 #endif /* ALTQ */
9304 
9305 	/*
9306 	 * connections redirected to loopback should not match sockets
9307 	 * bound specifically to loopback due to security implications,
9308 	 * see tcp_input() and in_pcblookup_listen().
9309 	 */
9310 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
9311 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL &&
9312 	    (s->nat_rule.ptr->action == PF_RDR ||
9313 	    s->nat_rule.ptr->action == PF_BINAT) &&
9314 	    pf_is_loopback(af, pd.dst))
9315 		m->m_flags |= M_SKIP_FIREWALL;
9316 
9317 	if (af == AF_INET && __predict_false(ip_divert_ptr != NULL) &&
9318 	    action == PF_PASS && r->divert.port && !PACKET_LOOPED(&pd)) {
9319 		mtag = m_tag_alloc(MTAG_PF_DIVERT, 0,
9320 		    sizeof(struct pf_divert_mtag), M_NOWAIT | M_ZERO);
9321 		if (mtag != NULL) {
9322 			((struct pf_divert_mtag *)(mtag+1))->port =
9323 			    ntohs(r->divert.port);
9324 			((struct pf_divert_mtag *)(mtag+1))->idir =
9325 			    (dir == PF_IN) ? PF_DIVERT_MTAG_DIR_IN :
9326 			    PF_DIVERT_MTAG_DIR_OUT;
9327 
9328 			if (s)
9329 				PF_STATE_UNLOCK(s);
9330 
9331 			m_tag_prepend(m, mtag);
9332 			if (m->m_flags & M_FASTFWD_OURS) {
9333 				if (pd.pf_mtag == NULL &&
9334 				    ((pd.pf_mtag = pf_get_mtag(m)) == NULL)) {
9335 					action = PF_DROP;
9336 					REASON_SET(&reason, PFRES_MEMORY);
9337 					pd.act.log = PF_LOG_FORCE;
9338 					DPFPRINTF(PF_DEBUG_MISC,
9339 					    ("pf: failed to allocate tag\n"));
9340 				} else {
9341 					pd.pf_mtag->flags |=
9342 					    PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
9343 					m->m_flags &= ~M_FASTFWD_OURS;
9344 				}
9345 			}
9346 			ip_divert_ptr(*m0, dir == PF_IN);
9347 			*m0 = NULL;
9348 
9349 			return (action);
9350 		} else {
9351 			/* XXX: ipfw has the same behaviour! */
9352 			action = PF_DROP;
9353 			REASON_SET(&reason, PFRES_MEMORY);
9354 			pd.act.log = PF_LOG_FORCE;
9355 			DPFPRINTF(PF_DEBUG_MISC,
9356 			    ("pf: failed to allocate divert tag\n"));
9357 		}
9358 	}
9359 	/* XXX: Anybody working on it?! */
9360 	if (af == AF_INET6 && r->divert.port)
9361 		printf("pf: divert(9) is not supported for IPv6\n");
9362 
9363 	/* this flag will need revising if the pkt is forwarded */
9364 	if (pd.pf_mtag)
9365 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_PACKET_LOOPED;
9366 
9367 	if (pd.act.log) {
9368 		struct pf_krule		*lr;
9369 		struct pf_krule_item	*ri;
9370 
9371 		if (s != NULL && s->nat_rule.ptr != NULL &&
9372 		    s->nat_rule.ptr->log & PF_LOG_ALL)
9373 			lr = s->nat_rule.ptr;
9374 		else
9375 			lr = r;
9376 
9377 		if (pd.act.log & PF_LOG_FORCE || lr->log & PF_LOG_ALL)
9378 			PFLOG_PACKET(kif, m, action, reason, lr, a,
9379 			    ruleset, &pd, (s == NULL));
9380 		if (s) {
9381 			SLIST_FOREACH(ri, &s->match_rules, entry)
9382 				if (ri->r->log & PF_LOG_ALL)
9383 					PFLOG_PACKET(kif, m, action,
9384 					    reason, ri->r, a, ruleset, &pd, 0);
9385 		}
9386 	}
9387 
9388 	pf_counters_inc(action, &pd, kif, s, r, a);
9389 
9390 	switch (action) {
9391 	case PF_SYNPROXY_DROP:
9392 		m_freem(*m0);
9393 	case PF_DEFER:
9394 		*m0 = NULL;
9395 		action = PF_PASS;
9396 		break;
9397 	case PF_DROP:
9398 		m_freem(*m0);
9399 		*m0 = NULL;
9400 		break;
9401 	default:
9402 		if (rt) {
9403 			switch (af) {
9404 #ifdef INET
9405 			case AF_INET:
9406 				/* pf_route() returns unlocked. */
9407 				pf_route(m0, r, kif->pfik_ifp, s, &pd, inp);
9408 				break;
9409 #endif
9410 #ifdef INET6
9411 			case AF_INET6:
9412 				/* pf_route6() returns unlocked. */
9413 				pf_route6(m0, r, kif->pfik_ifp, s, &pd, inp);
9414 				break;
9415 #endif
9416 			default:
9417 				panic("Unknown af %d", af);
9418 			}
9419 			goto out;
9420 		}
9421 		if (pf_dummynet(&pd, s, r, m0) != 0) {
9422 			action = PF_DROP;
9423 			REASON_SET(&reason, PFRES_MEMORY);
9424 		}
9425 		break;
9426 	}
9427 
9428 	SDT_PROBE4(pf, ip, test, done, action, reason, r, s);
9429 
9430 	if (s && action != PF_DROP) {
9431 		if (!s->if_index_in && dir == PF_IN)
9432 			s->if_index_in = ifp->if_index;
9433 		else if (!s->if_index_out && dir == PF_OUT)
9434 			s->if_index_out = ifp->if_index;
9435 	}
9436 
9437 	if (s)
9438 		PF_STATE_UNLOCK(s);
9439 
9440 #ifdef INET6
9441 	/* If reassembled packet passed, create new fragments. */
9442 	if (af == AF_INET6 && action == PF_PASS && *m0 && dir == PF_OUT &&
9443 	    (mtag = m_tag_find(m, PACKET_TAG_PF_REASSEMBLED, NULL)) != NULL)
9444 		action = pf_refragment6(ifp, m0, mtag, pflags & PFIL_FWD);
9445 #endif
9446 
9447 out:
9448 	pf_sctp_multihome_delayed(&pd, off, kif, s, action);
9449 
9450 	return (action);
9451 }
9452 #endif /* INET || INET6 */
9453