xref: /freebsd/sys/netpfil/pf/pf.c (revision c23eda976a8aad6bbd6c2042fa2ba1f0bc640e19) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2001 Daniel Hartmeier
5  * Copyright (c) 2002 - 2008 Henning Brauer
6  * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org>
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  *
13  *    - Redistributions of source code must retain the above copyright
14  *      notice, this list of conditions and the following disclaimer.
15  *    - Redistributions in binary form must reproduce the above
16  *      copyright notice, this list of conditions and the following
17  *      disclaimer in the documentation and/or other materials provided
18  *      with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGE.
32  *
33  * Effort sponsored in part by the Defense Advanced Research Projects
34  * Agency (DARPA) and Air Force Research Laboratory, Air Force
35  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
36  *
37  *	$OpenBSD: pf.c,v 1.634 2009/02/27 12:37:45 henning Exp $
38  */
39 
40 #include <sys/cdefs.h>
41 #include "opt_bpf.h"
42 #include "opt_inet.h"
43 #include "opt_inet6.h"
44 #include "opt_pf.h"
45 #include "opt_sctp.h"
46 
47 #include <sys/param.h>
48 #include <sys/bus.h>
49 #include <sys/endian.h>
50 #include <sys/gsb_crc32.h>
51 #include <sys/hash.h>
52 #include <sys/interrupt.h>
53 #include <sys/kernel.h>
54 #include <sys/kthread.h>
55 #include <sys/limits.h>
56 #include <sys/mbuf.h>
57 #include <sys/random.h>
58 #include <sys/refcount.h>
59 #include <sys/sdt.h>
60 #include <sys/socket.h>
61 #include <sys/sysctl.h>
62 #include <sys/taskqueue.h>
63 #include <sys/ucred.h>
64 
65 #include <crypto/sha2/sha512.h>
66 
67 #include <net/if.h>
68 #include <net/if_var.h>
69 #include <net/if_private.h>
70 #include <net/if_types.h>
71 #include <net/if_vlan_var.h>
72 #include <net/route.h>
73 #include <net/route/nhop.h>
74 #include <net/vnet.h>
75 
76 #include <net/pfil.h>
77 #include <net/pfvar.h>
78 #include <net/if_pflog.h>
79 #include <net/if_pfsync.h>
80 
81 #include <netinet/in_pcb.h>
82 #include <netinet/in_var.h>
83 #include <netinet/in_fib.h>
84 #include <netinet/ip.h>
85 #include <netinet/ip_fw.h>
86 #include <netinet/ip_icmp.h>
87 #include <netinet/icmp_var.h>
88 #include <netinet/ip_var.h>
89 #include <netinet/tcp.h>
90 #include <netinet/tcp_fsm.h>
91 #include <netinet/tcp_seq.h>
92 #include <netinet/tcp_timer.h>
93 #include <netinet/tcp_var.h>
94 #include <netinet/udp.h>
95 #include <netinet/udp_var.h>
96 
97 /* dummynet */
98 #include <netinet/ip_dummynet.h>
99 #include <netinet/ip_fw.h>
100 #include <netpfil/ipfw/dn_heap.h>
101 #include <netpfil/ipfw/ip_fw_private.h>
102 #include <netpfil/ipfw/ip_dn_private.h>
103 
104 #ifdef INET6
105 #include <netinet/ip6.h>
106 #include <netinet/icmp6.h>
107 #include <netinet6/nd6.h>
108 #include <netinet6/ip6_var.h>
109 #include <netinet6/in6_pcb.h>
110 #include <netinet6/in6_fib.h>
111 #include <netinet6/scope6_var.h>
112 #endif /* INET6 */
113 
114 #include <netinet/sctp_header.h>
115 #include <netinet/sctp_crc32.h>
116 
117 #include <netipsec/ah.h>
118 
119 #include <machine/in_cksum.h>
120 #include <security/mac/mac_framework.h>
121 
122 SDT_PROVIDER_DEFINE(pf);
123 SDT_PROBE_DEFINE2(pf, , test, reason_set, "int", "int");
124 SDT_PROBE_DEFINE4(pf, ip, test, done, "int", "int", "struct pf_krule *",
125     "struct pf_kstate *");
126 SDT_PROBE_DEFINE5(pf, ip, state, lookup, "struct pfi_kkif *",
127     "struct pf_state_key_cmp *", "int", "struct pf_pdesc *",
128     "struct pf_kstate *");
129 SDT_PROBE_DEFINE2(pf, ip, , bound_iface, "struct pf_kstate *",
130     "struct pfi_kkif *");
131 SDT_PROBE_DEFINE4(pf, ip, route_to, entry, "struct mbuf *",
132     "struct pf_pdesc *", "struct pf_kstate *", "struct ifnet *");
133 SDT_PROBE_DEFINE1(pf, ip, route_to, drop, "int");
134 SDT_PROBE_DEFINE2(pf, ip, route_to, output, "struct ifnet *", "int");
135 SDT_PROBE_DEFINE4(pf, ip6, route_to, entry, "struct mbuf *",
136     "struct pf_pdesc *", "struct pf_kstate *", "struct ifnet *");
137 SDT_PROBE_DEFINE1(pf, ip6, route_to, drop, "int");
138 SDT_PROBE_DEFINE2(pf, ip6, route_to, output, "struct ifnet *", "int");
139 SDT_PROBE_DEFINE4(pf, sctp, multihome, test, "struct pfi_kkif *",
140     "struct pf_krule *", "struct mbuf *", "int");
141 SDT_PROBE_DEFINE2(pf, sctp, multihome, add, "uint32_t",
142     "struct pf_sctp_source *");
143 SDT_PROBE_DEFINE3(pf, sctp, multihome, remove, "uint32_t",
144     "struct pf_kstate *", "struct pf_sctp_source *");
145 SDT_PROBE_DEFINE4(pf, sctp, multihome_scan, entry, "int",
146     "int", "struct pf_pdesc *", "int");
147 SDT_PROBE_DEFINE2(pf, sctp, multihome_scan, param, "uint16_t", "uint16_t");
148 SDT_PROBE_DEFINE2(pf, sctp, multihome_scan, ipv4, "struct in_addr *",
149     "int");
150 SDT_PROBE_DEFINE2(pf, sctp, multihome_scan, ipv6, "struct in_addr6 *",
151     "int");
152 
153 SDT_PROBE_DEFINE3(pf, eth, test_rule, entry, "int", "struct ifnet *",
154     "struct mbuf *");
155 SDT_PROBE_DEFINE2(pf, eth, test_rule, test, "int", "struct pf_keth_rule *");
156 SDT_PROBE_DEFINE3(pf, eth, test_rule, mismatch,
157     "int", "struct pf_keth_rule *", "char *");
158 SDT_PROBE_DEFINE2(pf, eth, test_rule, match, "int", "struct pf_keth_rule *");
159 SDT_PROBE_DEFINE2(pf, eth, test_rule, final_match,
160     "int", "struct pf_keth_rule *");
161 SDT_PROBE_DEFINE2(pf, purge, state, rowcount, "int", "size_t");
162 SDT_PROBE_DEFINE2(pf, , log, log, "int", "const char *");
163 
164 /*
165  * Global variables
166  */
167 
168 /* state tables */
169 VNET_DEFINE(struct pf_altqqueue,	 pf_altqs[4]);
170 VNET_DEFINE(struct pf_kpalist,		 pf_pabuf[3]);
171 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_active);
172 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_active);
173 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_inactive);
174 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_inactive);
175 VNET_DEFINE(struct pf_kstatus,		 pf_status);
176 
177 VNET_DEFINE(u_int32_t,			 ticket_altqs_active);
178 VNET_DEFINE(u_int32_t,			 ticket_altqs_inactive);
179 VNET_DEFINE(int,			 altqs_inactive_open);
180 VNET_DEFINE(u_int32_t,			 ticket_pabuf);
181 
182 static const int			 PF_HDR_LIMIT = 20;	/* arbitrary limit */
183 
184 VNET_DEFINE(SHA512_CTX,			 pf_tcp_secret_ctx);
185 #define	V_pf_tcp_secret_ctx		 VNET(pf_tcp_secret_ctx)
186 VNET_DEFINE(u_char,			 pf_tcp_secret[16]);
187 #define	V_pf_tcp_secret			 VNET(pf_tcp_secret)
188 VNET_DEFINE(int,			 pf_tcp_secret_init);
189 #define	V_pf_tcp_secret_init		 VNET(pf_tcp_secret_init)
190 VNET_DEFINE(int,			 pf_tcp_iss_off);
191 #define	V_pf_tcp_iss_off		 VNET(pf_tcp_iss_off)
192 VNET_DECLARE(int,			 pf_vnet_active);
193 #define	V_pf_vnet_active		 VNET(pf_vnet_active)
194 
195 VNET_DEFINE_STATIC(uint32_t, pf_purge_idx);
196 #define V_pf_purge_idx	VNET(pf_purge_idx)
197 
198 #ifdef PF_WANT_32_TO_64_COUNTER
199 VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter);
200 #define	V_pf_counter_periodic_iter	VNET(pf_counter_periodic_iter)
201 
202 VNET_DEFINE(struct allrulelist_head, pf_allrulelist);
203 VNET_DEFINE(size_t, pf_allrulecount);
204 VNET_DEFINE(struct pf_krule *, pf_rulemarker);
205 #endif
206 
207 #define PF_SCTP_MAX_ENDPOINTS		8
208 
209 struct pf_sctp_endpoint;
210 RB_HEAD(pf_sctp_endpoints, pf_sctp_endpoint);
211 struct pf_sctp_source {
212 	sa_family_t			af;
213 	struct pf_addr			addr;
214 	TAILQ_ENTRY(pf_sctp_source)	entry;
215 };
216 TAILQ_HEAD(pf_sctp_sources, pf_sctp_source);
217 struct pf_sctp_endpoint
218 {
219 	uint32_t		 v_tag;
220 	struct pf_sctp_sources	 sources;
221 	RB_ENTRY(pf_sctp_endpoint)	entry;
222 };
223 static int
224 pf_sctp_endpoint_compare(struct pf_sctp_endpoint *a, struct pf_sctp_endpoint *b)
225 {
226 	return (a->v_tag - b->v_tag);
227 }
228 RB_PROTOTYPE(pf_sctp_endpoints, pf_sctp_endpoint, entry, pf_sctp_endpoint_compare);
229 RB_GENERATE(pf_sctp_endpoints, pf_sctp_endpoint, entry, pf_sctp_endpoint_compare);
230 VNET_DEFINE_STATIC(struct pf_sctp_endpoints, pf_sctp_endpoints);
231 #define V_pf_sctp_endpoints	VNET(pf_sctp_endpoints)
232 static struct mtx_padalign pf_sctp_endpoints_mtx;
233 MTX_SYSINIT(pf_sctp_endpoints_mtx, &pf_sctp_endpoints_mtx, "SCTP endpoints", MTX_DEF);
234 #define	PF_SCTP_ENDPOINTS_LOCK()	mtx_lock(&pf_sctp_endpoints_mtx)
235 #define	PF_SCTP_ENDPOINTS_UNLOCK()	mtx_unlock(&pf_sctp_endpoints_mtx)
236 
237 /*
238  * Queue for pf_intr() sends.
239  */
240 static MALLOC_DEFINE(M_PFTEMP, "pf_temp", "pf(4) temporary allocations");
241 struct pf_send_entry {
242 	STAILQ_ENTRY(pf_send_entry)	pfse_next;
243 	struct mbuf			*pfse_m;
244 	enum {
245 		PFSE_IP,
246 		PFSE_IP6,
247 		PFSE_ICMP,
248 		PFSE_ICMP6,
249 	}				pfse_type;
250 	struct {
251 		int		type;
252 		int		code;
253 		int		mtu;
254 	} icmpopts;
255 };
256 
257 STAILQ_HEAD(pf_send_head, pf_send_entry);
258 VNET_DEFINE_STATIC(struct pf_send_head, pf_sendqueue);
259 #define	V_pf_sendqueue	VNET(pf_sendqueue)
260 
261 static struct mtx_padalign pf_sendqueue_mtx;
262 MTX_SYSINIT(pf_sendqueue_mtx, &pf_sendqueue_mtx, "pf send queue", MTX_DEF);
263 #define	PF_SENDQ_LOCK()		mtx_lock(&pf_sendqueue_mtx)
264 #define	PF_SENDQ_UNLOCK()	mtx_unlock(&pf_sendqueue_mtx)
265 
266 /*
267  * Queue for pf_overload_task() tasks.
268  */
269 struct pf_overload_entry {
270 	SLIST_ENTRY(pf_overload_entry)	next;
271 	struct pf_addr  		addr;
272 	sa_family_t			af;
273 	uint8_t				dir;
274 	struct pf_krule  		*rule;
275 };
276 
277 SLIST_HEAD(pf_overload_head, pf_overload_entry);
278 VNET_DEFINE_STATIC(struct pf_overload_head, pf_overloadqueue);
279 #define V_pf_overloadqueue	VNET(pf_overloadqueue)
280 VNET_DEFINE_STATIC(struct task, pf_overloadtask);
281 #define	V_pf_overloadtask	VNET(pf_overloadtask)
282 
283 static struct mtx_padalign pf_overloadqueue_mtx;
284 MTX_SYSINIT(pf_overloadqueue_mtx, &pf_overloadqueue_mtx,
285     "pf overload/flush queue", MTX_DEF);
286 #define	PF_OVERLOADQ_LOCK()	mtx_lock(&pf_overloadqueue_mtx)
287 #define	PF_OVERLOADQ_UNLOCK()	mtx_unlock(&pf_overloadqueue_mtx)
288 
289 VNET_DEFINE(struct pf_krulequeue, pf_unlinked_rules);
290 struct mtx_padalign pf_unlnkdrules_mtx;
291 MTX_SYSINIT(pf_unlnkdrules_mtx, &pf_unlnkdrules_mtx, "pf unlinked rules",
292     MTX_DEF);
293 
294 struct sx pf_config_lock;
295 SX_SYSINIT(pf_config_lock, &pf_config_lock, "pf config");
296 
297 struct mtx_padalign pf_table_stats_lock;
298 MTX_SYSINIT(pf_table_stats_lock, &pf_table_stats_lock, "pf table stats",
299     MTX_DEF);
300 
301 VNET_DEFINE_STATIC(uma_zone_t,	pf_sources_z);
302 #define	V_pf_sources_z	VNET(pf_sources_z)
303 uma_zone_t		pf_mtag_z;
304 VNET_DEFINE(uma_zone_t,	 pf_state_z);
305 VNET_DEFINE(uma_zone_t,	 pf_state_key_z);
306 VNET_DEFINE(uma_zone_t,	 pf_udp_mapping_z);
307 
308 VNET_DEFINE(struct unrhdr64, pf_stateid);
309 
310 static void		 pf_src_tree_remove_state(struct pf_kstate *);
311 static int		 pf_check_threshold(struct pf_kthreshold *);
312 
313 static void		 pf_change_ap(struct pf_pdesc *, struct pf_addr *, u_int16_t *,
314 			    struct pf_addr *, u_int16_t);
315 static int		 pf_modulate_sack(struct pf_pdesc *,
316 			    struct tcphdr *, struct pf_state_peer *);
317 int			 pf_icmp_mapping(struct pf_pdesc *, u_int8_t, int *,
318 			    u_int16_t *, u_int16_t *);
319 static void		 pf_change_icmp(struct pf_addr *, u_int16_t *,
320 			    struct pf_addr *, struct pf_addr *, u_int16_t,
321 			    u_int16_t *, u_int16_t *, u_int16_t *,
322 			    u_int16_t *, u_int8_t, sa_family_t);
323 int			 pf_change_icmp_af(struct mbuf *, int,
324 			    struct pf_pdesc *, struct pf_pdesc *,
325 			    struct pf_addr *, struct pf_addr *, sa_family_t,
326 			    sa_family_t);
327 int			 pf_translate_icmp_af(int, void *);
328 static void		 pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t,
329 			    int, sa_family_t, struct pf_krule *, int);
330 static void		 pf_detach_state(struct pf_kstate *);
331 static int		 pf_state_key_attach(struct pf_state_key *,
332 			    struct pf_state_key *, struct pf_kstate *);
333 static void		 pf_state_key_detach(struct pf_kstate *, int);
334 static int		 pf_state_key_ctor(void *, int, void *, int);
335 static u_int32_t	 pf_tcp_iss(struct pf_pdesc *);
336 static __inline void	 pf_dummynet_flag_remove(struct mbuf *m,
337 			    struct pf_mtag *pf_mtag);
338 static int		 pf_dummynet(struct pf_pdesc *, struct pf_kstate *,
339 			    struct pf_krule *, struct mbuf **);
340 static int		 pf_dummynet_route(struct pf_pdesc *,
341 			    struct pf_kstate *, struct pf_krule *,
342 			    struct ifnet *, const struct sockaddr *, struct mbuf **);
343 static int		 pf_test_eth_rule(int, struct pfi_kkif *,
344 			    struct mbuf **);
345 static int		 pf_test_rule(struct pf_krule **, struct pf_kstate **,
346 			    struct pf_pdesc *, struct pf_krule **,
347 			    struct pf_kruleset **, u_short *, struct inpcb *);
348 static int		 pf_create_state(struct pf_krule *,
349 			    struct pf_test_ctx *,
350 			    struct pf_kstate **, u_int16_t, u_int16_t);
351 static int		 pf_state_key_addr_setup(struct pf_pdesc *,
352 			    struct pf_state_key_cmp *, int);
353 static int		 pf_tcp_track_full(struct pf_kstate *,
354 			    struct pf_pdesc *, u_short *, int *,
355 			    struct pf_state_peer *, struct pf_state_peer *,
356 			    u_int8_t, u_int8_t);
357 static int		 pf_tcp_track_sloppy(struct pf_kstate *,
358 			    struct pf_pdesc *, u_short *,
359 			    struct pf_state_peer *, struct pf_state_peer *,
360 			    u_int8_t, u_int8_t);
361 static int		 pf_test_state(struct pf_kstate **, struct pf_pdesc *,
362 			    u_short *);
363 int			 pf_icmp_state_lookup(struct pf_state_key_cmp *,
364 			    struct pf_pdesc *, struct pf_kstate **,
365 			    u_int16_t, u_int16_t, int, int *, int, int);
366 static int		 pf_test_state_icmp(struct pf_kstate **,
367 			    struct pf_pdesc *, u_short *);
368 static int		 pf_sctp_track(struct pf_kstate *, struct pf_pdesc *,
369 			    u_short *);
370 static void		 pf_sctp_multihome_detach_addr(const struct pf_kstate *);
371 static void		 pf_sctp_multihome_delayed(struct pf_pdesc *,
372 			    struct pfi_kkif *, struct pf_kstate *, int);
373 static u_int16_t	 pf_calc_mss(struct pf_addr *, sa_family_t,
374 				int, u_int16_t);
375 static int		 pf_check_proto_cksum(struct mbuf *, int, int,
376 			    u_int8_t, sa_family_t);
377 static int		 pf_walk_option(struct pf_pdesc *, struct ip *,
378 			    int, int, u_short *);
379 static int		 pf_walk_header(struct pf_pdesc *, struct ip *, u_short *);
380 #ifdef INET6
381 static int		 pf_walk_option6(struct pf_pdesc *, struct ip6_hdr *,
382 			    int, int, u_short *);
383 static int		 pf_walk_header6(struct pf_pdesc *, struct ip6_hdr *,
384 			    u_short *);
385 #endif
386 static void		 pf_print_state_parts(struct pf_kstate *,
387 			    struct pf_state_key *, struct pf_state_key *);
388 static int		 pf_patch_8(struct pf_pdesc *, u_int8_t *, u_int8_t,
389 			    bool);
390 static int		 pf_find_state(struct pf_pdesc *,
391 			    const struct pf_state_key_cmp *, struct pf_kstate **);
392 static bool		 pf_src_connlimit(struct pf_kstate *);
393 static int		 pf_match_rcvif(struct mbuf *, struct pf_krule *);
394 static void		 pf_counters_inc(int, struct pf_pdesc *,
395 			    struct pf_kstate *, struct pf_krule *,
396 			    struct pf_krule *);
397 static void		 pf_log_matches(struct pf_pdesc *, struct pf_krule *,
398 			    struct pf_krule *, struct pf_kruleset *,
399 			    struct pf_krule_slist *);
400 static void		 pf_overload_task(void *v, int pending);
401 static u_short		 pf_insert_src_node(struct pf_ksrc_node *[PF_SN_MAX],
402 			    struct pf_srchash *[PF_SN_MAX], struct pf_krule *,
403 			    struct pf_addr *, sa_family_t, struct pf_addr *,
404 			    struct pfi_kkif *, sa_family_t, pf_sn_types_t);
405 static u_int		 pf_purge_expired_states(u_int, int);
406 static void		 pf_purge_unlinked_rules(void);
407 static int		 pf_mtag_uminit(void *, int, int);
408 static void		 pf_mtag_free(struct m_tag *);
409 static void		 pf_packet_rework_nat(struct pf_pdesc *, int,
410 			    struct pf_state_key *);
411 #ifdef INET
412 static int		 pf_route(struct pf_krule *,
413 			    struct ifnet *, struct pf_kstate *,
414 			    struct pf_pdesc *, struct inpcb *);
415 #endif /* INET */
416 #ifdef INET6
417 static void		 pf_change_a6(struct pf_addr *, u_int16_t *,
418 			    struct pf_addr *, u_int8_t);
419 static int		 pf_route6(struct pf_krule *,
420 			    struct ifnet *, struct pf_kstate *,
421 			    struct pf_pdesc *, struct inpcb *);
422 #endif /* INET6 */
423 static __inline void pf_set_protostate(struct pf_kstate *, int, u_int8_t);
424 
425 int in4_cksum(struct mbuf *m, u_int8_t nxt, int off, int len);
426 
427 extern int pf_end_threads;
428 extern struct proc *pf_purge_proc;
429 
430 VNET_DEFINE(struct pf_limit, pf_limits[PF_LIMIT_MAX]);
431 
432 #define	PACKET_UNDO_NAT(_pd, _off, _s)					\
433 	do {								\
434 		struct pf_state_key *nk;				\
435 		if ((pd->dir) == PF_OUT)				\
436 			nk = (_s)->key[PF_SK_STACK];			\
437 		else							\
438 			nk = (_s)->key[PF_SK_WIRE];			\
439 		pf_packet_rework_nat(_pd, _off, nk);		\
440 	} while (0)
441 
442 #define	PACKET_LOOPED(pd)	((pd)->pf_mtag &&			\
443 				 (pd)->pf_mtag->flags & PF_MTAG_FLAG_PACKET_LOOPED)
444 
445 static struct pfi_kkif *
446 BOUND_IFACE(struct pf_kstate *st, struct pf_pdesc *pd)
447 {
448 	struct pfi_kkif *k = pd->kif;
449 
450 	SDT_PROBE2(pf, ip, , bound_iface, st, k);
451 
452 	/* Floating unless otherwise specified. */
453 	if (! (st->rule->rule_flag & PFRULE_IFBOUND))
454 		return (V_pfi_all);
455 
456 	/*
457 	 * Initially set to all, because we don't know what interface we'll be
458 	 * sending this out when we create the state.
459 	 */
460 	if (st->rule->rt == PF_REPLYTO || (pd->af != pd->naf && st->direction == PF_IN))
461 		return (V_pfi_all);
462 
463 	/*
464 	 * If this state is created based on another state (e.g. SCTP
465 	 * multihome) always set it floating initially. We can't know for sure
466 	 * what interface the actual traffic for this state will come in on.
467 	 */
468 	if (pd->related_rule)
469 		return (V_pfi_all);
470 
471 	/* Don't overrule the interface for states created on incoming packets. */
472 	if (st->direction == PF_IN)
473 		return (k);
474 
475 	/* No route-to, so don't overrule. */
476 	if (st->act.rt != PF_ROUTETO)
477 		return (k);
478 
479 	/* Bind to the route-to interface. */
480 	return (st->act.rt_kif);
481 }
482 
483 #define	STATE_INC_COUNTERS(s)						\
484 	do {								\
485 		struct pf_krule_item *mrm;				\
486 		counter_u64_add(s->rule->states_cur, 1);		\
487 		counter_u64_add(s->rule->states_tot, 1);		\
488 		if (s->anchor != NULL) {				\
489 			counter_u64_add(s->anchor->states_cur, 1);	\
490 			counter_u64_add(s->anchor->states_tot, 1);	\
491 		}							\
492 		if (s->nat_rule != NULL) {				\
493 			counter_u64_add(s->nat_rule->states_cur, 1);\
494 			counter_u64_add(s->nat_rule->states_tot, 1);\
495 		}							\
496 		SLIST_FOREACH(mrm, &s->match_rules, entry) {		\
497 			counter_u64_add(mrm->r->states_cur, 1);		\
498 			counter_u64_add(mrm->r->states_tot, 1);		\
499 		}							\
500 	} while (0)
501 
502 #define	STATE_DEC_COUNTERS(s)						\
503 	do {								\
504 		struct pf_krule_item *mrm;				\
505 		if (s->nat_rule != NULL)				\
506 			counter_u64_add(s->nat_rule->states_cur, -1);\
507 		if (s->anchor != NULL)				\
508 			counter_u64_add(s->anchor->states_cur, -1);	\
509 		counter_u64_add(s->rule->states_cur, -1);		\
510 		SLIST_FOREACH(mrm, &s->match_rules, entry)		\
511 			counter_u64_add(mrm->r->states_cur, -1);	\
512 	} while (0)
513 
514 MALLOC_DEFINE(M_PFHASH, "pf_hash", "pf(4) hash header structures");
515 MALLOC_DEFINE(M_PF_RULE_ITEM, "pf_krule_item", "pf(4) rule items");
516 VNET_DEFINE(struct pf_keyhash *, pf_keyhash);
517 VNET_DEFINE(struct pf_idhash *, pf_idhash);
518 VNET_DEFINE(struct pf_srchash *, pf_srchash);
519 VNET_DEFINE(struct pf_udpendpointhash *, pf_udpendpointhash);
520 VNET_DEFINE(struct pf_udpendpointmapping *, pf_udpendpointmapping);
521 
522 SYSCTL_NODE(_net, OID_AUTO, pf, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
523     "pf(4)");
524 
525 VNET_DEFINE(u_long, pf_hashmask);
526 VNET_DEFINE(u_long, pf_srchashmask);
527 VNET_DEFINE(u_long, pf_udpendpointhashmask);
528 VNET_DEFINE_STATIC(u_long, pf_hashsize);
529 #define V_pf_hashsize	VNET(pf_hashsize)
530 VNET_DEFINE_STATIC(u_long, pf_srchashsize);
531 #define V_pf_srchashsize	VNET(pf_srchashsize)
532 VNET_DEFINE_STATIC(u_long, pf_udpendpointhashsize);
533 #define V_pf_udpendpointhashsize	VNET(pf_udpendpointhashsize)
534 u_long	pf_ioctl_maxcount = 65535;
535 
536 SYSCTL_ULONG(_net_pf, OID_AUTO, states_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
537     &VNET_NAME(pf_hashsize), 0, "Size of pf(4) states hashtable");
538 SYSCTL_ULONG(_net_pf, OID_AUTO, source_nodes_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
539     &VNET_NAME(pf_srchashsize), 0, "Size of pf(4) source nodes hashtable");
540 SYSCTL_ULONG(_net_pf, OID_AUTO, udpendpoint_hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
541     &VNET_NAME(pf_udpendpointhashsize), 0, "Size of pf(4) endpoint hashtable");
542 SYSCTL_ULONG(_net_pf, OID_AUTO, request_maxcount, CTLFLAG_RWTUN,
543     &pf_ioctl_maxcount, 0, "Maximum number of tables, addresses, ... in a single ioctl() call");
544 
545 VNET_DEFINE(void *, pf_swi_cookie);
546 VNET_DEFINE(struct intr_event *, pf_swi_ie);
547 
548 VNET_DEFINE(uint32_t, pf_hashseed);
549 #define	V_pf_hashseed	VNET(pf_hashseed)
550 
551 static void
552 pf_sctp_checksum(struct mbuf *m, int off)
553 {
554 	uint32_t sum = 0;
555 
556 	/* Zero out the checksum, to enable recalculation. */
557 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
558 	    sizeof(sum), (caddr_t)&sum);
559 
560 	sum = sctp_calculate_cksum(m, off);
561 
562 	m_copyback(m, off + offsetof(struct sctphdr, checksum),
563 	    sizeof(sum), (caddr_t)&sum);
564 }
565 
566 int
567 pf_addr_cmp(struct pf_addr *a, struct pf_addr *b, sa_family_t af)
568 {
569 
570 	switch (af) {
571 #ifdef INET
572 	case AF_INET:
573 		if (a->addr32[0] > b->addr32[0])
574 			return (1);
575 		if (a->addr32[0] < b->addr32[0])
576 			return (-1);
577 		break;
578 #endif /* INET */
579 #ifdef INET6
580 	case AF_INET6:
581 		if (a->addr32[3] > b->addr32[3])
582 			return (1);
583 		if (a->addr32[3] < b->addr32[3])
584 			return (-1);
585 		if (a->addr32[2] > b->addr32[2])
586 			return (1);
587 		if (a->addr32[2] < b->addr32[2])
588 			return (-1);
589 		if (a->addr32[1] > b->addr32[1])
590 			return (1);
591 		if (a->addr32[1] < b->addr32[1])
592 			return (-1);
593 		if (a->addr32[0] > b->addr32[0])
594 			return (1);
595 		if (a->addr32[0] < b->addr32[0])
596 			return (-1);
597 		break;
598 #endif /* INET6 */
599 	default:
600 		unhandled_af(af);
601 	}
602 	return (0);
603 }
604 
605 static bool
606 pf_is_loopback(sa_family_t af, struct pf_addr *addr)
607 {
608 	switch (af) {
609 #ifdef INET
610 	case AF_INET:
611 		return IN_LOOPBACK(ntohl(addr->v4.s_addr));
612 #endif /* INET */
613 	case AF_INET6:
614 		return IN6_IS_ADDR_LOOPBACK(&addr->v6);
615 	default:
616 		unhandled_af(af);
617 	}
618 }
619 
620 static void
621 pf_packet_rework_nat(struct pf_pdesc *pd, int off, struct pf_state_key *nk)
622 {
623 
624 	switch (pd->virtual_proto) {
625 	case IPPROTO_TCP: {
626 		struct tcphdr *th = &pd->hdr.tcp;
627 
628 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
629 			pf_change_ap(pd, pd->src, &th->th_sport,
630 			    &nk->addr[pd->sidx], nk->port[pd->sidx]);
631 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
632 			pf_change_ap(pd, pd->dst, &th->th_dport,
633 			    &nk->addr[pd->didx], nk->port[pd->didx]);
634 		m_copyback(pd->m, off, sizeof(*th), (caddr_t)th);
635 		break;
636 	}
637 	case IPPROTO_UDP: {
638 		struct udphdr *uh = &pd->hdr.udp;
639 
640 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af))
641 			pf_change_ap(pd, pd->src, &uh->uh_sport,
642 			    &nk->addr[pd->sidx], nk->port[pd->sidx]);
643 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af))
644 			pf_change_ap(pd, pd->dst, &uh->uh_dport,
645 			    &nk->addr[pd->didx], nk->port[pd->didx]);
646 		m_copyback(pd->m, off, sizeof(*uh), (caddr_t)uh);
647 		break;
648 	}
649 	case IPPROTO_SCTP: {
650 		struct sctphdr *sh = &pd->hdr.sctp;
651 
652 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
653 			pf_change_ap(pd, pd->src, &sh->src_port,
654 			    &nk->addr[pd->sidx], nk->port[pd->sidx]);
655 		}
656 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
657 			pf_change_ap(pd, pd->dst, &sh->dest_port,
658 			    &nk->addr[pd->didx], nk->port[pd->didx]);
659 		}
660 
661 		break;
662 	}
663 	case IPPROTO_ICMP: {
664 		struct icmp *ih = &pd->hdr.icmp;
665 
666 		if (nk->port[pd->sidx] != ih->icmp_id) {
667 			pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
668 			    ih->icmp_cksum, ih->icmp_id,
669 			    nk->port[pd->sidx], 0);
670 			ih->icmp_id = nk->port[pd->sidx];
671 			pd->sport = &ih->icmp_id;
672 
673 			m_copyback(pd->m, off, ICMP_MINLEN, (caddr_t)ih);
674 		}
675 		/* FALLTHROUGH */
676 	}
677 	default:
678 		if (PF_ANEQ(pd->src, &nk->addr[pd->sidx], pd->af)) {
679 			switch (pd->af) {
680 			case AF_INET:
681 				pf_change_a(&pd->src->v4.s_addr,
682 				    pd->ip_sum, nk->addr[pd->sidx].v4.s_addr,
683 				    0);
684 				break;
685 			case AF_INET6:
686 				pf_addrcpy(pd->src, &nk->addr[pd->sidx],
687 				    pd->af);
688 				break;
689 			default:
690 				unhandled_af(pd->af);
691 			}
692 		}
693 		if (PF_ANEQ(pd->dst, &nk->addr[pd->didx], pd->af)) {
694 			switch (pd->af) {
695 			case AF_INET:
696 				pf_change_a(&pd->dst->v4.s_addr,
697 				    pd->ip_sum, nk->addr[pd->didx].v4.s_addr,
698 				    0);
699 				break;
700 			case AF_INET6:
701 				pf_addrcpy(pd->dst, &nk->addr[pd->didx],
702 				    pd->af);
703 				break;
704 			default:
705 				unhandled_af(pd->af);
706 			}
707 		}
708 		break;
709 	}
710 }
711 
712 static __inline uint32_t
713 pf_hashkey(const struct pf_state_key *sk)
714 {
715 	uint32_t h;
716 
717 	h = murmur3_32_hash32((const uint32_t *)sk,
718 	    sizeof(struct pf_state_key_cmp)/sizeof(uint32_t),
719 	    V_pf_hashseed);
720 
721 	return (h & V_pf_hashmask);
722 }
723 
724 __inline uint32_t
725 pf_hashsrc(struct pf_addr *addr, sa_family_t af)
726 {
727 	uint32_t h;
728 
729 	switch (af) {
730 	case AF_INET:
731 		h = murmur3_32_hash32((uint32_t *)&addr->v4,
732 		    sizeof(addr->v4)/sizeof(uint32_t), V_pf_hashseed);
733 		break;
734 	case AF_INET6:
735 		h = murmur3_32_hash32((uint32_t *)&addr->v6,
736 		    sizeof(addr->v6)/sizeof(uint32_t), V_pf_hashseed);
737 		break;
738 	default:
739 		unhandled_af(af);
740 	}
741 
742 	return (h & V_pf_srchashmask);
743 }
744 
745 static inline uint32_t
746 pf_hashudpendpoint(struct pf_udp_endpoint *endpoint)
747 {
748 	uint32_t h;
749 
750 	h = murmur3_32_hash32((uint32_t *)endpoint,
751 	    sizeof(struct pf_udp_endpoint_cmp)/sizeof(uint32_t),
752 	    V_pf_hashseed);
753 	return (h & V_pf_udpendpointhashmask);
754 }
755 
756 #ifdef ALTQ
757 static int
758 pf_state_hash(struct pf_kstate *s)
759 {
760 	u_int32_t hv = (intptr_t)s / sizeof(*s);
761 
762 	hv ^= crc32(&s->src, sizeof(s->src));
763 	hv ^= crc32(&s->dst, sizeof(s->dst));
764 	if (hv == 0)
765 		hv = 1;
766 	return (hv);
767 }
768 #endif /* ALTQ */
769 
770 static __inline void
771 pf_set_protostate(struct pf_kstate *s, int which, u_int8_t newstate)
772 {
773 	if (which == PF_PEER_DST || which == PF_PEER_BOTH)
774 		s->dst.state = newstate;
775 	if (which == PF_PEER_DST)
776 		return;
777 	if (s->src.state == newstate)
778 		return;
779 	if (s->creatorid == V_pf_status.hostid &&
780 	    s->key[PF_SK_STACK] != NULL &&
781 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP &&
782 	    !(TCPS_HAVEESTABLISHED(s->src.state) ||
783 	    s->src.state == TCPS_CLOSED) &&
784 	    (TCPS_HAVEESTABLISHED(newstate) || newstate == TCPS_CLOSED))
785 		atomic_add_32(&V_pf_status.states_halfopen, -1);
786 
787 	s->src.state = newstate;
788 }
789 
790 bool
791 pf_init_threshold(struct pf_kthreshold *threshold,
792     u_int32_t limit, u_int32_t seconds)
793 {
794 	threshold->limit = limit;
795 	threshold->seconds = seconds;
796 	threshold->cr = counter_rate_alloc(M_NOWAIT, seconds);
797 
798 	return (threshold->cr != NULL);
799 }
800 
801 static int
802 pf_check_threshold(struct pf_kthreshold *threshold)
803 {
804 	return (counter_ratecheck(threshold->cr, threshold->limit) < 0);
805 }
806 
807 static bool
808 pf_src_connlimit(struct pf_kstate *state)
809 {
810 	struct pf_overload_entry	*pfoe;
811 	struct pf_ksrc_node		*src_node = state->sns[PF_SN_LIMIT];
812 	bool				 limited = false;
813 
814 	PF_STATE_LOCK_ASSERT(state);
815 	PF_SRC_NODE_LOCK(src_node);
816 
817 	src_node->conn++;
818 	state->src.tcp_est = 1;
819 
820 	if (state->rule->max_src_conn &&
821 	    state->rule->max_src_conn <
822 	    src_node->conn) {
823 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONN], 1);
824 		limited = true;
825 	}
826 
827 	if (state->rule->max_src_conn_rate.limit &&
828 	    pf_check_threshold(&src_node->conn_rate)) {
829 		counter_u64_add(V_pf_status.lcounters[LCNT_SRCCONNRATE], 1);
830 		limited = true;
831 	}
832 
833 	if (!limited)
834 		goto done;
835 
836 	/* Kill this state. */
837 	state->timeout = PFTM_PURGE;
838 	pf_set_protostate(state, PF_PEER_BOTH, TCPS_CLOSED);
839 
840 	if (state->rule->overload_tbl == NULL)
841 		goto done;
842 
843 	/* Schedule overloading and flushing task. */
844 	pfoe = malloc(sizeof(*pfoe), M_PFTEMP, M_NOWAIT);
845 	if (pfoe == NULL)
846 		goto done;  /* too bad :( */
847 
848 	bcopy(&src_node->addr, &pfoe->addr, sizeof(pfoe->addr));
849 	pfoe->af = state->key[PF_SK_WIRE]->af;
850 	pfoe->rule = state->rule;
851 	pfoe->dir = state->direction;
852 	PF_OVERLOADQ_LOCK();
853 	SLIST_INSERT_HEAD(&V_pf_overloadqueue, pfoe, next);
854 	PF_OVERLOADQ_UNLOCK();
855 	taskqueue_enqueue(taskqueue_swi, &V_pf_overloadtask);
856 
857 done:
858 	PF_SRC_NODE_UNLOCK(src_node);
859 	return (limited);
860 }
861 
862 static void
863 pf_overload_task(void *v, int pending)
864 {
865 	struct pf_overload_head queue;
866 	struct pfr_addr p;
867 	struct pf_overload_entry *pfoe, *pfoe1;
868 	uint32_t killed = 0;
869 
870 	CURVNET_SET((struct vnet *)v);
871 
872 	PF_OVERLOADQ_LOCK();
873 	queue = V_pf_overloadqueue;
874 	SLIST_INIT(&V_pf_overloadqueue);
875 	PF_OVERLOADQ_UNLOCK();
876 
877 	bzero(&p, sizeof(p));
878 	SLIST_FOREACH(pfoe, &queue, next) {
879 		counter_u64_add(V_pf_status.lcounters[LCNT_OVERLOAD_TABLE], 1);
880 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
881 			printf("%s: blocking address ", __func__);
882 			pf_print_host(&pfoe->addr, 0, pfoe->af);
883 			printf("\n");
884 		}
885 
886 		p.pfra_af = pfoe->af;
887 		switch (pfoe->af) {
888 #ifdef INET
889 		case AF_INET:
890 			p.pfra_net = 32;
891 			p.pfra_ip4addr = pfoe->addr.v4;
892 			break;
893 #endif /* INET */
894 #ifdef INET6
895 		case AF_INET6:
896 			p.pfra_net = 128;
897 			p.pfra_ip6addr = pfoe->addr.v6;
898 			break;
899 #endif /* INET6 */
900 		default:
901 			unhandled_af(pfoe->af);
902 		}
903 
904 		PF_RULES_WLOCK();
905 		pfr_insert_kentry(pfoe->rule->overload_tbl, &p, time_second);
906 		PF_RULES_WUNLOCK();
907 	}
908 
909 	/*
910 	 * Remove those entries, that don't need flushing.
911 	 */
912 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
913 		if (pfoe->rule->flush == 0) {
914 			SLIST_REMOVE(&queue, pfoe, pf_overload_entry, next);
915 			free(pfoe, M_PFTEMP);
916 		} else
917 			counter_u64_add(
918 			    V_pf_status.lcounters[LCNT_OVERLOAD_FLUSH], 1);
919 
920 	/* If nothing to flush, return. */
921 	if (SLIST_EMPTY(&queue)) {
922 		CURVNET_RESTORE();
923 		return;
924 	}
925 
926 	for (int i = 0; i <= V_pf_hashmask; i++) {
927 		struct pf_idhash *ih = &V_pf_idhash[i];
928 		struct pf_state_key *sk;
929 		struct pf_kstate *s;
930 
931 		PF_HASHROW_LOCK(ih);
932 		LIST_FOREACH(s, &ih->states, entry) {
933 		    sk = s->key[PF_SK_WIRE];
934 		    SLIST_FOREACH(pfoe, &queue, next)
935 			if (sk->af == pfoe->af &&
936 			    ((pfoe->rule->flush & PF_FLUSH_GLOBAL) ||
937 			    pfoe->rule == s->rule) &&
938 			    ((pfoe->dir == PF_OUT &&
939 			    PF_AEQ(&pfoe->addr, &sk->addr[1], sk->af)) ||
940 			    (pfoe->dir == PF_IN &&
941 			    PF_AEQ(&pfoe->addr, &sk->addr[0], sk->af)))) {
942 				s->timeout = PFTM_PURGE;
943 				pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
944 				killed++;
945 			}
946 		}
947 		PF_HASHROW_UNLOCK(ih);
948 	}
949 	SLIST_FOREACH_SAFE(pfoe, &queue, next, pfoe1)
950 		free(pfoe, M_PFTEMP);
951 	if (V_pf_status.debug >= PF_DEBUG_MISC)
952 		printf("%s: %u states killed", __func__, killed);
953 
954 	CURVNET_RESTORE();
955 }
956 
957 /*
958  * On node found always returns locked. On not found its configurable.
959  */
960 struct pf_ksrc_node *
961 pf_find_src_node(struct pf_addr *src, struct pf_krule *rule, sa_family_t af,
962     struct pf_srchash **sh, pf_sn_types_t sn_type, bool returnlocked)
963 {
964 	struct pf_ksrc_node *n;
965 
966 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_SEARCH], 1);
967 
968 	*sh = &V_pf_srchash[pf_hashsrc(src, af)];
969 	PF_HASHROW_LOCK(*sh);
970 	LIST_FOREACH(n, &(*sh)->nodes, entry)
971 		if (n->rule == rule && n->af == af && n->type == sn_type &&
972 		    ((af == AF_INET && n->addr.v4.s_addr == src->v4.s_addr) ||
973 		    (af == AF_INET6 && bcmp(&n->addr, src, sizeof(*src)) == 0)))
974 			break;
975 
976 	if (n == NULL && !returnlocked)
977 		PF_HASHROW_UNLOCK(*sh);
978 
979 	return (n);
980 }
981 
982 bool
983 pf_src_node_exists(struct pf_ksrc_node **sn, struct pf_srchash *sh)
984 {
985 	struct pf_ksrc_node	*cur;
986 
987 	if ((*sn) == NULL)
988 		return (false);
989 
990 	KASSERT(sh != NULL, ("%s: sh is NULL", __func__));
991 
992 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_SEARCH], 1);
993 	PF_HASHROW_LOCK(sh);
994 	LIST_FOREACH(cur, &(sh->nodes), entry) {
995 		if (cur == (*sn) &&
996 		    cur->expire != 1) /* Ignore nodes being killed */
997 			return (true);
998 	}
999 	PF_HASHROW_UNLOCK(sh);
1000 	(*sn) = NULL;
1001 	return (false);
1002 }
1003 
1004 static void
1005 pf_free_src_node(struct pf_ksrc_node *sn)
1006 {
1007 
1008 	for (int i = 0; i < 2; i++) {
1009 		counter_u64_free(sn->bytes[i]);
1010 		counter_u64_free(sn->packets[i]);
1011 	}
1012 	counter_rate_free(sn->conn_rate.cr);
1013 	uma_zfree(V_pf_sources_z, sn);
1014 }
1015 
1016 static u_short
1017 pf_insert_src_node(struct pf_ksrc_node *sns[PF_SN_MAX],
1018     struct pf_srchash *snhs[PF_SN_MAX], struct pf_krule *rule,
1019     struct pf_addr *src, sa_family_t af, struct pf_addr *raddr,
1020     struct pfi_kkif *rkif, sa_family_t raf, pf_sn_types_t sn_type)
1021 {
1022 	u_short			 reason = 0;
1023 	struct pf_krule		*r_track = rule;
1024 	struct pf_ksrc_node	**sn = &(sns[sn_type]);
1025 	struct pf_srchash	**sh = &(snhs[sn_type]);
1026 
1027 	KASSERT(sn_type != PF_SN_LIMIT || (raddr == NULL && rkif == NULL),
1028 	    ("%s: raddr and rkif must be NULL for PF_SN_LIMIT", __func__));
1029 
1030 	KASSERT(sn_type != PF_SN_LIMIT || (rule->rule_flag & PFRULE_SRCTRACK),
1031 	    ("%s: PF_SN_LIMIT only valid for rules with PFRULE_SRCTRACK", __func__));
1032 
1033 	/*
1034 	 * XXX: There could be a KASSERT for
1035 	 * sn_type == PF_SN_LIMIT || (pool->opts & PF_POOL_STICKYADDR)
1036 	 * but we'd need to pass pool *only* for this KASSERT.
1037 	 */
1038 
1039 	if ( (rule->rule_flag & PFRULE_SRCTRACK) &&
1040 	    !(rule->rule_flag & PFRULE_RULESRCTRACK))
1041 		r_track = &V_pf_default_rule;
1042 
1043 	/*
1044 	 * Request the sh to always be locked, as we might insert a new sn.
1045 	 */
1046 	if (*sn == NULL)
1047 		*sn = pf_find_src_node(src, r_track, af, sh, sn_type, true);
1048 
1049 	if (*sn == NULL) {
1050 		PF_HASHROW_ASSERT(*sh);
1051 
1052 		if (sn_type == PF_SN_LIMIT && rule->max_src_nodes &&
1053 		    counter_u64_fetch(r_track->src_nodes[sn_type]) >= rule->max_src_nodes) {
1054 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCNODES], 1);
1055 			reason = PFRES_SRCLIMIT;
1056 			goto done;
1057 		}
1058 
1059 		(*sn) = uma_zalloc(V_pf_sources_z, M_NOWAIT | M_ZERO);
1060 		if ((*sn) == NULL) {
1061 			reason = PFRES_MEMORY;
1062 			goto done;
1063 		}
1064 
1065 		for (int i = 0; i < 2; i++) {
1066 			(*sn)->bytes[i] = counter_u64_alloc(M_NOWAIT);
1067 			(*sn)->packets[i] = counter_u64_alloc(M_NOWAIT);
1068 
1069 			if ((*sn)->bytes[i] == NULL || (*sn)->packets[i] == NULL) {
1070 				pf_free_src_node(*sn);
1071 				reason = PFRES_MEMORY;
1072 				goto done;
1073 			}
1074 		}
1075 
1076 		if (sn_type == PF_SN_LIMIT)
1077 			if (! pf_init_threshold(&(*sn)->conn_rate,
1078 			    rule->max_src_conn_rate.limit,
1079 			    rule->max_src_conn_rate.seconds)) {
1080 				pf_free_src_node(*sn);
1081 				reason = PFRES_MEMORY;
1082 				goto done;
1083 			}
1084 
1085 		MPASS((*sn)->lock == NULL);
1086 		(*sn)->lock = &(*sh)->lock;
1087 
1088 		(*sn)->af = af;
1089 		(*sn)->rule = r_track;
1090 		pf_addrcpy(&(*sn)->addr, src, af);
1091 		if (raddr != NULL)
1092 			pf_addrcpy(&(*sn)->raddr, raddr, raf);
1093 		(*sn)->rkif = rkif;
1094 		(*sn)->raf = raf;
1095 		LIST_INSERT_HEAD(&(*sh)->nodes, *sn, entry);
1096 		(*sn)->creation = time_uptime;
1097 		(*sn)->ruletype = rule->action;
1098 		(*sn)->type = sn_type;
1099 		counter_u64_add(r_track->src_nodes[sn_type], 1);
1100 		counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_INSERT], 1);
1101 	} else {
1102 		if (sn_type == PF_SN_LIMIT && rule->max_src_states &&
1103 		    (*sn)->states >= rule->max_src_states) {
1104 			counter_u64_add(V_pf_status.lcounters[LCNT_SRCSTATES],
1105 			    1);
1106 			reason = PFRES_SRCLIMIT;
1107 			goto done;
1108 		}
1109 	}
1110 done:
1111 	if (reason == 0)
1112 		(*sn)->states++;
1113 	else
1114 		(*sn) = NULL;
1115 
1116 	PF_HASHROW_UNLOCK(*sh);
1117 	return (reason);
1118 }
1119 
1120 void
1121 pf_unlink_src_node(struct pf_ksrc_node *src)
1122 {
1123 	PF_SRC_NODE_LOCK_ASSERT(src);
1124 
1125 	LIST_REMOVE(src, entry);
1126 	if (src->rule)
1127 		counter_u64_add(src->rule->src_nodes[src->type], -1);
1128 }
1129 
1130 u_int
1131 pf_free_src_nodes(struct pf_ksrc_node_list *head)
1132 {
1133 	struct pf_ksrc_node *sn, *tmp;
1134 	u_int count = 0;
1135 
1136 	LIST_FOREACH_SAFE(sn, head, entry, tmp) {
1137 		pf_free_src_node(sn);
1138 		count++;
1139 	}
1140 
1141 	counter_u64_add(V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], count);
1142 
1143 	return (count);
1144 }
1145 
1146 void
1147 pf_mtag_initialize(void)
1148 {
1149 
1150 	pf_mtag_z = uma_zcreate("pf mtags", sizeof(struct m_tag) +
1151 	    sizeof(struct pf_mtag), NULL, NULL, pf_mtag_uminit, NULL,
1152 	    UMA_ALIGN_PTR, 0);
1153 }
1154 
1155 /* Per-vnet data storage structures initialization. */
1156 void
1157 pf_initialize(void)
1158 {
1159 	struct pf_keyhash	*kh;
1160 	struct pf_idhash	*ih;
1161 	struct pf_srchash	*sh;
1162 	struct pf_udpendpointhash	*uh;
1163 	u_int i;
1164 
1165 	if (V_pf_hashsize == 0 || !powerof2(V_pf_hashsize))
1166 		V_pf_hashsize = PF_HASHSIZ;
1167 	if (V_pf_srchashsize == 0 || !powerof2(V_pf_srchashsize))
1168 		V_pf_srchashsize = PF_SRCHASHSIZ;
1169 	if (V_pf_udpendpointhashsize == 0 || !powerof2(V_pf_udpendpointhashsize))
1170 		V_pf_udpendpointhashsize = PF_UDPENDHASHSIZ;
1171 
1172 	V_pf_hashseed = arc4random();
1173 
1174 	/* States and state keys storage. */
1175 	V_pf_state_z = uma_zcreate("pf states", sizeof(struct pf_kstate),
1176 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1177 	V_pf_limits[PF_LIMIT_STATES].zone = V_pf_state_z;
1178 	uma_zone_set_max(V_pf_state_z, PFSTATE_HIWAT);
1179 	uma_zone_set_warning(V_pf_state_z, "PF states limit reached");
1180 
1181 	V_pf_state_key_z = uma_zcreate("pf state keys",
1182 	    sizeof(struct pf_state_key), pf_state_key_ctor, NULL, NULL, NULL,
1183 	    UMA_ALIGN_PTR, 0);
1184 
1185 	V_pf_keyhash = mallocarray(V_pf_hashsize, sizeof(struct pf_keyhash),
1186 	    M_PFHASH, M_NOWAIT | M_ZERO);
1187 	V_pf_idhash = mallocarray(V_pf_hashsize, sizeof(struct pf_idhash),
1188 	    M_PFHASH, M_NOWAIT | M_ZERO);
1189 	if (V_pf_keyhash == NULL || V_pf_idhash == NULL) {
1190 		printf("pf: Unable to allocate memory for "
1191 		    "state_hashsize %lu.\n", V_pf_hashsize);
1192 
1193 		free(V_pf_keyhash, M_PFHASH);
1194 		free(V_pf_idhash, M_PFHASH);
1195 
1196 		V_pf_hashsize = PF_HASHSIZ;
1197 		V_pf_keyhash = mallocarray(V_pf_hashsize,
1198 		    sizeof(struct pf_keyhash), M_PFHASH, M_WAITOK | M_ZERO);
1199 		V_pf_idhash = mallocarray(V_pf_hashsize,
1200 		    sizeof(struct pf_idhash), M_PFHASH, M_WAITOK | M_ZERO);
1201 	}
1202 
1203 	V_pf_hashmask = V_pf_hashsize - 1;
1204 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash; i <= V_pf_hashmask;
1205 	    i++, kh++, ih++) {
1206 		mtx_init(&kh->lock, "pf_keyhash", NULL, MTX_DEF | MTX_DUPOK);
1207 		mtx_init(&ih->lock, "pf_idhash", NULL, MTX_DEF);
1208 	}
1209 
1210 	/* Source nodes. */
1211 	V_pf_sources_z = uma_zcreate("pf source nodes",
1212 	    sizeof(struct pf_ksrc_node), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
1213 	    0);
1214 	V_pf_limits[PF_LIMIT_SRC_NODES].zone = V_pf_sources_z;
1215 	uma_zone_set_max(V_pf_sources_z, PFSNODE_HIWAT);
1216 	uma_zone_set_warning(V_pf_sources_z, "PF source nodes limit reached");
1217 
1218 	V_pf_srchash = mallocarray(V_pf_srchashsize,
1219 	    sizeof(struct pf_srchash), M_PFHASH, M_NOWAIT | M_ZERO);
1220 	if (V_pf_srchash == NULL) {
1221 		printf("pf: Unable to allocate memory for "
1222 		    "source_hashsize %lu.\n", V_pf_srchashsize);
1223 
1224 		V_pf_srchashsize = PF_SRCHASHSIZ;
1225 		V_pf_srchash = mallocarray(V_pf_srchashsize,
1226 		    sizeof(struct pf_srchash), M_PFHASH, M_WAITOK | M_ZERO);
1227 	}
1228 
1229 	V_pf_srchashmask = V_pf_srchashsize - 1;
1230 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++)
1231 		mtx_init(&sh->lock, "pf_srchash", NULL, MTX_DEF);
1232 
1233 
1234 	/* UDP endpoint mappings. */
1235 	V_pf_udp_mapping_z = uma_zcreate("pf UDP mappings",
1236 	    sizeof(struct pf_udp_mapping), NULL, NULL, NULL, NULL,
1237 	    UMA_ALIGN_PTR, 0);
1238 	V_pf_udpendpointhash = mallocarray(V_pf_udpendpointhashsize,
1239 	    sizeof(struct pf_udpendpointhash), M_PFHASH, M_NOWAIT | M_ZERO);
1240 	if (V_pf_udpendpointhash == NULL) {
1241 		printf("pf: Unable to allocate memory for "
1242 		    "udpendpoint_hashsize %lu.\n", V_pf_udpendpointhashsize);
1243 
1244 		V_pf_udpendpointhashsize = PF_UDPENDHASHSIZ;
1245 		V_pf_udpendpointhash = mallocarray(V_pf_udpendpointhashsize,
1246 		    sizeof(struct pf_udpendpointhash), M_PFHASH, M_WAITOK | M_ZERO);
1247 	}
1248 
1249 	V_pf_udpendpointhashmask = V_pf_udpendpointhashsize - 1;
1250 	for (i = 0, uh = V_pf_udpendpointhash;
1251 	    i <= V_pf_udpendpointhashmask;
1252 	    i++, uh++) {
1253 		mtx_init(&uh->lock, "pf_udpendpointhash", NULL,
1254 		    MTX_DEF | MTX_DUPOK);
1255 	}
1256 
1257 	/* Anchors */
1258 	V_pf_anchor_z = uma_zcreate("pf anchors",
1259 	    sizeof(struct pf_kanchor), NULL, NULL, NULL, NULL,
1260 	    UMA_ALIGN_PTR, 0);
1261 	V_pf_limits[PF_LIMIT_ANCHORS].zone = V_pf_anchor_z;
1262 	uma_zone_set_max(V_pf_anchor_z, PF_ANCHOR_HIWAT);
1263 	uma_zone_set_warning(V_pf_anchor_z, "PF anchor limit reached");
1264 
1265 	V_pf_eth_anchor_z = uma_zcreate("pf Ethernet anchors",
1266 	    sizeof(struct pf_keth_anchor), NULL, NULL, NULL, NULL,
1267 	    UMA_ALIGN_PTR, 0);
1268 	V_pf_limits[PF_LIMIT_ETH_ANCHORS].zone = V_pf_eth_anchor_z;
1269 	uma_zone_set_max(V_pf_eth_anchor_z, PF_ANCHOR_HIWAT);
1270 	uma_zone_set_warning(V_pf_eth_anchor_z, "PF Ethernet anchor limit reached");
1271 
1272 	/* ALTQ */
1273 	TAILQ_INIT(&V_pf_altqs[0]);
1274 	TAILQ_INIT(&V_pf_altqs[1]);
1275 	TAILQ_INIT(&V_pf_altqs[2]);
1276 	TAILQ_INIT(&V_pf_altqs[3]);
1277 	TAILQ_INIT(&V_pf_pabuf[0]);
1278 	TAILQ_INIT(&V_pf_pabuf[1]);
1279 	TAILQ_INIT(&V_pf_pabuf[2]);
1280 	V_pf_altqs_active = &V_pf_altqs[0];
1281 	V_pf_altq_ifs_active = &V_pf_altqs[1];
1282 	V_pf_altqs_inactive = &V_pf_altqs[2];
1283 	V_pf_altq_ifs_inactive = &V_pf_altqs[3];
1284 
1285 	/* Send & overload+flush queues. */
1286 	STAILQ_INIT(&V_pf_sendqueue);
1287 	SLIST_INIT(&V_pf_overloadqueue);
1288 	TASK_INIT(&V_pf_overloadtask, 0, pf_overload_task, curvnet);
1289 
1290 	/* Unlinked, but may be referenced rules. */
1291 	TAILQ_INIT(&V_pf_unlinked_rules);
1292 }
1293 
1294 void
1295 pf_mtag_cleanup(void)
1296 {
1297 
1298 	uma_zdestroy(pf_mtag_z);
1299 }
1300 
1301 void
1302 pf_cleanup(void)
1303 {
1304 	struct pf_keyhash	*kh;
1305 	struct pf_idhash	*ih;
1306 	struct pf_srchash	*sh;
1307 	struct pf_udpendpointhash	*uh;
1308 	struct pf_send_entry	*pfse, *next;
1309 	u_int i;
1310 
1311 	for (i = 0, kh = V_pf_keyhash, ih = V_pf_idhash;
1312 	    i <= V_pf_hashmask;
1313 	    i++, kh++, ih++) {
1314 		KASSERT(LIST_EMPTY(&kh->keys), ("%s: key hash not empty",
1315 		    __func__));
1316 		KASSERT(LIST_EMPTY(&ih->states), ("%s: id hash not empty",
1317 		    __func__));
1318 		mtx_destroy(&kh->lock);
1319 		mtx_destroy(&ih->lock);
1320 	}
1321 	free(V_pf_keyhash, M_PFHASH);
1322 	free(V_pf_idhash, M_PFHASH);
1323 
1324 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
1325 		KASSERT(LIST_EMPTY(&sh->nodes),
1326 		    ("%s: source node hash not empty", __func__));
1327 		mtx_destroy(&sh->lock);
1328 	}
1329 	free(V_pf_srchash, M_PFHASH);
1330 
1331 	for (i = 0, uh = V_pf_udpendpointhash;
1332 	    i <= V_pf_udpendpointhashmask;
1333 	    i++, uh++) {
1334 		KASSERT(LIST_EMPTY(&uh->endpoints),
1335 		    ("%s: udp endpoint hash not empty", __func__));
1336 		mtx_destroy(&uh->lock);
1337 	}
1338 	free(V_pf_udpendpointhash, M_PFHASH);
1339 
1340 	STAILQ_FOREACH_SAFE(pfse, &V_pf_sendqueue, pfse_next, next) {
1341 		m_freem(pfse->pfse_m);
1342 		free(pfse, M_PFTEMP);
1343 	}
1344 	MPASS(RB_EMPTY(&V_pf_sctp_endpoints));
1345 
1346 	uma_zdestroy(V_pf_sources_z);
1347 	uma_zdestroy(V_pf_state_z);
1348 	uma_zdestroy(V_pf_state_key_z);
1349 	uma_zdestroy(V_pf_udp_mapping_z);
1350 	uma_zdestroy(V_pf_anchor_z);
1351 	uma_zdestroy(V_pf_eth_anchor_z);
1352 }
1353 
1354 static int
1355 pf_mtag_uminit(void *mem, int size, int how)
1356 {
1357 	struct m_tag *t;
1358 
1359 	t = (struct m_tag *)mem;
1360 	t->m_tag_cookie = MTAG_ABI_COMPAT;
1361 	t->m_tag_id = PACKET_TAG_PF;
1362 	t->m_tag_len = sizeof(struct pf_mtag);
1363 	t->m_tag_free = pf_mtag_free;
1364 
1365 	return (0);
1366 }
1367 
1368 static void
1369 pf_mtag_free(struct m_tag *t)
1370 {
1371 
1372 	uma_zfree(pf_mtag_z, t);
1373 }
1374 
1375 struct pf_mtag *
1376 pf_get_mtag(struct mbuf *m)
1377 {
1378 	struct m_tag *mtag;
1379 
1380 	if ((mtag = m_tag_find(m, PACKET_TAG_PF, NULL)) != NULL)
1381 		return ((struct pf_mtag *)(mtag + 1));
1382 
1383 	mtag = uma_zalloc(pf_mtag_z, M_NOWAIT);
1384 	if (mtag == NULL)
1385 		return (NULL);
1386 	bzero(mtag + 1, sizeof(struct pf_mtag));
1387 	m_tag_prepend(m, mtag);
1388 
1389 	return ((struct pf_mtag *)(mtag + 1));
1390 }
1391 
1392 static int
1393 pf_state_key_attach(struct pf_state_key *skw, struct pf_state_key *sks,
1394     struct pf_kstate *s)
1395 {
1396 	struct pf_keyhash	*khs, *khw, *kh;
1397 	struct pf_state_key	*sk, *cur;
1398 	struct pf_kstate	*si, *olds = NULL;
1399 	int idx;
1400 
1401 	NET_EPOCH_ASSERT();
1402 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1403 	KASSERT(s->key[PF_SK_WIRE] == NULL, ("%s: state has key", __func__));
1404 	KASSERT(s->key[PF_SK_STACK] == NULL, ("%s: state has key", __func__));
1405 
1406 	/*
1407 	 * We need to lock hash slots of both keys. To avoid deadlock
1408 	 * we always lock the slot with lower address first. Unlock order
1409 	 * isn't important.
1410 	 *
1411 	 * We also need to lock ID hash slot before dropping key
1412 	 * locks. On success we return with ID hash slot locked.
1413 	 */
1414 
1415 	if (skw == sks) {
1416 		khs = khw = &V_pf_keyhash[pf_hashkey(skw)];
1417 		PF_HASHROW_LOCK(khs);
1418 	} else {
1419 		khs = &V_pf_keyhash[pf_hashkey(sks)];
1420 		khw = &V_pf_keyhash[pf_hashkey(skw)];
1421 		if (khs == khw) {
1422 			PF_HASHROW_LOCK(khs);
1423 		} else if (khs < khw) {
1424 			PF_HASHROW_LOCK(khs);
1425 			PF_HASHROW_LOCK(khw);
1426 		} else {
1427 			PF_HASHROW_LOCK(khw);
1428 			PF_HASHROW_LOCK(khs);
1429 		}
1430 	}
1431 
1432 #define	KEYS_UNLOCK()	do {			\
1433 	if (khs != khw) {			\
1434 		PF_HASHROW_UNLOCK(khs);		\
1435 		PF_HASHROW_UNLOCK(khw);		\
1436 	} else					\
1437 		PF_HASHROW_UNLOCK(khs);		\
1438 } while (0)
1439 
1440 	/*
1441 	 * First run: start with wire key.
1442 	 */
1443 	sk = skw;
1444 	kh = khw;
1445 	idx = PF_SK_WIRE;
1446 
1447 	MPASS(s->lock == NULL);
1448 	s->lock = &V_pf_idhash[PF_IDHASH(s)].lock;
1449 
1450 keyattach:
1451 	LIST_FOREACH(cur, &kh->keys, entry)
1452 		if (bcmp(cur, sk, sizeof(struct pf_state_key_cmp)) == 0)
1453 			break;
1454 
1455 	if (cur != NULL) {
1456 		/* Key exists. Check for same kif, if none, add to key. */
1457 		TAILQ_FOREACH(si, &cur->states[idx], key_list[idx]) {
1458 			struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(si)];
1459 
1460 			PF_HASHROW_LOCK(ih);
1461 			if (si->kif == s->kif &&
1462 			    ((si->key[PF_SK_WIRE]->af == sk->af &&
1463 			    si->direction == s->direction) ||
1464 			    (si->key[PF_SK_WIRE]->af !=
1465 			    si->key[PF_SK_STACK]->af &&
1466 			    sk->af == si->key[PF_SK_STACK]->af &&
1467 			    si->direction != s->direction))) {
1468 				bool reuse = false;
1469 
1470 				if (sk->proto == IPPROTO_TCP &&
1471 				    si->src.state >= TCPS_FIN_WAIT_2 &&
1472 				    si->dst.state >= TCPS_FIN_WAIT_2)
1473 					reuse = true;
1474 
1475 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
1476 					printf("pf: %s key attach "
1477 					    "%s on %s: ",
1478 					    (idx == PF_SK_WIRE) ?
1479 					    "wire" : "stack",
1480 					    reuse ? "reuse" : "failed",
1481 					    s->kif->pfik_name);
1482 					pf_print_state_parts(s,
1483 					    (idx == PF_SK_WIRE) ?
1484 					    sk : NULL,
1485 					    (idx == PF_SK_STACK) ?
1486 					    sk : NULL);
1487 					printf(", existing: ");
1488 					pf_print_state_parts(si,
1489 					    (idx == PF_SK_WIRE) ?
1490 					    sk : NULL,
1491 					    (idx == PF_SK_STACK) ?
1492 					    sk : NULL);
1493 					printf("\n");
1494 				}
1495 
1496 				if (reuse) {
1497 					/*
1498 					 * New state matches an old >FIN_WAIT_2
1499 					 * state. We can't drop key hash locks,
1500 					 * thus we can't unlink it properly.
1501 					 *
1502 					 * As a workaround we drop it into
1503 					 * TCPS_CLOSED state, schedule purge
1504 					 * ASAP and push it into the very end
1505 					 * of the slot TAILQ, so that it won't
1506 					 * conflict with our new state.
1507 					 */
1508 					pf_set_protostate(si, PF_PEER_BOTH,
1509 					    TCPS_CLOSED);
1510 					si->timeout = PFTM_PURGE;
1511 					olds = si;
1512 				} else {
1513 					s->timeout = PFTM_UNLINKED;
1514 					if (idx == PF_SK_STACK)
1515 						/*
1516 						 * Remove the wire key from
1517 						 * the hash. Other threads
1518 						 * can't be referencing it
1519 						 * because we still hold the
1520 						 * hash lock.
1521 						 */
1522 						pf_state_key_detach(s,
1523 						    PF_SK_WIRE);
1524 					PF_HASHROW_UNLOCK(ih);
1525 					KEYS_UNLOCK();
1526 					if (idx == PF_SK_WIRE)
1527 						/*
1528 						 * We've not inserted either key.
1529 						 * Free both.
1530 						 */
1531 						uma_zfree(V_pf_state_key_z, skw);
1532 					if (skw != sks)
1533 						uma_zfree(
1534 						    V_pf_state_key_z,
1535 						    sks);
1536 					return (EEXIST); /* collision! */
1537 				}
1538 			}
1539 			PF_HASHROW_UNLOCK(ih);
1540 		}
1541 		uma_zfree(V_pf_state_key_z, sk);
1542 		s->key[idx] = cur;
1543 	} else {
1544 		LIST_INSERT_HEAD(&kh->keys, sk, entry);
1545 		s->key[idx] = sk;
1546 	}
1547 
1548 stateattach:
1549 	/* List is sorted, if-bound states before floating. */
1550 	if (s->kif == V_pfi_all)
1551 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], s, key_list[idx]);
1552 	else
1553 		TAILQ_INSERT_HEAD(&s->key[idx]->states[idx], s, key_list[idx]);
1554 
1555 	if (olds) {
1556 		TAILQ_REMOVE(&s->key[idx]->states[idx], olds, key_list[idx]);
1557 		TAILQ_INSERT_TAIL(&s->key[idx]->states[idx], olds,
1558 		    key_list[idx]);
1559 		olds = NULL;
1560 	}
1561 
1562 	/*
1563 	 * Attach done. See how should we (or should not?)
1564 	 * attach a second key.
1565 	 */
1566 	if (sks == skw) {
1567 		s->key[PF_SK_STACK] = s->key[PF_SK_WIRE];
1568 		idx = PF_SK_STACK;
1569 		sks = NULL;
1570 		goto stateattach;
1571 	} else if (sks != NULL) {
1572 		/*
1573 		 * Continue attaching with stack key.
1574 		 */
1575 		sk = sks;
1576 		kh = khs;
1577 		idx = PF_SK_STACK;
1578 		sks = NULL;
1579 		goto keyattach;
1580 	}
1581 
1582 	PF_STATE_LOCK(s);
1583 	KEYS_UNLOCK();
1584 
1585 	KASSERT(s->key[PF_SK_WIRE] != NULL && s->key[PF_SK_STACK] != NULL,
1586 	    ("%s failure", __func__));
1587 
1588 	return (0);
1589 #undef	KEYS_UNLOCK
1590 }
1591 
1592 static void
1593 pf_detach_state(struct pf_kstate *s)
1594 {
1595 	struct pf_state_key *sks = s->key[PF_SK_STACK];
1596 	struct pf_keyhash *kh;
1597 
1598 	NET_EPOCH_ASSERT();
1599 	MPASS(s->timeout >= PFTM_MAX);
1600 
1601 	pf_sctp_multihome_detach_addr(s);
1602 
1603 	if ((s->state_flags & PFSTATE_PFLOW) && V_pflow_export_state_ptr)
1604 		V_pflow_export_state_ptr(s);
1605 
1606 	if (sks != NULL) {
1607 		kh = &V_pf_keyhash[pf_hashkey(sks)];
1608 		PF_HASHROW_LOCK(kh);
1609 		if (s->key[PF_SK_STACK] != NULL)
1610 			pf_state_key_detach(s, PF_SK_STACK);
1611 		/*
1612 		 * If both point to same key, then we are done.
1613 		 */
1614 		if (sks == s->key[PF_SK_WIRE]) {
1615 			pf_state_key_detach(s, PF_SK_WIRE);
1616 			PF_HASHROW_UNLOCK(kh);
1617 			return;
1618 		}
1619 		PF_HASHROW_UNLOCK(kh);
1620 	}
1621 
1622 	if (s->key[PF_SK_WIRE] != NULL) {
1623 		kh = &V_pf_keyhash[pf_hashkey(s->key[PF_SK_WIRE])];
1624 		PF_HASHROW_LOCK(kh);
1625 		if (s->key[PF_SK_WIRE] != NULL)
1626 			pf_state_key_detach(s, PF_SK_WIRE);
1627 		PF_HASHROW_UNLOCK(kh);
1628 	}
1629 }
1630 
1631 static void
1632 pf_state_key_detach(struct pf_kstate *s, int idx)
1633 {
1634 	struct pf_state_key *sk = s->key[idx];
1635 #ifdef INVARIANTS
1636 	struct pf_keyhash *kh = &V_pf_keyhash[pf_hashkey(sk)];
1637 
1638 	PF_HASHROW_ASSERT(kh);
1639 #endif /* INVARIANTS */
1640 	TAILQ_REMOVE(&sk->states[idx], s, key_list[idx]);
1641 	s->key[idx] = NULL;
1642 
1643 	if (TAILQ_EMPTY(&sk->states[0]) && TAILQ_EMPTY(&sk->states[1])) {
1644 		LIST_REMOVE(sk, entry);
1645 		uma_zfree(V_pf_state_key_z, sk);
1646 	}
1647 }
1648 
1649 static int
1650 pf_state_key_ctor(void *mem, int size, void *arg, int flags)
1651 {
1652 	struct pf_state_key *sk = mem;
1653 
1654 	bzero(sk, sizeof(struct pf_state_key_cmp));
1655 	TAILQ_INIT(&sk->states[PF_SK_WIRE]);
1656 	TAILQ_INIT(&sk->states[PF_SK_STACK]);
1657 
1658 	return (0);
1659 }
1660 
1661 static int
1662 pf_state_key_addr_setup(struct pf_pdesc *pd,
1663     struct pf_state_key_cmp *key, int multi)
1664 {
1665 	struct pf_addr *saddr = pd->src;
1666 	struct pf_addr *daddr = pd->dst;
1667 #ifdef INET6
1668 	struct nd_neighbor_solicit nd;
1669 	struct pf_addr *target;
1670 
1671 	if (pd->af == AF_INET || pd->proto != IPPROTO_ICMPV6)
1672 		goto copy;
1673 
1674 	switch (pd->hdr.icmp6.icmp6_type) {
1675 	case ND_NEIGHBOR_SOLICIT:
1676 		if (multi)
1677 			return (-1);
1678 		if (!pf_pull_hdr(pd->m, pd->off, &nd, sizeof(nd), NULL,
1679 		    NULL, pd->af))
1680 			return (-1);
1681 		target = (struct pf_addr *)&nd.nd_ns_target;
1682 		daddr = target;
1683 		break;
1684 	case ND_NEIGHBOR_ADVERT:
1685 		if (multi)
1686 			return (-1);
1687 		if (!pf_pull_hdr(pd->m, pd->off, &nd, sizeof(nd), NULL,
1688 		    NULL, pd->af))
1689 			return (-1);
1690 		target = (struct pf_addr *)&nd.nd_ns_target;
1691 		saddr = target;
1692 		if (IN6_IS_ADDR_MULTICAST(&pd->dst->v6)) {
1693 			key->addr[pd->didx].addr32[0] = 0;
1694 			key->addr[pd->didx].addr32[1] = 0;
1695 			key->addr[pd->didx].addr32[2] = 0;
1696 			key->addr[pd->didx].addr32[3] = 0;
1697 			daddr = NULL; /* overwritten */
1698 		}
1699 		break;
1700 	default:
1701 		if (multi) {
1702 			key->addr[pd->sidx].addr32[0] = IPV6_ADDR_INT32_MLL;
1703 			key->addr[pd->sidx].addr32[1] = 0;
1704 			key->addr[pd->sidx].addr32[2] = 0;
1705 			key->addr[pd->sidx].addr32[3] = IPV6_ADDR_INT32_ONE;
1706 			saddr = NULL; /* overwritten */
1707 		}
1708 	}
1709 copy:
1710 #endif /* INET6 */
1711 	if (saddr)
1712 		pf_addrcpy(&key->addr[pd->sidx], saddr, pd->af);
1713 	if (daddr)
1714 		pf_addrcpy(&key->addr[pd->didx], daddr, pd->af);
1715 
1716 	return (0);
1717 }
1718 
1719 int
1720 pf_state_key_setup(struct pf_pdesc *pd, u_int16_t sport, u_int16_t dport,
1721     struct pf_state_key **sk, struct pf_state_key **nk)
1722 {
1723 	*sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1724 	if (*sk == NULL)
1725 		return (ENOMEM);
1726 
1727 	if (pf_state_key_addr_setup(pd, (struct pf_state_key_cmp *)*sk,
1728 	    0)) {
1729 		uma_zfree(V_pf_state_key_z, *sk);
1730 		*sk = NULL;
1731 		return (ENOMEM);
1732 	}
1733 
1734 	(*sk)->port[pd->sidx] = sport;
1735 	(*sk)->port[pd->didx] = dport;
1736 	(*sk)->proto = pd->proto;
1737 	(*sk)->af = pd->af;
1738 
1739 	*nk = pf_state_key_clone(*sk);
1740 	if (*nk == NULL) {
1741 		uma_zfree(V_pf_state_key_z, *sk);
1742 		*sk = NULL;
1743 		return (ENOMEM);
1744 	}
1745 
1746 	if (pd->af != pd->naf) {
1747 		(*sk)->port[pd->sidx] = pd->osport;
1748 		(*sk)->port[pd->didx] = pd->odport;
1749 
1750 		(*nk)->af = pd->naf;
1751 
1752 		/*
1753 		 * We're overwriting an address here, so potentially there's bits of an IPv6
1754 		 * address left in here. Clear that out first.
1755 		 */
1756 		bzero(&(*nk)->addr[0], sizeof((*nk)->addr[0]));
1757 		bzero(&(*nk)->addr[1], sizeof((*nk)->addr[1]));
1758 		if (pd->dir == PF_IN) {
1759 			pf_addrcpy(&(*nk)->addr[pd->didx], &pd->nsaddr,
1760 			    pd->naf);
1761 			pf_addrcpy(&(*nk)->addr[pd->sidx], &pd->ndaddr,
1762 			    pd->naf);
1763 			(*nk)->port[pd->didx] = pd->nsport;
1764 			(*nk)->port[pd->sidx] = pd->ndport;
1765 		} else {
1766 			pf_addrcpy(&(*nk)->addr[pd->sidx], &pd->nsaddr,
1767 			    pd->naf);
1768 			pf_addrcpy(&(*nk)->addr[pd->didx], &pd->ndaddr,
1769 			    pd->naf);
1770 			(*nk)->port[pd->sidx] = pd->nsport;
1771 			(*nk)->port[pd->didx] = pd->ndport;
1772 		}
1773 
1774 		switch (pd->proto) {
1775 		case IPPROTO_ICMP:
1776 			(*nk)->proto = IPPROTO_ICMPV6;
1777 			break;
1778 		case IPPROTO_ICMPV6:
1779 			(*nk)->proto = IPPROTO_ICMP;
1780 			break;
1781 		default:
1782 			(*nk)->proto = pd->proto;
1783 		}
1784 	}
1785 
1786 	return (0);
1787 }
1788 
1789 struct pf_state_key *
1790 pf_state_key_clone(const struct pf_state_key *orig)
1791 {
1792 	struct pf_state_key *sk;
1793 
1794 	sk = uma_zalloc(V_pf_state_key_z, M_NOWAIT);
1795 	if (sk == NULL)
1796 		return (NULL);
1797 
1798 	bcopy(orig, sk, sizeof(struct pf_state_key_cmp));
1799 
1800 	return (sk);
1801 }
1802 
1803 int
1804 pf_state_insert(struct pfi_kkif *kif, struct pfi_kkif *orig_kif,
1805     struct pf_state_key *skw, struct pf_state_key *sks, struct pf_kstate *s)
1806 {
1807 	struct pf_idhash *ih;
1808 	struct pf_kstate *cur;
1809 	int error;
1810 
1811 	NET_EPOCH_ASSERT();
1812 
1813 	KASSERT(TAILQ_EMPTY(&sks->states[0]) && TAILQ_EMPTY(&sks->states[1]),
1814 	    ("%s: sks not pristine", __func__));
1815 	KASSERT(TAILQ_EMPTY(&skw->states[0]) && TAILQ_EMPTY(&skw->states[1]),
1816 	    ("%s: skw not pristine", __func__));
1817 	KASSERT(s->refs == 0, ("%s: state not pristine", __func__));
1818 
1819 	s->kif = kif;
1820 	s->orig_kif = orig_kif;
1821 
1822 	if (s->id == 0 && s->creatorid == 0) {
1823 		s->id = alloc_unr64(&V_pf_stateid);
1824 		s->id = htobe64(s->id);
1825 		s->creatorid = V_pf_status.hostid;
1826 	}
1827 
1828 	/* Returns with ID locked on success. */
1829 	if ((error = pf_state_key_attach(skw, sks, s)) != 0)
1830 		return (error);
1831 	skw = sks = NULL;
1832 
1833 	ih = &V_pf_idhash[PF_IDHASH(s)];
1834 	PF_HASHROW_ASSERT(ih);
1835 	LIST_FOREACH(cur, &ih->states, entry)
1836 		if (cur->id == s->id && cur->creatorid == s->creatorid)
1837 			break;
1838 
1839 	if (cur != NULL) {
1840 		s->timeout = PFTM_UNLINKED;
1841 		PF_HASHROW_UNLOCK(ih);
1842 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
1843 			printf("pf: state ID collision: "
1844 			    "id: %016llx creatorid: %08x\n",
1845 			    (unsigned long long)be64toh(s->id),
1846 			    ntohl(s->creatorid));
1847 		}
1848 		pf_detach_state(s);
1849 		return (EEXIST);
1850 	}
1851 	LIST_INSERT_HEAD(&ih->states, s, entry);
1852 	/* One for keys, one for ID hash. */
1853 	refcount_init(&s->refs, 2);
1854 
1855 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_INSERT], 1);
1856 	if (V_pfsync_insert_state_ptr != NULL)
1857 		V_pfsync_insert_state_ptr(s);
1858 
1859 	/* Returns locked. */
1860 	return (0);
1861 }
1862 
1863 /*
1864  * Find state by ID: returns with locked row on success.
1865  */
1866 struct pf_kstate *
1867 pf_find_state_byid(uint64_t id, uint32_t creatorid)
1868 {
1869 	struct pf_idhash *ih;
1870 	struct pf_kstate *s;
1871 
1872 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1873 
1874 	ih = &V_pf_idhash[PF_IDHASHID(id)];
1875 
1876 	PF_HASHROW_LOCK(ih);
1877 	LIST_FOREACH(s, &ih->states, entry)
1878 		if (s->id == id && s->creatorid == creatorid)
1879 			break;
1880 
1881 	if (s == NULL)
1882 		PF_HASHROW_UNLOCK(ih);
1883 
1884 	return (s);
1885 }
1886 
1887 /*
1888  * Find state by key.
1889  * Returns with ID hash slot locked on success.
1890  */
1891 static int
1892 pf_find_state(struct pf_pdesc *pd, const struct pf_state_key_cmp *key,
1893     struct pf_kstate **state)
1894 {
1895 	struct pf_keyhash	*kh;
1896 	struct pf_state_key	*sk;
1897 	struct pf_kstate	*s;
1898 	int idx;
1899 
1900 	*state = NULL;
1901 
1902 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
1903 
1904 	kh = &V_pf_keyhash[pf_hashkey((const struct pf_state_key *)key)];
1905 
1906 	PF_HASHROW_LOCK(kh);
1907 	LIST_FOREACH(sk, &kh->keys, entry)
1908 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
1909 			break;
1910 	if (sk == NULL) {
1911 		PF_HASHROW_UNLOCK(kh);
1912 		return (PF_DROP);
1913 	}
1914 
1915 	idx = (pd->dir == PF_IN ? PF_SK_WIRE : PF_SK_STACK);
1916 
1917 	/* List is sorted, if-bound states before floating ones. */
1918 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx])
1919 		if (s->kif == V_pfi_all || s->kif == pd->kif ||
1920 		    s->orig_kif == pd->kif) {
1921 			PF_STATE_LOCK(s);
1922 			PF_HASHROW_UNLOCK(kh);
1923 			if (__predict_false(s->timeout >= PFTM_MAX)) {
1924 				/*
1925 				 * State is either being processed by
1926 				 * pf_remove_state() in an other thread, or
1927 				 * is scheduled for immediate expiry.
1928 				 */
1929 				PF_STATE_UNLOCK(s);
1930 				SDT_PROBE5(pf, ip, state, lookup, pd->kif,
1931 				    key, (pd->dir), pd, *state);
1932 				return (PF_DROP);
1933 			}
1934 			goto out;
1935 		}
1936 
1937 	/* Look through the other list, in case of AF-TO */
1938 	idx = idx == PF_SK_WIRE ? PF_SK_STACK : PF_SK_WIRE;
1939 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx]) {
1940 		if (s->key[PF_SK_WIRE]->af == s->key[PF_SK_STACK]->af)
1941 			continue;
1942 		if (s->kif == V_pfi_all || s->kif == pd->kif ||
1943 		    s->orig_kif == pd->kif) {
1944 			PF_STATE_LOCK(s);
1945 			PF_HASHROW_UNLOCK(kh);
1946 			if (__predict_false(s->timeout >= PFTM_MAX)) {
1947 				/*
1948 				 * State is either being processed by
1949 				 * pf_remove_state() in an other thread, or
1950 				 * is scheduled for immediate expiry.
1951 				 */
1952 				PF_STATE_UNLOCK(s);
1953 				SDT_PROBE5(pf, ip, state, lookup, pd->kif,
1954 				    key, (pd->dir), pd, NULL);
1955 				return (PF_DROP);
1956 			}
1957 			goto out;
1958 		}
1959 	}
1960 
1961 	PF_HASHROW_UNLOCK(kh);
1962 
1963 out:
1964 	SDT_PROBE5(pf, ip, state, lookup, pd->kif, key, (pd->dir), pd, *state);
1965 
1966 	if (s == NULL || s->timeout == PFTM_PURGE) {
1967 		if (s)
1968 			PF_STATE_UNLOCK(s);
1969 		return (PF_DROP);
1970 	}
1971 
1972 	if ((s)->rule->pktrate.limit && pd->dir == (s)->direction) {
1973 		if (pf_check_threshold(&(s)->rule->pktrate)) {
1974 			PF_STATE_UNLOCK(s);
1975 			return (PF_DROP);
1976 		}
1977 	}
1978 	if (PACKET_LOOPED(pd)) {
1979 		PF_STATE_UNLOCK(s);
1980 		return (PF_PASS);
1981 	}
1982 
1983 	*state = s;
1984 
1985 	return (PF_MATCH);
1986 }
1987 
1988 /*
1989  * Returns with ID hash slot locked on success.
1990  */
1991 struct pf_kstate *
1992 pf_find_state_all(const struct pf_state_key_cmp *key, u_int dir, int *more)
1993 {
1994 	struct pf_keyhash	*kh;
1995 	struct pf_state_key	*sk;
1996 	struct pf_kstate	*s, *ret = NULL;
1997 	int			 idx, inout = 0;
1998 
1999 	if (more != NULL)
2000 		*more = 0;
2001 
2002 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_SEARCH], 1);
2003 
2004 	kh = &V_pf_keyhash[pf_hashkey((const struct pf_state_key *)key)];
2005 
2006 	PF_HASHROW_LOCK(kh);
2007 	LIST_FOREACH(sk, &kh->keys, entry)
2008 		if (bcmp(sk, key, sizeof(struct pf_state_key_cmp)) == 0)
2009 			break;
2010 	if (sk == NULL) {
2011 		PF_HASHROW_UNLOCK(kh);
2012 		return (NULL);
2013 	}
2014 	switch (dir) {
2015 	case PF_IN:
2016 		idx = PF_SK_WIRE;
2017 		break;
2018 	case PF_OUT:
2019 		idx = PF_SK_STACK;
2020 		break;
2021 	case PF_INOUT:
2022 		idx = PF_SK_WIRE;
2023 		inout = 1;
2024 		break;
2025 	default:
2026 		panic("%s: dir %u", __func__, dir);
2027 	}
2028 second_run:
2029 	TAILQ_FOREACH(s, &sk->states[idx], key_list[idx]) {
2030 		if (more == NULL) {
2031 			PF_STATE_LOCK(s);
2032 			PF_HASHROW_UNLOCK(kh);
2033 			return (s);
2034 		}
2035 
2036 		if (ret)
2037 			(*more)++;
2038 		else {
2039 			ret = s;
2040 			PF_STATE_LOCK(s);
2041 		}
2042 	}
2043 	if (inout == 1) {
2044 		inout = 0;
2045 		idx = PF_SK_STACK;
2046 		goto second_run;
2047 	}
2048 	PF_HASHROW_UNLOCK(kh);
2049 
2050 	return (ret);
2051 }
2052 
2053 /*
2054  * FIXME
2055  * This routine is inefficient -- locks the state only to unlock immediately on
2056  * return.
2057  * It is racy -- after the state is unlocked nothing stops other threads from
2058  * removing it.
2059  */
2060 bool
2061 pf_find_state_all_exists(const struct pf_state_key_cmp *key, u_int dir)
2062 {
2063 	struct pf_kstate *s;
2064 
2065 	s = pf_find_state_all(key, dir, NULL);
2066 	if (s != NULL) {
2067 		PF_STATE_UNLOCK(s);
2068 		return (true);
2069 	}
2070 	return (false);
2071 }
2072 
2073 void
2074 pf_state_peer_hton(const struct pf_state_peer *s, struct pf_state_peer_export *d)
2075 {
2076 	d->seqlo = htonl(s->seqlo);
2077 	d->seqhi = htonl(s->seqhi);
2078 	d->seqdiff = htonl(s->seqdiff);
2079 	d->max_win = htons(s->max_win);
2080 	d->mss = htons(s->mss);
2081 	d->state = s->state;
2082 	d->wscale = s->wscale;
2083 	if (s->scrub) {
2084 		d->scrub.pfss_flags = htons(
2085 		    s->scrub->pfss_flags & PFSS_TIMESTAMP);
2086 		d->scrub.pfss_ttl = (s)->scrub->pfss_ttl;
2087 		d->scrub.pfss_ts_mod = htonl((s)->scrub->pfss_ts_mod);
2088 		d->scrub.scrub_flag = PF_SCRUB_FLAG_VALID;
2089 	}
2090 }
2091 
2092 void
2093 pf_state_peer_ntoh(const struct pf_state_peer_export *s, struct pf_state_peer *d)
2094 {
2095 	d->seqlo = ntohl(s->seqlo);
2096 	d->seqhi = ntohl(s->seqhi);
2097 	d->seqdiff = ntohl(s->seqdiff);
2098 	d->max_win = ntohs(s->max_win);
2099 	d->mss = ntohs(s->mss);
2100 	d->state = s->state;
2101 	d->wscale = s->wscale;
2102 	if (s->scrub.scrub_flag == PF_SCRUB_FLAG_VALID &&
2103 	    d->scrub != NULL) {
2104 		d->scrub->pfss_flags = ntohs(s->scrub.pfss_flags) &
2105 		    PFSS_TIMESTAMP;
2106 		d->scrub->pfss_ttl = s->scrub.pfss_ttl;
2107 		d->scrub->pfss_ts_mod = ntohl(s->scrub.pfss_ts_mod);
2108 	}
2109 }
2110 
2111 struct pf_udp_mapping *
2112 pf_udp_mapping_create(sa_family_t af, struct pf_addr *src_addr, uint16_t src_port,
2113     struct pf_addr *nat_addr, uint16_t nat_port)
2114 {
2115 	struct pf_udp_mapping *mapping;
2116 
2117 	mapping = uma_zalloc(V_pf_udp_mapping_z, M_NOWAIT | M_ZERO);
2118 	if (mapping == NULL)
2119 		return (NULL);
2120 	pf_addrcpy(&mapping->endpoints[0].addr, src_addr, af);
2121 	mapping->endpoints[0].port = src_port;
2122 	mapping->endpoints[0].af = af;
2123 	mapping->endpoints[0].mapping = mapping;
2124 	pf_addrcpy(&mapping->endpoints[1].addr, nat_addr, af);
2125 	mapping->endpoints[1].port = nat_port;
2126 	mapping->endpoints[1].af = af;
2127 	mapping->endpoints[1].mapping = mapping;
2128 	refcount_init(&mapping->refs, 1);
2129 	return (mapping);
2130 }
2131 
2132 int
2133 pf_udp_mapping_insert(struct pf_udp_mapping *mapping)
2134 {
2135 	struct pf_udpendpointhash *h0, *h1;
2136 	struct pf_udp_endpoint *endpoint;
2137 	int ret = EEXIST;
2138 
2139 	h0 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[0])];
2140 	h1 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[1])];
2141 	if (h0 == h1) {
2142 		PF_HASHROW_LOCK(h0);
2143 	} else if (h0 < h1) {
2144 		PF_HASHROW_LOCK(h0);
2145 		PF_HASHROW_LOCK(h1);
2146 	} else {
2147 		PF_HASHROW_LOCK(h1);
2148 		PF_HASHROW_LOCK(h0);
2149 	}
2150 
2151 	LIST_FOREACH(endpoint, &h0->endpoints, entry) {
2152 		if (bcmp(endpoint, &mapping->endpoints[0],
2153 		    sizeof(struct pf_udp_endpoint_cmp)) == 0)
2154 			break;
2155 	}
2156 	if (endpoint != NULL)
2157 		goto cleanup;
2158 	LIST_FOREACH(endpoint, &h1->endpoints, entry) {
2159 		if (bcmp(endpoint, &mapping->endpoints[1],
2160 		    sizeof(struct pf_udp_endpoint_cmp)) == 0)
2161 			break;
2162 	}
2163 	if (endpoint != NULL)
2164 		goto cleanup;
2165 	LIST_INSERT_HEAD(&h0->endpoints, &mapping->endpoints[0], entry);
2166 	LIST_INSERT_HEAD(&h1->endpoints, &mapping->endpoints[1], entry);
2167 	ret = 0;
2168 
2169 cleanup:
2170 	if (h0 != h1) {
2171 		PF_HASHROW_UNLOCK(h0);
2172 		PF_HASHROW_UNLOCK(h1);
2173 	} else {
2174 		PF_HASHROW_UNLOCK(h0);
2175 	}
2176 	return (ret);
2177 }
2178 
2179 void
2180 pf_udp_mapping_release(struct pf_udp_mapping *mapping)
2181 {
2182 	/* refcount is synchronized on the source endpoint's row lock */
2183 	struct pf_udpendpointhash *h0, *h1;
2184 
2185 	if (mapping == NULL)
2186 		return;
2187 
2188 	h0 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[0])];
2189 	PF_HASHROW_LOCK(h0);
2190 	if (refcount_release(&mapping->refs)) {
2191 		LIST_REMOVE(&mapping->endpoints[0], entry);
2192 		PF_HASHROW_UNLOCK(h0);
2193 		h1 = &V_pf_udpendpointhash[pf_hashudpendpoint(&mapping->endpoints[1])];
2194 		PF_HASHROW_LOCK(h1);
2195 		LIST_REMOVE(&mapping->endpoints[1], entry);
2196 		PF_HASHROW_UNLOCK(h1);
2197 
2198 		uma_zfree(V_pf_udp_mapping_z, mapping);
2199 	} else {
2200 			PF_HASHROW_UNLOCK(h0);
2201 	}
2202 }
2203 
2204 
2205 struct pf_udp_mapping *
2206 pf_udp_mapping_find(struct pf_udp_endpoint_cmp *key)
2207 {
2208 	struct pf_udpendpointhash *uh;
2209 	struct pf_udp_endpoint *endpoint;
2210 
2211 	uh = &V_pf_udpendpointhash[pf_hashudpendpoint((struct pf_udp_endpoint*)key)];
2212 
2213 	PF_HASHROW_LOCK(uh);
2214 	LIST_FOREACH(endpoint, &uh->endpoints, entry) {
2215 		if (bcmp(endpoint, key, sizeof(struct pf_udp_endpoint_cmp)) == 0 &&
2216 			bcmp(endpoint, &endpoint->mapping->endpoints[0],
2217 			    sizeof(struct pf_udp_endpoint_cmp)) == 0)
2218 			break;
2219 	}
2220 	if (endpoint == NULL) {
2221 		PF_HASHROW_UNLOCK(uh);
2222 		return (NULL);
2223 	}
2224 	refcount_acquire(&endpoint->mapping->refs);
2225 	PF_HASHROW_UNLOCK(uh);
2226 	return (endpoint->mapping);
2227 }
2228 /* END state table stuff */
2229 
2230 static void
2231 pf_send(struct pf_send_entry *pfse)
2232 {
2233 
2234 	PF_SENDQ_LOCK();
2235 	STAILQ_INSERT_TAIL(&V_pf_sendqueue, pfse, pfse_next);
2236 	PF_SENDQ_UNLOCK();
2237 	swi_sched(V_pf_swi_cookie, 0);
2238 }
2239 
2240 static bool
2241 pf_isforlocal(struct mbuf *m, int af)
2242 {
2243 	switch (af) {
2244 #ifdef INET
2245 	case AF_INET: {
2246 		struct ip *ip = mtod(m, struct ip *);
2247 
2248 		return (in_localip(ip->ip_dst));
2249 	}
2250 #endif /* INET */
2251 #ifdef INET6
2252 	case AF_INET6: {
2253 		struct ip6_hdr *ip6;
2254 		struct in6_ifaddr *ia;
2255 		ip6 = mtod(m, struct ip6_hdr *);
2256 		ia = in6ifa_ifwithaddr(&ip6->ip6_dst, 0 /* XXX */, false);
2257 		if (ia == NULL)
2258 			return (false);
2259 		return (! (ia->ia6_flags & IN6_IFF_NOTREADY));
2260 	}
2261 #endif /* INET6 */
2262 	default:
2263 		unhandled_af(af);
2264 	}
2265 
2266 	return (false);
2267 }
2268 
2269 int
2270 pf_icmp_mapping(struct pf_pdesc *pd, u_int8_t type,
2271     int *icmp_dir, u_int16_t *virtual_id, u_int16_t *virtual_type)
2272 {
2273 	/*
2274 	 * ICMP types marked with PF_OUT are typically responses to
2275 	 * PF_IN, and will match states in the opposite direction.
2276 	 * PF_IN ICMP types need to match a state with that type.
2277 	 */
2278 	*icmp_dir = PF_OUT;
2279 
2280 	/* Queries (and responses) */
2281 	switch (pd->af) {
2282 #ifdef INET
2283 	case AF_INET:
2284 		switch (type) {
2285 		case ICMP_ECHO:
2286 			*icmp_dir = PF_IN;
2287 			/* FALLTHROUGH */
2288 		case ICMP_ECHOREPLY:
2289 			*virtual_type = ICMP_ECHO;
2290 			*virtual_id = pd->hdr.icmp.icmp_id;
2291 			break;
2292 
2293 		case ICMP_TSTAMP:
2294 			*icmp_dir = PF_IN;
2295 			/* FALLTHROUGH */
2296 		case ICMP_TSTAMPREPLY:
2297 			*virtual_type = ICMP_TSTAMP;
2298 			*virtual_id = pd->hdr.icmp.icmp_id;
2299 			break;
2300 
2301 		case ICMP_IREQ:
2302 			*icmp_dir = PF_IN;
2303 			/* FALLTHROUGH */
2304 		case ICMP_IREQREPLY:
2305 			*virtual_type = ICMP_IREQ;
2306 			*virtual_id = pd->hdr.icmp.icmp_id;
2307 			break;
2308 
2309 		case ICMP_MASKREQ:
2310 			*icmp_dir = PF_IN;
2311 			/* FALLTHROUGH */
2312 		case ICMP_MASKREPLY:
2313 			*virtual_type = ICMP_MASKREQ;
2314 			*virtual_id = pd->hdr.icmp.icmp_id;
2315 			break;
2316 
2317 		case ICMP_IPV6_WHEREAREYOU:
2318 			*icmp_dir = PF_IN;
2319 			/* FALLTHROUGH */
2320 		case ICMP_IPV6_IAMHERE:
2321 			*virtual_type = ICMP_IPV6_WHEREAREYOU;
2322 			*virtual_id = 0; /* Nothing sane to match on! */
2323 			break;
2324 
2325 		case ICMP_MOBILE_REGREQUEST:
2326 			*icmp_dir = PF_IN;
2327 			/* FALLTHROUGH */
2328 		case ICMP_MOBILE_REGREPLY:
2329 			*virtual_type = ICMP_MOBILE_REGREQUEST;
2330 			*virtual_id = 0; /* Nothing sane to match on! */
2331 			break;
2332 
2333 		case ICMP_ROUTERSOLICIT:
2334 			*icmp_dir = PF_IN;
2335 			/* FALLTHROUGH */
2336 		case ICMP_ROUTERADVERT:
2337 			*virtual_type = ICMP_ROUTERSOLICIT;
2338 			*virtual_id = 0; /* Nothing sane to match on! */
2339 			break;
2340 
2341 		/* These ICMP types map to other connections */
2342 		case ICMP_UNREACH:
2343 		case ICMP_SOURCEQUENCH:
2344 		case ICMP_REDIRECT:
2345 		case ICMP_TIMXCEED:
2346 		case ICMP_PARAMPROB:
2347 			/* These will not be used, but set them anyway */
2348 			*icmp_dir = PF_IN;
2349 			*virtual_type = type;
2350 			*virtual_id = 0;
2351 			*virtual_type = htons(*virtual_type);
2352 			return (1);  /* These types match to another state */
2353 
2354 		/*
2355 		 * All remaining ICMP types get their own states,
2356 		 * and will only match in one direction.
2357 		 */
2358 		default:
2359 			*icmp_dir = PF_IN;
2360 			*virtual_type = type;
2361 			*virtual_id = 0;
2362 			break;
2363 		}
2364 		break;
2365 #endif /* INET */
2366 #ifdef INET6
2367 	case AF_INET6:
2368 		switch (type) {
2369 		case ICMP6_ECHO_REQUEST:
2370 			*icmp_dir = PF_IN;
2371 			/* FALLTHROUGH */
2372 		case ICMP6_ECHO_REPLY:
2373 			*virtual_type = ICMP6_ECHO_REQUEST;
2374 			*virtual_id = pd->hdr.icmp6.icmp6_id;
2375 			break;
2376 
2377 		case MLD_LISTENER_QUERY:
2378 		case MLD_LISTENER_REPORT: {
2379 			/*
2380 			 * Listener Report can be sent by clients
2381 			 * without an associated Listener Query.
2382 			 * In addition to that, when Report is sent as a
2383 			 * reply to a Query its source and destination
2384 			 * address are different.
2385 			 */
2386 			*icmp_dir = PF_IN;
2387 			*virtual_type = MLD_LISTENER_QUERY;
2388 			*virtual_id = 0;
2389 			break;
2390 		}
2391 		case MLD_MTRACE:
2392 			*icmp_dir = PF_IN;
2393 			/* FALLTHROUGH */
2394 		case MLD_MTRACE_RESP:
2395 			*virtual_type = MLD_MTRACE;
2396 			*virtual_id = 0; /* Nothing sane to match on! */
2397 			break;
2398 
2399 		case ND_NEIGHBOR_SOLICIT:
2400 			*icmp_dir = PF_IN;
2401 			/* FALLTHROUGH */
2402 		case ND_NEIGHBOR_ADVERT: {
2403 			*virtual_type = ND_NEIGHBOR_SOLICIT;
2404 			*virtual_id = 0;
2405 			break;
2406 		}
2407 
2408 		/*
2409 		 * These ICMP types map to other connections.
2410 		 * ND_REDIRECT can't be in this list because the triggering
2411 		 * packet header is optional.
2412 		 */
2413 		case ICMP6_DST_UNREACH:
2414 		case ICMP6_PACKET_TOO_BIG:
2415 		case ICMP6_TIME_EXCEEDED:
2416 		case ICMP6_PARAM_PROB:
2417 			/* These will not be used, but set them anyway */
2418 			*icmp_dir = PF_IN;
2419 			*virtual_type = type;
2420 			*virtual_id = 0;
2421 			*virtual_type = htons(*virtual_type);
2422 			return (1);  /* These types match to another state */
2423 		/*
2424 		 * All remaining ICMP6 types get their own states,
2425 		 * and will only match in one direction.
2426 		 */
2427 		default:
2428 			*icmp_dir = PF_IN;
2429 			*virtual_type = type;
2430 			*virtual_id = 0;
2431 			break;
2432 		}
2433 		break;
2434 #endif /* INET6 */
2435 	default:
2436 		unhandled_af(pd->af);
2437 	}
2438 	*virtual_type = htons(*virtual_type);
2439 	return (0);  /* These types match to their own state */
2440 }
2441 
2442 void
2443 pf_intr(void *v)
2444 {
2445 	struct epoch_tracker et;
2446 	struct pf_send_head queue;
2447 	struct pf_send_entry *pfse, *next;
2448 
2449 	CURVNET_SET((struct vnet *)v);
2450 
2451 	PF_SENDQ_LOCK();
2452 	queue = V_pf_sendqueue;
2453 	STAILQ_INIT(&V_pf_sendqueue);
2454 	PF_SENDQ_UNLOCK();
2455 
2456 	NET_EPOCH_ENTER(et);
2457 
2458 	STAILQ_FOREACH_SAFE(pfse, &queue, pfse_next, next) {
2459 		switch (pfse->pfse_type) {
2460 #ifdef INET
2461 		case PFSE_IP: {
2462 			if (pf_isforlocal(pfse->pfse_m, AF_INET)) {
2463 				KASSERT(pfse->pfse_m->m_pkthdr.rcvif == V_loif,
2464 				    ("%s: rcvif != loif", __func__));
2465 
2466 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL;
2467 				pfse->pfse_m->m_pkthdr.csum_flags |=
2468 				    CSUM_IP_VALID | CSUM_IP_CHECKED |
2469 				    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
2470 				pfse->pfse_m->m_pkthdr.csum_data = 0xffff;
2471 				ip_input(pfse->pfse_m);
2472 			} else {
2473 				ip_output(pfse->pfse_m, NULL, NULL, 0, NULL,
2474 				    NULL);
2475 			}
2476 			break;
2477 		}
2478 		case PFSE_ICMP:
2479 			icmp_error(pfse->pfse_m, pfse->icmpopts.type,
2480 			    pfse->icmpopts.code, 0, pfse->icmpopts.mtu);
2481 			break;
2482 #endif /* INET */
2483 #ifdef INET6
2484 		case PFSE_IP6:
2485 			if (pf_isforlocal(pfse->pfse_m, AF_INET6)) {
2486 				KASSERT(pfse->pfse_m->m_pkthdr.rcvif == V_loif,
2487 				    ("%s: rcvif != loif", __func__));
2488 
2489 				pfse->pfse_m->m_flags |= M_SKIP_FIREWALL |
2490 				    M_LOOP;
2491 				pfse->pfse_m->m_pkthdr.csum_flags |=
2492 				    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
2493 				pfse->pfse_m->m_pkthdr.csum_data = 0xffff;
2494 				ip6_input(pfse->pfse_m);
2495 			} else {
2496 				ip6_output(pfse->pfse_m, NULL, NULL, 0, NULL,
2497 				    NULL, NULL);
2498 			}
2499 			break;
2500 		case PFSE_ICMP6:
2501 			icmp6_error(pfse->pfse_m, pfse->icmpopts.type,
2502 			    pfse->icmpopts.code, pfse->icmpopts.mtu);
2503 			break;
2504 #endif /* INET6 */
2505 		default:
2506 			panic("%s: unknown type", __func__);
2507 		}
2508 		free(pfse, M_PFTEMP);
2509 	}
2510 	NET_EPOCH_EXIT(et);
2511 	CURVNET_RESTORE();
2512 }
2513 
2514 #define	pf_purge_thread_period	(hz / 10)
2515 
2516 #ifdef PF_WANT_32_TO_64_COUNTER
2517 static void
2518 pf_status_counter_u64_periodic(void)
2519 {
2520 
2521 	PF_RULES_RASSERT();
2522 
2523 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 60)) != 0) {
2524 		return;
2525 	}
2526 
2527 	for (int i = 0; i < FCNT_MAX; i++) {
2528 		pf_counter_u64_periodic(&V_pf_status.fcounters[i]);
2529 	}
2530 }
2531 
2532 static void
2533 pf_kif_counter_u64_periodic(void)
2534 {
2535 	struct pfi_kkif *kif;
2536 	size_t r, run;
2537 
2538 	PF_RULES_RASSERT();
2539 
2540 	if (__predict_false(V_pf_allkifcount == 0)) {
2541 		return;
2542 	}
2543 
2544 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
2545 		return;
2546 	}
2547 
2548 	run = V_pf_allkifcount / 10;
2549 	if (run < 5)
2550 		run = 5;
2551 
2552 	for (r = 0; r < run; r++) {
2553 		kif = LIST_NEXT(V_pf_kifmarker, pfik_allkiflist);
2554 		if (kif == NULL) {
2555 			LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
2556 			LIST_INSERT_HEAD(&V_pf_allkiflist, V_pf_kifmarker, pfik_allkiflist);
2557 			break;
2558 		}
2559 
2560 		LIST_REMOVE(V_pf_kifmarker, pfik_allkiflist);
2561 		LIST_INSERT_AFTER(kif, V_pf_kifmarker, pfik_allkiflist);
2562 
2563 		for (int i = 0; i < 2; i++) {
2564 			for (int j = 0; j < 2; j++) {
2565 				for (int k = 0; k < 2; k++) {
2566 					pf_counter_u64_periodic(&kif->pfik_packets[i][j][k]);
2567 					pf_counter_u64_periodic(&kif->pfik_bytes[i][j][k]);
2568 				}
2569 			}
2570 		}
2571 	}
2572 }
2573 
2574 static void
2575 pf_rule_counter_u64_periodic(void)
2576 {
2577 	struct pf_krule *rule;
2578 	size_t r, run;
2579 
2580 	PF_RULES_RASSERT();
2581 
2582 	if (__predict_false(V_pf_allrulecount == 0)) {
2583 		return;
2584 	}
2585 
2586 	if ((V_pf_counter_periodic_iter % (pf_purge_thread_period * 10 * 300)) != 0) {
2587 		return;
2588 	}
2589 
2590 	run = V_pf_allrulecount / 10;
2591 	if (run < 5)
2592 		run = 5;
2593 
2594 	for (r = 0; r < run; r++) {
2595 		rule = LIST_NEXT(V_pf_rulemarker, allrulelist);
2596 		if (rule == NULL) {
2597 			LIST_REMOVE(V_pf_rulemarker, allrulelist);
2598 			LIST_INSERT_HEAD(&V_pf_allrulelist, V_pf_rulemarker, allrulelist);
2599 			break;
2600 		}
2601 
2602 		LIST_REMOVE(V_pf_rulemarker, allrulelist);
2603 		LIST_INSERT_AFTER(rule, V_pf_rulemarker, allrulelist);
2604 
2605 		pf_counter_u64_periodic(&rule->evaluations);
2606 		for (int i = 0; i < 2; i++) {
2607 			pf_counter_u64_periodic(&rule->packets[i]);
2608 			pf_counter_u64_periodic(&rule->bytes[i]);
2609 		}
2610 	}
2611 }
2612 
2613 static void
2614 pf_counter_u64_periodic_main(void)
2615 {
2616 	PF_RULES_RLOCK_TRACKER;
2617 
2618 	V_pf_counter_periodic_iter++;
2619 
2620 	PF_RULES_RLOCK();
2621 	pf_counter_u64_critical_enter();
2622 	pf_status_counter_u64_periodic();
2623 	pf_kif_counter_u64_periodic();
2624 	pf_rule_counter_u64_periodic();
2625 	pf_counter_u64_critical_exit();
2626 	PF_RULES_RUNLOCK();
2627 }
2628 #else
2629 #define	pf_counter_u64_periodic_main()	do { } while (0)
2630 #endif
2631 
2632 void
2633 pf_purge_thread(void *unused __unused)
2634 {
2635 	struct epoch_tracker	 et;
2636 
2637 	VNET_ITERATOR_DECL(vnet_iter);
2638 
2639 	sx_xlock(&pf_end_lock);
2640 	while (pf_end_threads == 0) {
2641 		sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
2642 
2643 		VNET_LIST_RLOCK();
2644 		NET_EPOCH_ENTER(et);
2645 		VNET_FOREACH(vnet_iter) {
2646 			CURVNET_SET(vnet_iter);
2647 
2648 			/* Wait until V_pf_default_rule is initialized. */
2649 			if (V_pf_vnet_active == 0) {
2650 				CURVNET_RESTORE();
2651 				continue;
2652 			}
2653 
2654 			pf_counter_u64_periodic_main();
2655 
2656 			/*
2657 			 *  Process 1/interval fraction of the state
2658 			 * table every run.
2659 			 */
2660 			V_pf_purge_idx =
2661 			    pf_purge_expired_states(V_pf_purge_idx, V_pf_hashmask /
2662 			    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
2663 
2664 			/*
2665 			 * Purge other expired types every
2666 			 * PFTM_INTERVAL seconds.
2667 			 */
2668 			if (V_pf_purge_idx == 0) {
2669 				/*
2670 				 * Order is important:
2671 				 * - states and src nodes reference rules
2672 				 * - states and rules reference kifs
2673 				 */
2674 				pf_purge_expired_fragments();
2675 				pf_purge_expired_src_nodes();
2676 				pf_purge_unlinked_rules();
2677 				pfi_kkif_purge();
2678 			}
2679 			CURVNET_RESTORE();
2680 		}
2681 		NET_EPOCH_EXIT(et);
2682 		VNET_LIST_RUNLOCK();
2683 	}
2684 
2685 	pf_end_threads++;
2686 	sx_xunlock(&pf_end_lock);
2687 	kproc_exit(0);
2688 }
2689 
2690 void
2691 pf_unload_vnet_purge(void)
2692 {
2693 
2694 	/*
2695 	 * To cleanse up all kifs and rules we need
2696 	 * two runs: first one clears reference flags,
2697 	 * then pf_purge_expired_states() doesn't
2698 	 * raise them, and then second run frees.
2699 	 */
2700 	pf_purge_unlinked_rules();
2701 	pfi_kkif_purge();
2702 
2703 	/*
2704 	 * Now purge everything.
2705 	 */
2706 	pf_purge_expired_states(0, V_pf_hashmask);
2707 	pf_purge_fragments(UINT_MAX);
2708 	pf_purge_expired_src_nodes();
2709 
2710 	/*
2711 	 * Now all kifs & rules should be unreferenced,
2712 	 * thus should be successfully freed.
2713 	 */
2714 	pf_purge_unlinked_rules();
2715 	pfi_kkif_purge();
2716 }
2717 
2718 u_int32_t
2719 pf_state_expires(const struct pf_kstate *state)
2720 {
2721 	u_int32_t	timeout;
2722 	u_int32_t	start;
2723 	u_int32_t	end;
2724 	u_int32_t	states;
2725 
2726 	/* handle all PFTM_* > PFTM_MAX here */
2727 	if (state->timeout == PFTM_PURGE)
2728 		return (time_uptime);
2729 	KASSERT(state->timeout != PFTM_UNLINKED,
2730 	    ("pf_state_expires: timeout == PFTM_UNLINKED"));
2731 	KASSERT((state->timeout < PFTM_MAX),
2732 	    ("pf_state_expires: timeout > PFTM_MAX"));
2733 	timeout = state->rule->timeout[state->timeout];
2734 	if (!timeout)
2735 		timeout = V_pf_default_rule.timeout[state->timeout];
2736 	start = state->rule->timeout[PFTM_ADAPTIVE_START];
2737 	if (start && state->rule != &V_pf_default_rule) {
2738 		end = state->rule->timeout[PFTM_ADAPTIVE_END];
2739 		states = counter_u64_fetch(state->rule->states_cur);
2740 	} else {
2741 		start = V_pf_default_rule.timeout[PFTM_ADAPTIVE_START];
2742 		end = V_pf_default_rule.timeout[PFTM_ADAPTIVE_END];
2743 		states = V_pf_status.states;
2744 	}
2745 	if (end && states > start && start < end) {
2746 		if (states < end) {
2747 			timeout = (u_int64_t)timeout * (end - states) /
2748 			    (end - start);
2749 			return ((state->expire / 1000) + timeout);
2750 		}
2751 		else
2752 			return (time_uptime);
2753 	}
2754 	return ((state->expire / 1000) + timeout);
2755 }
2756 
2757 void
2758 pf_purge_expired_src_nodes(void)
2759 {
2760 	struct pf_ksrc_node_list	 freelist;
2761 	struct pf_srchash	*sh;
2762 	struct pf_ksrc_node	*cur, *next;
2763 	int i;
2764 
2765 	LIST_INIT(&freelist);
2766 	for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) {
2767 	    PF_HASHROW_LOCK(sh);
2768 	    LIST_FOREACH_SAFE(cur, &sh->nodes, entry, next)
2769 		if (cur->states == 0 && cur->expire <= time_uptime) {
2770 			pf_unlink_src_node(cur);
2771 			LIST_INSERT_HEAD(&freelist, cur, entry);
2772 		} else if (cur->rule != NULL)
2773 			cur->rule->rule_ref |= PFRULE_REFS;
2774 	    PF_HASHROW_UNLOCK(sh);
2775 	}
2776 
2777 	pf_free_src_nodes(&freelist);
2778 
2779 	V_pf_status.src_nodes = uma_zone_get_cur(V_pf_sources_z);
2780 }
2781 
2782 static void
2783 pf_src_tree_remove_state(struct pf_kstate *s)
2784 {
2785 	uint32_t timeout;
2786 
2787 	timeout = s->rule->timeout[PFTM_SRC_NODE] ?
2788 	    s->rule->timeout[PFTM_SRC_NODE] :
2789 	    V_pf_default_rule.timeout[PFTM_SRC_NODE];
2790 
2791 	for (pf_sn_types_t sn_type=0; sn_type<PF_SN_MAX; sn_type++) {
2792 		if (s->sns[sn_type] == NULL)
2793 			continue;
2794 		PF_SRC_NODE_LOCK(s->sns[sn_type]);
2795 		if (sn_type == PF_SN_LIMIT && s->src.tcp_est)
2796 			--(s->sns[sn_type]->conn);
2797 		if (--(s->sns[sn_type]->states) == 0)
2798 			s->sns[sn_type]->expire = time_uptime + timeout;
2799 		PF_SRC_NODE_UNLOCK(s->sns[sn_type]);
2800 		s->sns[sn_type] = NULL;
2801 	}
2802 
2803 }
2804 
2805 /*
2806  * Unlink and potentilly free a state. Function may be
2807  * called with ID hash row locked, but always returns
2808  * unlocked, since it needs to go through key hash locking.
2809  */
2810 int
2811 pf_remove_state(struct pf_kstate *s)
2812 {
2813 	struct pf_idhash *ih = &V_pf_idhash[PF_IDHASH(s)];
2814 
2815 	NET_EPOCH_ASSERT();
2816 	PF_HASHROW_ASSERT(ih);
2817 
2818 	if (s->timeout == PFTM_UNLINKED) {
2819 		/*
2820 		 * State is being processed
2821 		 * by pf_remove_state() in
2822 		 * an other thread.
2823 		 */
2824 		PF_HASHROW_UNLOCK(ih);
2825 		return (0);	/* XXXGL: undefined actually */
2826 	}
2827 
2828 	if (s->src.state == PF_TCPS_PROXY_DST) {
2829 		/* XXX wire key the right one? */
2830 		pf_send_tcp(s->rule, s->key[PF_SK_WIRE]->af,
2831 		    &s->key[PF_SK_WIRE]->addr[1],
2832 		    &s->key[PF_SK_WIRE]->addr[0],
2833 		    s->key[PF_SK_WIRE]->port[1],
2834 		    s->key[PF_SK_WIRE]->port[0],
2835 		    s->src.seqhi, s->src.seqlo + 1,
2836 		    TH_RST|TH_ACK, 0, 0, 0, M_SKIP_FIREWALL, s->tag, 0,
2837 		    s->act.rtableid);
2838 	}
2839 
2840 	LIST_REMOVE(s, entry);
2841 	pf_src_tree_remove_state(s);
2842 
2843 	if (V_pfsync_delete_state_ptr != NULL)
2844 		V_pfsync_delete_state_ptr(s);
2845 
2846 	STATE_DEC_COUNTERS(s);
2847 
2848 	s->timeout = PFTM_UNLINKED;
2849 
2850 	/* Ensure we remove it from the list of halfopen states, if needed. */
2851 	if (s->key[PF_SK_STACK] != NULL &&
2852 	    s->key[PF_SK_STACK]->proto == IPPROTO_TCP)
2853 		pf_set_protostate(s, PF_PEER_BOTH, TCPS_CLOSED);
2854 
2855 	PF_HASHROW_UNLOCK(ih);
2856 
2857 	pf_detach_state(s);
2858 
2859 	pf_udp_mapping_release(s->udp_mapping);
2860 
2861 	/* pf_state_insert() initialises refs to 2 */
2862 	return (pf_release_staten(s, 2));
2863 }
2864 
2865 struct pf_kstate *
2866 pf_alloc_state(int flags)
2867 {
2868 
2869 	return (uma_zalloc(V_pf_state_z, flags | M_ZERO));
2870 }
2871 
2872 void
2873 pf_free_state(struct pf_kstate *cur)
2874 {
2875 	struct pf_krule_item *ri;
2876 
2877 	KASSERT(cur->refs == 0, ("%s: %p has refs", __func__, cur));
2878 	KASSERT(cur->timeout == PFTM_UNLINKED, ("%s: timeout %u", __func__,
2879 	    cur->timeout));
2880 
2881 	while ((ri = SLIST_FIRST(&cur->match_rules))) {
2882 		SLIST_REMOVE_HEAD(&cur->match_rules, entry);
2883 		free(ri, M_PF_RULE_ITEM);
2884 	}
2885 
2886 	pf_normalize_tcp_cleanup(cur);
2887 	uma_zfree(V_pf_state_z, cur);
2888 	pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_REMOVALS], 1);
2889 }
2890 
2891 /*
2892  * Called only from pf_purge_thread(), thus serialized.
2893  */
2894 static u_int
2895 pf_purge_expired_states(u_int i, int maxcheck)
2896 {
2897 	struct pf_idhash *ih;
2898 	struct pf_kstate *s;
2899 	struct pf_krule_item *mrm;
2900 	size_t count __unused;
2901 
2902 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2903 
2904 	/*
2905 	 * Go through hash and unlink states that expire now.
2906 	 */
2907 	while (maxcheck > 0) {
2908 		count = 0;
2909 		ih = &V_pf_idhash[i];
2910 
2911 		/* only take the lock if we expect to do work */
2912 		if (!LIST_EMPTY(&ih->states)) {
2913 relock:
2914 			PF_HASHROW_LOCK(ih);
2915 			LIST_FOREACH(s, &ih->states, entry) {
2916 				if (pf_state_expires(s) <= time_uptime) {
2917 					V_pf_status.states -=
2918 					    pf_remove_state(s);
2919 					goto relock;
2920 				}
2921 				s->rule->rule_ref |= PFRULE_REFS;
2922 				if (s->nat_rule != NULL)
2923 					s->nat_rule->rule_ref |= PFRULE_REFS;
2924 				if (s->anchor != NULL)
2925 					s->anchor->rule_ref |= PFRULE_REFS;
2926 				s->kif->pfik_flags |= PFI_IFLAG_REFS;
2927 				SLIST_FOREACH(mrm, &s->match_rules, entry)
2928 					mrm->r->rule_ref |= PFRULE_REFS;
2929 				if (s->act.rt_kif)
2930 					s->act.rt_kif->pfik_flags |= PFI_IFLAG_REFS;
2931 				count++;
2932 			}
2933 			PF_HASHROW_UNLOCK(ih);
2934 		}
2935 
2936 		SDT_PROBE2(pf, purge, state, rowcount, i, count);
2937 
2938 		/* Return when we hit end of hash. */
2939 		if (++i > V_pf_hashmask) {
2940 			V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2941 			return (0);
2942 		}
2943 
2944 		maxcheck--;
2945 	}
2946 
2947 	V_pf_status.states = uma_zone_get_cur(V_pf_state_z);
2948 
2949 	return (i);
2950 }
2951 
2952 static void
2953 pf_purge_unlinked_rules(void)
2954 {
2955 	struct pf_krulequeue tmpq;
2956 	struct pf_krule *r, *r1;
2957 
2958 	/*
2959 	 * If we have overloading task pending, then we'd
2960 	 * better skip purging this time. There is a tiny
2961 	 * probability that overloading task references
2962 	 * an already unlinked rule.
2963 	 */
2964 	PF_OVERLOADQ_LOCK();
2965 	if (!SLIST_EMPTY(&V_pf_overloadqueue)) {
2966 		PF_OVERLOADQ_UNLOCK();
2967 		return;
2968 	}
2969 	PF_OVERLOADQ_UNLOCK();
2970 
2971 	/*
2972 	 * Do naive mark-and-sweep garbage collecting of old rules.
2973 	 * Reference flag is raised by pf_purge_expired_states()
2974 	 * and pf_purge_expired_src_nodes().
2975 	 *
2976 	 * To avoid LOR between PF_UNLNKDRULES_LOCK/PF_RULES_WLOCK,
2977 	 * use a temporary queue.
2978 	 */
2979 	TAILQ_INIT(&tmpq);
2980 	PF_UNLNKDRULES_LOCK();
2981 	TAILQ_FOREACH_SAFE(r, &V_pf_unlinked_rules, entries, r1) {
2982 		if (!(r->rule_ref & PFRULE_REFS)) {
2983 			TAILQ_REMOVE(&V_pf_unlinked_rules, r, entries);
2984 			TAILQ_INSERT_TAIL(&tmpq, r, entries);
2985 		} else
2986 			r->rule_ref &= ~PFRULE_REFS;
2987 	}
2988 	PF_UNLNKDRULES_UNLOCK();
2989 
2990 	if (!TAILQ_EMPTY(&tmpq)) {
2991 		PF_CONFIG_LOCK();
2992 		PF_RULES_WLOCK();
2993 		TAILQ_FOREACH_SAFE(r, &tmpq, entries, r1) {
2994 			TAILQ_REMOVE(&tmpq, r, entries);
2995 			pf_free_rule(r);
2996 		}
2997 		PF_RULES_WUNLOCK();
2998 		PF_CONFIG_UNLOCK();
2999 	}
3000 }
3001 
3002 void
3003 pf_print_host(struct pf_addr *addr, u_int16_t p, sa_family_t af)
3004 {
3005 	switch (af) {
3006 #ifdef INET
3007 	case AF_INET: {
3008 		u_int32_t a = ntohl(addr->addr32[0]);
3009 		printf("%u.%u.%u.%u", (a>>24)&255, (a>>16)&255,
3010 		    (a>>8)&255, a&255);
3011 		if (p) {
3012 			p = ntohs(p);
3013 			printf(":%u", p);
3014 		}
3015 		break;
3016 	}
3017 #endif /* INET */
3018 #ifdef INET6
3019 	case AF_INET6: {
3020 		u_int16_t b;
3021 		u_int8_t i, curstart, curend, maxstart, maxend;
3022 		curstart = curend = maxstart = maxend = 255;
3023 		for (i = 0; i < 8; i++) {
3024 			if (!addr->addr16[i]) {
3025 				if (curstart == 255)
3026 					curstart = i;
3027 				curend = i;
3028 			} else {
3029 				if ((curend - curstart) >
3030 				    (maxend - maxstart)) {
3031 					maxstart = curstart;
3032 					maxend = curend;
3033 				}
3034 				curstart = curend = 255;
3035 			}
3036 		}
3037 		if ((curend - curstart) >
3038 		    (maxend - maxstart)) {
3039 			maxstart = curstart;
3040 			maxend = curend;
3041 		}
3042 		for (i = 0; i < 8; i++) {
3043 			if (i >= maxstart && i <= maxend) {
3044 				if (i == 0)
3045 					printf(":");
3046 				if (i == maxend)
3047 					printf(":");
3048 			} else {
3049 				b = ntohs(addr->addr16[i]);
3050 				printf("%x", b);
3051 				if (i < 7)
3052 					printf(":");
3053 			}
3054 		}
3055 		if (p) {
3056 			p = ntohs(p);
3057 			printf("[%u]", p);
3058 		}
3059 		break;
3060 	}
3061 #endif /* INET6 */
3062 	default:
3063 		unhandled_af(af);
3064 	}
3065 }
3066 
3067 void
3068 pf_print_state(struct pf_kstate *s)
3069 {
3070 	pf_print_state_parts(s, NULL, NULL);
3071 }
3072 
3073 static void
3074 pf_print_state_parts(struct pf_kstate *s,
3075     struct pf_state_key *skwp, struct pf_state_key *sksp)
3076 {
3077 	struct pf_state_key *skw, *sks;
3078 	u_int8_t proto, dir;
3079 
3080 	/* Do our best to fill these, but they're skipped if NULL */
3081 	skw = skwp ? skwp : (s ? s->key[PF_SK_WIRE] : NULL);
3082 	sks = sksp ? sksp : (s ? s->key[PF_SK_STACK] : NULL);
3083 	proto = skw ? skw->proto : (sks ? sks->proto : 0);
3084 	dir = s ? s->direction : 0;
3085 
3086 	switch (proto) {
3087 	case IPPROTO_IPV4:
3088 		printf("IPv4");
3089 		break;
3090 	case IPPROTO_IPV6:
3091 		printf("IPv6");
3092 		break;
3093 	case IPPROTO_TCP:
3094 		printf("TCP");
3095 		break;
3096 	case IPPROTO_UDP:
3097 		printf("UDP");
3098 		break;
3099 	case IPPROTO_ICMP:
3100 		printf("ICMP");
3101 		break;
3102 	case IPPROTO_ICMPV6:
3103 		printf("ICMPv6");
3104 		break;
3105 	default:
3106 		printf("%u", proto);
3107 		break;
3108 	}
3109 	switch (dir) {
3110 	case PF_IN:
3111 		printf(" in");
3112 		break;
3113 	case PF_OUT:
3114 		printf(" out");
3115 		break;
3116 	}
3117 	if (skw) {
3118 		printf(" wire: ");
3119 		pf_print_host(&skw->addr[0], skw->port[0], skw->af);
3120 		printf(" ");
3121 		pf_print_host(&skw->addr[1], skw->port[1], skw->af);
3122 	}
3123 	if (sks) {
3124 		printf(" stack: ");
3125 		if (sks != skw) {
3126 			pf_print_host(&sks->addr[0], sks->port[0], sks->af);
3127 			printf(" ");
3128 			pf_print_host(&sks->addr[1], sks->port[1], sks->af);
3129 		} else
3130 			printf("-");
3131 	}
3132 	if (s) {
3133 		if (proto == IPPROTO_TCP) {
3134 			printf(" [lo=%u high=%u win=%u modulator=%u",
3135 			    s->src.seqlo, s->src.seqhi,
3136 			    s->src.max_win, s->src.seqdiff);
3137 			if (s->src.wscale && s->dst.wscale)
3138 				printf(" wscale=%u",
3139 				    s->src.wscale & PF_WSCALE_MASK);
3140 			printf("]");
3141 			printf(" [lo=%u high=%u win=%u modulator=%u",
3142 			    s->dst.seqlo, s->dst.seqhi,
3143 			    s->dst.max_win, s->dst.seqdiff);
3144 			if (s->src.wscale && s->dst.wscale)
3145 				printf(" wscale=%u",
3146 				s->dst.wscale & PF_WSCALE_MASK);
3147 			printf("]");
3148 		}
3149 		printf(" %u:%u", s->src.state, s->dst.state);
3150 		if (s->rule)
3151 			printf(" @%d", s->rule->nr);
3152 	}
3153 }
3154 
3155 void
3156 pf_print_flags(uint16_t f)
3157 {
3158 	if (f)
3159 		printf(" ");
3160 	if (f & TH_FIN)
3161 		printf("F");
3162 	if (f & TH_SYN)
3163 		printf("S");
3164 	if (f & TH_RST)
3165 		printf("R");
3166 	if (f & TH_PUSH)
3167 		printf("P");
3168 	if (f & TH_ACK)
3169 		printf("A");
3170 	if (f & TH_URG)
3171 		printf("U");
3172 	if (f & TH_ECE)
3173 		printf("E");
3174 	if (f & TH_CWR)
3175 		printf("W");
3176 	if (f & TH_AE)
3177 		printf("e");
3178 }
3179 
3180 #define	PF_SET_SKIP_STEPS(i)					\
3181 	do {							\
3182 		while (head[i] != cur) {			\
3183 			head[i]->skip[i] = cur;			\
3184 			head[i] = TAILQ_NEXT(head[i], entries);	\
3185 		}						\
3186 	} while (0)
3187 
3188 void
3189 pf_calc_skip_steps(struct pf_krulequeue *rules)
3190 {
3191 	struct pf_krule *cur, *prev, *head[PF_SKIP_COUNT];
3192 	int i;
3193 
3194 	cur = TAILQ_FIRST(rules);
3195 	prev = cur;
3196 	for (i = 0; i < PF_SKIP_COUNT; ++i)
3197 		head[i] = cur;
3198 	while (cur != NULL) {
3199 		if (cur->kif != prev->kif || cur->ifnot != prev->ifnot)
3200 			PF_SET_SKIP_STEPS(PF_SKIP_IFP);
3201 		if (cur->direction != prev->direction)
3202 			PF_SET_SKIP_STEPS(PF_SKIP_DIR);
3203 		if (cur->af != prev->af)
3204 			PF_SET_SKIP_STEPS(PF_SKIP_AF);
3205 		if (cur->proto != prev->proto)
3206 			PF_SET_SKIP_STEPS(PF_SKIP_PROTO);
3207 		if (cur->src.neg != prev->src.neg ||
3208 		    pf_addr_wrap_neq(&cur->src.addr, &prev->src.addr))
3209 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_ADDR);
3210 		if (cur->dst.neg != prev->dst.neg ||
3211 		    pf_addr_wrap_neq(&cur->dst.addr, &prev->dst.addr))
3212 			PF_SET_SKIP_STEPS(PF_SKIP_DST_ADDR);
3213 		if (cur->src.port[0] != prev->src.port[0] ||
3214 		    cur->src.port[1] != prev->src.port[1] ||
3215 		    cur->src.port_op != prev->src.port_op)
3216 			PF_SET_SKIP_STEPS(PF_SKIP_SRC_PORT);
3217 		if (cur->dst.port[0] != prev->dst.port[0] ||
3218 		    cur->dst.port[1] != prev->dst.port[1] ||
3219 		    cur->dst.port_op != prev->dst.port_op)
3220 			PF_SET_SKIP_STEPS(PF_SKIP_DST_PORT);
3221 
3222 		prev = cur;
3223 		cur = TAILQ_NEXT(cur, entries);
3224 	}
3225 	for (i = 0; i < PF_SKIP_COUNT; ++i)
3226 		PF_SET_SKIP_STEPS(i);
3227 }
3228 
3229 int
3230 pf_addr_wrap_neq(struct pf_addr_wrap *aw1, struct pf_addr_wrap *aw2)
3231 {
3232 	if (aw1->type != aw2->type)
3233 		return (1);
3234 	switch (aw1->type) {
3235 	case PF_ADDR_ADDRMASK:
3236 	case PF_ADDR_RANGE:
3237 		if (PF_ANEQ(&aw1->v.a.addr, &aw2->v.a.addr, AF_INET6))
3238 			return (1);
3239 		if (PF_ANEQ(&aw1->v.a.mask, &aw2->v.a.mask, AF_INET6))
3240 			return (1);
3241 		return (0);
3242 	case PF_ADDR_DYNIFTL:
3243 		return (aw1->p.dyn->pfid_kt != aw2->p.dyn->pfid_kt);
3244 	case PF_ADDR_NONE:
3245 	case PF_ADDR_NOROUTE:
3246 	case PF_ADDR_URPFFAILED:
3247 		return (0);
3248 	case PF_ADDR_TABLE:
3249 		return (aw1->p.tbl != aw2->p.tbl);
3250 	default:
3251 		printf("invalid address type: %d\n", aw1->type);
3252 		return (1);
3253 	}
3254 }
3255 
3256 /**
3257  * Checksum updates are a little complicated because the checksum in the TCP/UDP
3258  * header isn't always a full checksum. In some cases (i.e. output) it's a
3259  * pseudo-header checksum, which is a partial checksum over src/dst IP
3260  * addresses, protocol number and length.
3261  *
3262  * That means we have the following cases:
3263  *  * Input or forwarding: we don't have TSO, the checksum fields are full
3264  *  	checksums, we need to update the checksum whenever we change anything.
3265  *  * Output (i.e. the checksum is a pseudo-header checksum):
3266  *  	x The field being updated is src/dst address or affects the length of
3267  *  	the packet. We need to update the pseudo-header checksum (note that this
3268  *  	checksum is not ones' complement).
3269  *  	x Some other field is being modified (e.g. src/dst port numbers): We
3270  *  	don't have to update anything.
3271  **/
3272 u_int16_t
3273 pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp)
3274 {
3275 	u_int32_t x;
3276 
3277 	x = cksum + old - new;
3278 	x = (x + (x >> 16)) & 0xffff;
3279 
3280 	/* optimise: eliminate a branch when not udp */
3281 	if (udp && cksum == 0x0000)
3282 		return cksum;
3283 	if (udp && x == 0x0000)
3284 		x = 0xffff;
3285 
3286 	return (u_int16_t)(x);
3287 }
3288 
3289 static int
3290 pf_patch_8(struct pf_pdesc *pd, u_int8_t *f, u_int8_t v, bool hi)
3291 {
3292 	int	 rewrite = 0;
3293 
3294 	if (*f != v) {
3295 		uint16_t old = htons(hi ? (*f << 8) : *f);
3296 		uint16_t new = htons(hi ? ( v << 8) :  v);
3297 
3298 		*f = v;
3299 
3300 		if (! (pd->m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA |
3301 		    CSUM_DELAY_DATA_IPV6)))
3302 			*pd->pcksum = pf_cksum_fixup(*pd->pcksum, old, new,
3303 			    pd->proto == IPPROTO_UDP);
3304 
3305 		rewrite = 1;
3306 	}
3307 
3308 	return (rewrite);
3309 }
3310 
3311 int
3312 pf_patch_16(struct pf_pdesc *pd, void *f, u_int16_t v, bool hi)
3313 {
3314 	int rewrite = 0;
3315 	u_int8_t *fb = (u_int8_t *)f;
3316 	u_int8_t *vb = (u_int8_t *)&v;
3317 
3318 	rewrite += pf_patch_8(pd, fb++, *vb++, hi);
3319 	rewrite += pf_patch_8(pd, fb++, *vb++, !hi);
3320 
3321 	return (rewrite);
3322 }
3323 
3324 int
3325 pf_patch_32(struct pf_pdesc *pd, void *f, u_int32_t v, bool hi)
3326 {
3327 	int rewrite = 0;
3328 	u_int8_t *fb = (u_int8_t *)f;
3329 	u_int8_t *vb = (u_int8_t *)&v;
3330 
3331 	rewrite += pf_patch_8(pd, fb++, *vb++, hi);
3332 	rewrite += pf_patch_8(pd, fb++, *vb++, !hi);
3333 	rewrite += pf_patch_8(pd, fb++, *vb++, hi);
3334 	rewrite += pf_patch_8(pd, fb++, *vb++, !hi);
3335 
3336 	return (rewrite);
3337 }
3338 
3339 u_int16_t
3340 pf_proto_cksum_fixup(struct mbuf *m, u_int16_t cksum, u_int16_t old,
3341         u_int16_t new, u_int8_t udp)
3342 {
3343 	if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3344 		return (cksum);
3345 
3346 	return (pf_cksum_fixup(cksum, old, new, udp));
3347 }
3348 
3349 static void
3350 pf_change_ap(struct pf_pdesc *pd, struct pf_addr *a, u_int16_t *p,
3351         struct pf_addr *an, u_int16_t pn)
3352 {
3353 	struct pf_addr	ao;
3354 	u_int16_t	po;
3355 	uint8_t		u = pd->virtual_proto == IPPROTO_UDP;
3356 
3357 	MPASS(pd->pcksum);
3358 	if (pd->af == AF_INET) {
3359 		MPASS(pd->ip_sum);
3360 	}
3361 
3362 	pf_addrcpy(&ao, a, pd->af);
3363 	if (pd->af == pd->naf)
3364 		pf_addrcpy(a, an, pd->af);
3365 
3366 	if (pd->m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6))
3367 		*pd->pcksum = ~*pd->pcksum;
3368 
3369 	if (p == NULL)  /* no port -> done. no cksum to worry about. */
3370 		return;
3371 	po = *p;
3372 	*p = pn;
3373 
3374 	switch (pd->af) {
3375 #ifdef INET
3376 	case AF_INET:
3377 		switch (pd->naf) {
3378 		case AF_INET:
3379 			*pd->ip_sum = pf_cksum_fixup(pf_cksum_fixup(*pd->ip_sum,
3380 			    ao.addr16[0], an->addr16[0], 0),
3381 			    ao.addr16[1], an->addr16[1], 0);
3382 			*p = pn;
3383 
3384 			*pd->pcksum = pf_cksum_fixup(pf_cksum_fixup(*pd->pcksum,
3385 			    ao.addr16[0], an->addr16[0], u),
3386 			    ao.addr16[1], an->addr16[1], u);
3387 
3388 			*pd->pcksum = pf_proto_cksum_fixup(pd->m, *pd->pcksum, po, pn, u);
3389 			break;
3390 #ifdef INET6
3391 		case AF_INET6:
3392 			*pd->pcksum = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3393 			   pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3394 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(*pd->pcksum,
3395 			    ao.addr16[0], an->addr16[0], u),
3396 			    ao.addr16[1], an->addr16[1], u),
3397 			    0,            an->addr16[2], u),
3398 			    0,            an->addr16[3], u),
3399 			    0,            an->addr16[4], u),
3400 			    0,            an->addr16[5], u),
3401 			    0,            an->addr16[6], u),
3402 			    0,            an->addr16[7], u),
3403 			    po, pn, u);
3404 			break;
3405 #endif /* INET6 */
3406 		default:
3407 			unhandled_af(pd->naf);
3408 		}
3409 		break;
3410 #endif /* INET */
3411 #ifdef INET6
3412 	case AF_INET6:
3413 		switch (pd->naf) {
3414 #ifdef INET
3415 		case AF_INET:
3416 			*pd->pcksum = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3417 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3418 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(*pd->pcksum,
3419 			    ao.addr16[0], an->addr16[0], u),
3420 			    ao.addr16[1], an->addr16[1], u),
3421 			    ao.addr16[2], 0,             u),
3422 			    ao.addr16[3], 0,             u),
3423 			    ao.addr16[4], 0,             u),
3424 			    ao.addr16[5], 0,             u),
3425 			    ao.addr16[6], 0,             u),
3426 			    ao.addr16[7], 0,             u),
3427 			    po, pn, u);
3428 			break;
3429 #endif /* INET */
3430 		case AF_INET6:
3431 			*pd->pcksum  = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3432 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3433 			    pf_cksum_fixup(pf_cksum_fixup(*pd->pcksum,
3434 			    ao.addr16[0], an->addr16[0], u),
3435 			    ao.addr16[1], an->addr16[1], u),
3436 			    ao.addr16[2], an->addr16[2], u),
3437 			    ao.addr16[3], an->addr16[3], u),
3438 			    ao.addr16[4], an->addr16[4], u),
3439 			    ao.addr16[5], an->addr16[5], u),
3440 			    ao.addr16[6], an->addr16[6], u),
3441 			    ao.addr16[7], an->addr16[7], u);
3442 
3443 			*pd->pcksum = pf_proto_cksum_fixup(pd->m, *pd->pcksum, po, pn, u);
3444 			break;
3445 		default:
3446 			unhandled_af(pd->naf);
3447 		}
3448 		break;
3449 #endif /* INET6 */
3450 	default:
3451 		unhandled_af(pd->af);
3452 	}
3453 
3454 	if (pd->m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA |
3455 	    CSUM_DELAY_DATA_IPV6)) {
3456 		*pd->pcksum = ~*pd->pcksum;
3457 		if (! *pd->pcksum)
3458 			*pd->pcksum = 0xffff;
3459 	}
3460 }
3461 
3462 /* Changes a u_int32_t.  Uses a void * so there are no align restrictions */
3463 void
3464 pf_change_a(void *a, u_int16_t *c, u_int32_t an, u_int8_t u)
3465 {
3466 	u_int32_t	ao;
3467 
3468 	memcpy(&ao, a, sizeof(ao));
3469 	memcpy(a, &an, sizeof(u_int32_t));
3470 	*c = pf_cksum_fixup(pf_cksum_fixup(*c, ao / 65536, an / 65536, u),
3471 	    ao % 65536, an % 65536, u);
3472 }
3473 
3474 void
3475 pf_change_proto_a(struct mbuf *m, void *a, u_int16_t *c, u_int32_t an, u_int8_t udp)
3476 {
3477 	u_int32_t	ao;
3478 
3479 	memcpy(&ao, a, sizeof(ao));
3480 	memcpy(a, &an, sizeof(u_int32_t));
3481 
3482 	*c = pf_proto_cksum_fixup(m,
3483 	    pf_proto_cksum_fixup(m, *c, ao / 65536, an / 65536, udp),
3484 	    ao % 65536, an % 65536, udp);
3485 }
3486 
3487 #ifdef INET6
3488 static void
3489 pf_change_a6(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u)
3490 {
3491 	struct pf_addr	ao;
3492 
3493 	pf_addrcpy(&ao, a, AF_INET6);
3494 	pf_addrcpy(a, an, AF_INET6);
3495 
3496 	*c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3497 	    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3498 	    pf_cksum_fixup(pf_cksum_fixup(*c,
3499 	    ao.addr16[0], an->addr16[0], u),
3500 	    ao.addr16[1], an->addr16[1], u),
3501 	    ao.addr16[2], an->addr16[2], u),
3502 	    ao.addr16[3], an->addr16[3], u),
3503 	    ao.addr16[4], an->addr16[4], u),
3504 	    ao.addr16[5], an->addr16[5], u),
3505 	    ao.addr16[6], an->addr16[6], u),
3506 	    ao.addr16[7], an->addr16[7], u);
3507 }
3508 #endif /* INET6 */
3509 
3510 static void
3511 pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa,
3512     struct pf_addr *na, u_int16_t np, u_int16_t *pc, u_int16_t *h2c,
3513     u_int16_t *ic, u_int16_t *hc, u_int8_t u, sa_family_t af)
3514 {
3515 	struct pf_addr	oia, ooa;
3516 
3517 	pf_addrcpy(&oia, ia, af);
3518 	if (oa)
3519 		pf_addrcpy(&ooa, oa, af);
3520 
3521 	/* Change inner protocol port, fix inner protocol checksum. */
3522 	if (ip != NULL) {
3523 		u_int16_t	oip = *ip;
3524 		u_int32_t	opc;
3525 
3526 		if (pc != NULL)
3527 			opc = *pc;
3528 		*ip = np;
3529 		if (pc != NULL)
3530 			*pc = pf_cksum_fixup(*pc, oip, *ip, u);
3531 		*ic = pf_cksum_fixup(*ic, oip, *ip, 0);
3532 		if (pc != NULL)
3533 			*ic = pf_cksum_fixup(*ic, opc, *pc, 0);
3534 	}
3535 	/* Change inner ip address, fix inner ip and icmp checksums. */
3536 	pf_addrcpy(ia, na, af);
3537 	switch (af) {
3538 #ifdef INET
3539 	case AF_INET: {
3540 		u_int32_t	 oh2c = *h2c;
3541 
3542 		*h2c = pf_cksum_fixup(pf_cksum_fixup(*h2c,
3543 		    oia.addr16[0], ia->addr16[0], 0),
3544 		    oia.addr16[1], ia->addr16[1], 0);
3545 		*ic = pf_cksum_fixup(pf_cksum_fixup(*ic,
3546 		    oia.addr16[0], ia->addr16[0], 0),
3547 		    oia.addr16[1], ia->addr16[1], 0);
3548 		*ic = pf_cksum_fixup(*ic, oh2c, *h2c, 0);
3549 		break;
3550 	}
3551 #endif /* INET */
3552 #ifdef INET6
3553 	case AF_INET6:
3554 		*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3555 		    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3556 		    pf_cksum_fixup(pf_cksum_fixup(*ic,
3557 		    oia.addr16[0], ia->addr16[0], u),
3558 		    oia.addr16[1], ia->addr16[1], u),
3559 		    oia.addr16[2], ia->addr16[2], u),
3560 		    oia.addr16[3], ia->addr16[3], u),
3561 		    oia.addr16[4], ia->addr16[4], u),
3562 		    oia.addr16[5], ia->addr16[5], u),
3563 		    oia.addr16[6], ia->addr16[6], u),
3564 		    oia.addr16[7], ia->addr16[7], u);
3565 		break;
3566 #endif /* INET6 */
3567 	}
3568 	/* Outer ip address, fix outer ip or icmpv6 checksum, if necessary. */
3569 	if (oa) {
3570 		pf_addrcpy(oa, na, af);
3571 		switch (af) {
3572 #ifdef INET
3573 		case AF_INET:
3574 			*hc = pf_cksum_fixup(pf_cksum_fixup(*hc,
3575 			    ooa.addr16[0], oa->addr16[0], 0),
3576 			    ooa.addr16[1], oa->addr16[1], 0);
3577 			break;
3578 #endif /* INET */
3579 #ifdef INET6
3580 		case AF_INET6:
3581 			*ic = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3582 			    pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup(
3583 			    pf_cksum_fixup(pf_cksum_fixup(*ic,
3584 			    ooa.addr16[0], oa->addr16[0], u),
3585 			    ooa.addr16[1], oa->addr16[1], u),
3586 			    ooa.addr16[2], oa->addr16[2], u),
3587 			    ooa.addr16[3], oa->addr16[3], u),
3588 			    ooa.addr16[4], oa->addr16[4], u),
3589 			    ooa.addr16[5], oa->addr16[5], u),
3590 			    ooa.addr16[6], oa->addr16[6], u),
3591 			    ooa.addr16[7], oa->addr16[7], u);
3592 			break;
3593 #endif /* INET6 */
3594 		}
3595 	}
3596 }
3597 
3598 int
3599 pf_translate_af(struct pf_pdesc *pd)
3600 {
3601 #if defined(INET) && defined(INET6)
3602 	struct mbuf		*mp;
3603 	struct ip		*ip4;
3604 	struct ip6_hdr		*ip6;
3605 	struct icmp6_hdr	*icmp;
3606 	struct m_tag		*mtag;
3607 	struct pf_fragment_tag	*ftag;
3608 	int			 hlen;
3609 
3610 	hlen = pd->naf == AF_INET ? sizeof(*ip4) : sizeof(*ip6);
3611 
3612 	/* trim the old header */
3613 	m_adj(pd->m, pd->off);
3614 
3615 	/* prepend a new one */
3616 	M_PREPEND(pd->m, hlen, M_NOWAIT);
3617 	if (pd->m == NULL)
3618 		return (-1);
3619 
3620 	switch (pd->naf) {
3621 	case AF_INET:
3622 		ip4 = mtod(pd->m, struct ip *);
3623 		bzero(ip4, hlen);
3624 		ip4->ip_v = IPVERSION;
3625 		ip4->ip_hl = hlen >> 2;
3626 		ip4->ip_tos = pd->tos;
3627 		ip4->ip_len = htons(hlen + (pd->tot_len - pd->off));
3628 		ip_fillid(ip4, V_ip_random_id);
3629 		ip4->ip_ttl = pd->ttl;
3630 		ip4->ip_p = pd->proto;
3631 		ip4->ip_src = pd->nsaddr.v4;
3632 		ip4->ip_dst = pd->ndaddr.v4;
3633 		pd->src = (struct pf_addr *)&ip4->ip_src;
3634 		pd->dst = (struct pf_addr *)&ip4->ip_dst;
3635 		pd->off = sizeof(struct ip);
3636 		if (pd->m->m_pkthdr.csum_flags & CSUM_TCP_IPV6) {
3637 			pd->m->m_pkthdr.csum_flags &= ~CSUM_TCP_IPV6;
3638 			pd->m->m_pkthdr.csum_flags |= CSUM_TCP;
3639 		}
3640 		if (pd->m->m_pkthdr.csum_flags & CSUM_UDP_IPV6) {
3641 			pd->m->m_pkthdr.csum_flags &= ~CSUM_UDP_IPV6;
3642 			pd->m->m_pkthdr.csum_flags |= CSUM_UDP;
3643 		}
3644 		if (pd->m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6) {
3645 			pd->m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
3646 			pd->m->m_pkthdr.csum_flags |= CSUM_SCTP;
3647 		}
3648 		break;
3649 	case AF_INET6:
3650 		ip6 = mtod(pd->m, struct ip6_hdr *);
3651 		bzero(ip6, hlen);
3652 		ip6->ip6_vfc = IPV6_VERSION;
3653 		ip6->ip6_flow |= htonl((u_int32_t)pd->tos << 20);
3654 		ip6->ip6_plen = htons(pd->tot_len - pd->off);
3655 		ip6->ip6_nxt = pd->proto;
3656 		if (!pd->ttl || pd->ttl > IPV6_DEFHLIM)
3657 			ip6->ip6_hlim = IPV6_DEFHLIM;
3658 		else
3659 			ip6->ip6_hlim = pd->ttl;
3660 		ip6->ip6_src = pd->nsaddr.v6;
3661 		ip6->ip6_dst = pd->ndaddr.v6;
3662 		pd->src = (struct pf_addr *)&ip6->ip6_src;
3663 		pd->dst = (struct pf_addr *)&ip6->ip6_dst;
3664 		pd->off = sizeof(struct ip6_hdr);
3665 		if (pd->m->m_pkthdr.csum_flags & CSUM_TCP) {
3666 			pd->m->m_pkthdr.csum_flags &= ~CSUM_TCP;
3667 			pd->m->m_pkthdr.csum_flags |= CSUM_TCP_IPV6;
3668 		}
3669 		if (pd->m->m_pkthdr.csum_flags & CSUM_UDP) {
3670 			pd->m->m_pkthdr.csum_flags &= ~CSUM_UDP;
3671 			pd->m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6;
3672 		}
3673 		if (pd->m->m_pkthdr.csum_flags & CSUM_SCTP) {
3674 			pd->m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
3675 			pd->m->m_pkthdr.csum_flags |= CSUM_SCTP_IPV6;
3676 		}
3677 
3678 		/*
3679 		 * If we're dealing with a reassembled packet we need to adjust
3680 		 * the header length from the IPv4 header size to IPv6 header
3681 		 * size.
3682 		 */
3683 		mtag = m_tag_find(pd->m, PACKET_TAG_PF_REASSEMBLED, NULL);
3684 		if (mtag) {
3685 			ftag = (struct pf_fragment_tag *)(mtag + 1);
3686 			ftag->ft_hdrlen = sizeof(*ip6);
3687 			ftag->ft_maxlen -= sizeof(struct ip6_hdr) -
3688 			    sizeof(struct ip) + sizeof(struct ip6_frag);
3689 		}
3690 		break;
3691 	default:
3692 		return (-1);
3693 	}
3694 
3695 	/* recalculate icmp/icmp6 checksums */
3696 	if (pd->proto == IPPROTO_ICMP || pd->proto == IPPROTO_ICMPV6) {
3697 		int off;
3698 		if ((mp = m_pulldown(pd->m, hlen, sizeof(*icmp), &off)) ==
3699 		    NULL) {
3700 			pd->m = NULL;
3701 			return (-1);
3702 		}
3703 		icmp = (struct icmp6_hdr *)(mp->m_data + off);
3704 		icmp->icmp6_cksum = 0;
3705 		icmp->icmp6_cksum = pd->naf == AF_INET ?
3706 		    in4_cksum(pd->m, 0, hlen, ntohs(ip4->ip_len) - hlen) :
3707 		    in6_cksum(pd->m, IPPROTO_ICMPV6, hlen,
3708 			ntohs(ip6->ip6_plen));
3709 	}
3710 #endif /* INET && INET6 */
3711 
3712 	return (0);
3713 }
3714 
3715 int
3716 pf_change_icmp_af(struct mbuf *m, int off, struct pf_pdesc *pd,
3717     struct pf_pdesc *pd2, struct pf_addr *src, struct pf_addr *dst,
3718     sa_family_t af, sa_family_t naf)
3719 {
3720 #if defined(INET) && defined(INET6)
3721 	struct mbuf	*n = NULL;
3722 	struct ip	*ip4;
3723 	struct ip6_hdr	*ip6;
3724 	int		 hlen, olen, mlen;
3725 
3726 	if (af == naf || (af != AF_INET && af != AF_INET6) ||
3727 	    (naf != AF_INET && naf != AF_INET6))
3728 		return (-1);
3729 
3730 	/* split the mbuf chain on the inner ip/ip6 header boundary */
3731 	if ((n = m_split(m, off, M_NOWAIT)) == NULL)
3732 		return (-1);
3733 
3734 	/* old header */
3735 	olen = pd2->off - off;
3736 	/* new header */
3737 	hlen = naf == AF_INET ? sizeof(*ip4) : sizeof(*ip6);
3738 
3739 	/* trim old header */
3740 	m_adj(n, olen);
3741 
3742 	/* prepend a new one */
3743 	M_PREPEND(n, hlen, M_NOWAIT);
3744 	if (n == NULL)
3745 		return (-1);
3746 
3747 	/* translate inner ip/ip6 header */
3748 	switch (naf) {
3749 	case AF_INET:
3750 		ip4 = mtod(n, struct ip *);
3751 		bzero(ip4, sizeof(*ip4));
3752 		ip4->ip_v = IPVERSION;
3753 		ip4->ip_hl = sizeof(*ip4) >> 2;
3754 		ip4->ip_len = htons(sizeof(*ip4) + pd2->tot_len - olen);
3755 		ip_fillid(ip4, V_ip_random_id);
3756 		ip4->ip_off = htons(IP_DF);
3757 		ip4->ip_ttl = pd2->ttl;
3758 		if (pd2->proto == IPPROTO_ICMPV6)
3759 			ip4->ip_p = IPPROTO_ICMP;
3760 		else
3761 			ip4->ip_p = pd2->proto;
3762 		ip4->ip_src = src->v4;
3763 		ip4->ip_dst = dst->v4;
3764 		ip4->ip_sum = in_cksum(n, ip4->ip_hl << 2);
3765 		break;
3766 	case AF_INET6:
3767 		ip6 = mtod(n, struct ip6_hdr *);
3768 		bzero(ip6, sizeof(*ip6));
3769 		ip6->ip6_vfc = IPV6_VERSION;
3770 		ip6->ip6_plen = htons(pd2->tot_len - olen);
3771 		if (pd2->proto == IPPROTO_ICMP)
3772 			ip6->ip6_nxt = IPPROTO_ICMPV6;
3773 		else
3774 			ip6->ip6_nxt = pd2->proto;
3775 		if (!pd2->ttl || pd2->ttl > IPV6_DEFHLIM)
3776 			ip6->ip6_hlim = IPV6_DEFHLIM;
3777 		else
3778 			ip6->ip6_hlim = pd2->ttl;
3779 		ip6->ip6_src = src->v6;
3780 		ip6->ip6_dst = dst->v6;
3781 		break;
3782 	default:
3783 		unhandled_af(naf);
3784 	}
3785 
3786 	/* adjust payload offset and total packet length */
3787 	pd2->off += hlen - olen;
3788 	pd->tot_len += hlen - olen;
3789 
3790 	/* merge modified inner packet with the original header */
3791 	mlen = n->m_pkthdr.len;
3792 	m_cat(m, n);
3793 	m->m_pkthdr.len += mlen;
3794 #endif /* INET && INET6 */
3795 
3796 	return (0);
3797 }
3798 
3799 #define PTR_IP(field)	(offsetof(struct ip, field))
3800 #define PTR_IP6(field)	(offsetof(struct ip6_hdr, field))
3801 
3802 int
3803 pf_translate_icmp_af(int af, void *arg)
3804 {
3805 #if defined(INET) && defined(INET6)
3806 	struct icmp		*icmp4;
3807 	struct icmp6_hdr	*icmp6;
3808 	u_int32_t		 mtu;
3809 	int32_t			 ptr = -1;
3810 	u_int8_t		 type;
3811 	u_int8_t		 code;
3812 
3813 	switch (af) {
3814 	case AF_INET:
3815 		icmp6 = arg;
3816 		type = icmp6->icmp6_type;
3817 		code = icmp6->icmp6_code;
3818 		mtu = ntohl(icmp6->icmp6_mtu);
3819 
3820 		switch (type) {
3821 		case ICMP6_ECHO_REQUEST:
3822 			type = ICMP_ECHO;
3823 			break;
3824 		case ICMP6_ECHO_REPLY:
3825 			type = ICMP_ECHOREPLY;
3826 			break;
3827 		case ICMP6_DST_UNREACH:
3828 			type = ICMP_UNREACH;
3829 			switch (code) {
3830 			case ICMP6_DST_UNREACH_NOROUTE:
3831 			case ICMP6_DST_UNREACH_BEYONDSCOPE:
3832 			case ICMP6_DST_UNREACH_ADDR:
3833 				code = ICMP_UNREACH_HOST;
3834 				break;
3835 			case ICMP6_DST_UNREACH_ADMIN:
3836 				code = ICMP_UNREACH_HOST_PROHIB;
3837 				break;
3838 			case ICMP6_DST_UNREACH_NOPORT:
3839 				code = ICMP_UNREACH_PORT;
3840 				break;
3841 			default:
3842 				return (-1);
3843 			}
3844 			break;
3845 		case ICMP6_PACKET_TOO_BIG:
3846 			type = ICMP_UNREACH;
3847 			code = ICMP_UNREACH_NEEDFRAG;
3848 			mtu -= 20;
3849 			break;
3850 		case ICMP6_TIME_EXCEEDED:
3851 			type = ICMP_TIMXCEED;
3852 			break;
3853 		case ICMP6_PARAM_PROB:
3854 			switch (code) {
3855 			case ICMP6_PARAMPROB_HEADER:
3856 				type = ICMP_PARAMPROB;
3857 				code = ICMP_PARAMPROB_ERRATPTR;
3858 				ptr = ntohl(icmp6->icmp6_pptr);
3859 
3860 				if (ptr == PTR_IP6(ip6_vfc))
3861 					; /* preserve */
3862 				else if (ptr == PTR_IP6(ip6_vfc) + 1)
3863 					ptr = PTR_IP(ip_tos);
3864 				else if (ptr == PTR_IP6(ip6_plen) ||
3865 				    ptr == PTR_IP6(ip6_plen) + 1)
3866 					ptr = PTR_IP(ip_len);
3867 				else if (ptr == PTR_IP6(ip6_nxt))
3868 					ptr = PTR_IP(ip_p);
3869 				else if (ptr == PTR_IP6(ip6_hlim))
3870 					ptr = PTR_IP(ip_ttl);
3871 				else if (ptr >= PTR_IP6(ip6_src) &&
3872 				    ptr < PTR_IP6(ip6_dst))
3873 					ptr = PTR_IP(ip_src);
3874 				else if (ptr >= PTR_IP6(ip6_dst) &&
3875 				    ptr < sizeof(struct ip6_hdr))
3876 					ptr = PTR_IP(ip_dst);
3877 				else {
3878 					return (-1);
3879 				}
3880 				break;
3881 			case ICMP6_PARAMPROB_NEXTHEADER:
3882 				type = ICMP_UNREACH;
3883 				code = ICMP_UNREACH_PROTOCOL;
3884 				break;
3885 			default:
3886 				return (-1);
3887 			}
3888 			break;
3889 		default:
3890 			return (-1);
3891 		}
3892 		if (icmp6->icmp6_type != type) {
3893 			icmp6->icmp6_cksum = pf_cksum_fixup(icmp6->icmp6_cksum,
3894 			    icmp6->icmp6_type, type, 0);
3895 			icmp6->icmp6_type = type;
3896 		}
3897 		if (icmp6->icmp6_code != code) {
3898 			icmp6->icmp6_cksum = pf_cksum_fixup(icmp6->icmp6_cksum,
3899 			    icmp6->icmp6_code, code, 0);
3900 			icmp6->icmp6_code = code;
3901 		}
3902 		if (icmp6->icmp6_mtu != htonl(mtu)) {
3903 			icmp6->icmp6_cksum = pf_cksum_fixup(icmp6->icmp6_cksum,
3904 			    htons(ntohl(icmp6->icmp6_mtu)), htons(mtu), 0);
3905 			/* aligns well with a icmpv4 nextmtu */
3906 			icmp6->icmp6_mtu = htonl(mtu);
3907 		}
3908 		if (ptr >= 0 && icmp6->icmp6_pptr != htonl(ptr)) {
3909 			icmp6->icmp6_cksum = pf_cksum_fixup(icmp6->icmp6_cksum,
3910 			    htons(ntohl(icmp6->icmp6_pptr)), htons(ptr), 0);
3911 			/* icmpv4 pptr is a one most significant byte */
3912 			icmp6->icmp6_pptr = htonl(ptr << 24);
3913 		}
3914 		break;
3915 	case AF_INET6:
3916 		icmp4 = arg;
3917 		type = icmp4->icmp_type;
3918 		code = icmp4->icmp_code;
3919 		mtu = ntohs(icmp4->icmp_nextmtu);
3920 
3921 		switch (type) {
3922 		case ICMP_ECHO:
3923 			type = ICMP6_ECHO_REQUEST;
3924 			break;
3925 		case ICMP_ECHOREPLY:
3926 			type = ICMP6_ECHO_REPLY;
3927 			break;
3928 		case ICMP_UNREACH:
3929 			type = ICMP6_DST_UNREACH;
3930 			switch (code) {
3931 			case ICMP_UNREACH_NET:
3932 			case ICMP_UNREACH_HOST:
3933 			case ICMP_UNREACH_NET_UNKNOWN:
3934 			case ICMP_UNREACH_HOST_UNKNOWN:
3935 			case ICMP_UNREACH_ISOLATED:
3936 			case ICMP_UNREACH_TOSNET:
3937 			case ICMP_UNREACH_TOSHOST:
3938 				code = ICMP6_DST_UNREACH_NOROUTE;
3939 				break;
3940 			case ICMP_UNREACH_PORT:
3941 				code = ICMP6_DST_UNREACH_NOPORT;
3942 				break;
3943 			case ICMP_UNREACH_NET_PROHIB:
3944 			case ICMP_UNREACH_HOST_PROHIB:
3945 			case ICMP_UNREACH_FILTER_PROHIB:
3946 			case ICMP_UNREACH_PRECEDENCE_CUTOFF:
3947 				code = ICMP6_DST_UNREACH_ADMIN;
3948 				break;
3949 			case ICMP_UNREACH_PROTOCOL:
3950 				type = ICMP6_PARAM_PROB;
3951 				code = ICMP6_PARAMPROB_NEXTHEADER;
3952 				ptr = offsetof(struct ip6_hdr, ip6_nxt);
3953 				break;
3954 			case ICMP_UNREACH_NEEDFRAG:
3955 				type = ICMP6_PACKET_TOO_BIG;
3956 				code = 0;
3957 				mtu += 20;
3958 				break;
3959 			default:
3960 				return (-1);
3961 			}
3962 			break;
3963 		case ICMP_TIMXCEED:
3964 			type = ICMP6_TIME_EXCEEDED;
3965 			break;
3966 		case ICMP_PARAMPROB:
3967 			type = ICMP6_PARAM_PROB;
3968 			switch (code) {
3969 			case ICMP_PARAMPROB_ERRATPTR:
3970 				code = ICMP6_PARAMPROB_HEADER;
3971 				break;
3972 			case ICMP_PARAMPROB_LENGTH:
3973 				code = ICMP6_PARAMPROB_HEADER;
3974 				break;
3975 			default:
3976 				return (-1);
3977 			}
3978 
3979 			ptr = icmp4->icmp_pptr;
3980 			if (ptr == 0 || ptr == PTR_IP(ip_tos))
3981 				; /* preserve */
3982 			else if (ptr == PTR_IP(ip_len) ||
3983 			    ptr == PTR_IP(ip_len) + 1)
3984 				ptr = PTR_IP6(ip6_plen);
3985 			else if (ptr == PTR_IP(ip_ttl))
3986 				ptr = PTR_IP6(ip6_hlim);
3987 			else if (ptr == PTR_IP(ip_p))
3988 				ptr = PTR_IP6(ip6_nxt);
3989 			else if (ptr >= PTR_IP(ip_src) && ptr < PTR_IP(ip_dst))
3990 				ptr = PTR_IP6(ip6_src);
3991 			else if (ptr >= PTR_IP(ip_dst) &&
3992 			    ptr < sizeof(struct ip))
3993 				ptr = PTR_IP6(ip6_dst);
3994 			else {
3995 				return (-1);
3996 			}
3997 			break;
3998 		default:
3999 			return (-1);
4000 		}
4001 		if (icmp4->icmp_type != type) {
4002 			icmp4->icmp_cksum = pf_cksum_fixup(icmp4->icmp_cksum,
4003 			    icmp4->icmp_type, type, 0);
4004 			icmp4->icmp_type = type;
4005 		}
4006 		if (icmp4->icmp_code != code) {
4007 			icmp4->icmp_cksum = pf_cksum_fixup(icmp4->icmp_cksum,
4008 			    icmp4->icmp_code, code, 0);
4009 			icmp4->icmp_code = code;
4010 		}
4011 		if (icmp4->icmp_nextmtu != htons(mtu)) {
4012 			icmp4->icmp_cksum = pf_cksum_fixup(icmp4->icmp_cksum,
4013 			    icmp4->icmp_nextmtu, htons(mtu), 0);
4014 			icmp4->icmp_nextmtu = htons(mtu);
4015 		}
4016 		if (ptr >= 0 && icmp4->icmp_void != ptr) {
4017 			icmp4->icmp_cksum = pf_cksum_fixup(icmp4->icmp_cksum,
4018 			    htons(icmp4->icmp_pptr), htons(ptr), 0);
4019 			icmp4->icmp_void = htonl(ptr);
4020 		}
4021 		break;
4022 	default:
4023 		unhandled_af(af);
4024 	}
4025 #endif /* INET && INET6 */
4026 
4027 	return (0);
4028 }
4029 
4030 /*
4031  * Need to modulate the sequence numbers in the TCP SACK option
4032  * (credits to Krzysztof Pfaff for report and patch)
4033  */
4034 static int
4035 pf_modulate_sack(struct pf_pdesc *pd, struct tcphdr *th,
4036     struct pf_state_peer *dst)
4037 {
4038 	struct sackblk	 sack;
4039 	int		 copyback = 0, i;
4040 	int		 olen, optsoff;
4041 	uint8_t		 opts[MAX_TCPOPTLEN], *opt, *eoh;
4042 
4043 	olen = (pd->hdr.tcp.th_off << 2) - sizeof(struct tcphdr);
4044 	optsoff = pd->off + sizeof(struct tcphdr);
4045 #define	TCPOLEN_MINSACK	(TCPOLEN_SACK + 2)
4046 	if (olen < TCPOLEN_MINSACK ||
4047 	    !pf_pull_hdr(pd->m, optsoff, opts, olen, NULL, NULL, pd->af))
4048 		return (0);
4049 
4050 	eoh = opts + olen;
4051 	opt = opts;
4052 	while ((opt = pf_find_tcpopt(opt, opts, olen,
4053 	    TCPOPT_SACK, TCPOLEN_MINSACK)) != NULL)
4054 	{
4055 		size_t safelen = MIN(opt[1], (eoh - opt));
4056 		for (i = 2; i + TCPOLEN_SACK <= safelen; i += TCPOLEN_SACK) {
4057 			size_t startoff = (opt + i) - opts;
4058 			memcpy(&sack, &opt[i], sizeof(sack));
4059 			pf_patch_32(pd, &sack.start,
4060 			    htonl(ntohl(sack.start) - dst->seqdiff),
4061 			    PF_ALGNMNT(startoff));
4062 			pf_patch_32(pd, &sack.end,
4063 			    htonl(ntohl(sack.end) - dst->seqdiff),
4064 			    PF_ALGNMNT(startoff + sizeof(sack.start)));
4065 			memcpy(&opt[i], &sack, sizeof(sack));
4066 		}
4067 		copyback = 1;
4068 		opt += opt[1];
4069 	}
4070 
4071 	if (copyback)
4072 		m_copyback(pd->m, optsoff, olen, (caddr_t)opts);
4073 
4074 	return (copyback);
4075 }
4076 
4077 struct mbuf *
4078 pf_build_tcp(const struct pf_krule *r, sa_family_t af,
4079     const struct pf_addr *saddr, const struct pf_addr *daddr,
4080     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
4081     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
4082     int mbuf_flags, u_int16_t mtag_tag, u_int16_t mtag_flags, u_int sack,
4083     int rtableid)
4084 {
4085 	struct mbuf	*m;
4086 	int		 len, tlen;
4087 #ifdef INET
4088 	struct ip	*h = NULL;
4089 #endif /* INET */
4090 #ifdef INET6
4091 	struct ip6_hdr	*h6 = NULL;
4092 #endif /* INET6 */
4093 	struct tcphdr	*th;
4094 	char		*opt;
4095 	struct pf_mtag  *pf_mtag;
4096 
4097 	len = 0;
4098 	th = NULL;
4099 
4100 	/* maximum segment size tcp option */
4101 	tlen = sizeof(struct tcphdr);
4102 	if (mss)
4103 		tlen += 4;
4104 	if (sack)
4105 		tlen += 2;
4106 
4107 	switch (af) {
4108 #ifdef INET
4109 	case AF_INET:
4110 		len = sizeof(struct ip) + tlen;
4111 		break;
4112 #endif /* INET */
4113 #ifdef INET6
4114 	case AF_INET6:
4115 		len = sizeof(struct ip6_hdr) + tlen;
4116 		break;
4117 #endif /* INET6 */
4118 	default:
4119 		unhandled_af(af);
4120 	}
4121 
4122 	m = m_gethdr(M_NOWAIT, MT_DATA);
4123 	if (m == NULL)
4124 		return (NULL);
4125 
4126 #ifdef MAC
4127 	mac_netinet_firewall_send(m);
4128 #endif
4129 	if ((pf_mtag = pf_get_mtag(m)) == NULL) {
4130 		m_freem(m);
4131 		return (NULL);
4132 	}
4133 	m->m_flags |= mbuf_flags;
4134 	pf_mtag->tag = mtag_tag;
4135 	pf_mtag->flags = mtag_flags;
4136 
4137 	if (rtableid >= 0)
4138 		M_SETFIB(m, rtableid);
4139 
4140 #ifdef ALTQ
4141 	if (r != NULL && r->qid) {
4142 		pf_mtag->qid = r->qid;
4143 
4144 		/* add hints for ecn */
4145 		pf_mtag->hdr = mtod(m, struct ip *);
4146 	}
4147 #endif /* ALTQ */
4148 	m->m_data += max_linkhdr;
4149 	m->m_pkthdr.len = m->m_len = len;
4150 	/* The rest of the stack assumes a rcvif, so provide one.
4151 	 * This is a locally generated packet, so .. close enough. */
4152 	m->m_pkthdr.rcvif = V_loif;
4153 	bzero(m->m_data, len);
4154 	switch (af) {
4155 #ifdef INET
4156 	case AF_INET:
4157 		m->m_pkthdr.csum_flags |= CSUM_TCP;
4158 		m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
4159 
4160 		h = mtod(m, struct ip *);
4161 
4162 		h->ip_p = IPPROTO_TCP;
4163 		h->ip_len = htons(tlen);
4164 		h->ip_v = 4;
4165 		h->ip_hl = sizeof(*h) >> 2;
4166 		h->ip_tos = IPTOS_LOWDELAY;
4167 		h->ip_len = htons(len);
4168 		h->ip_off = htons(V_path_mtu_discovery ? IP_DF : 0);
4169 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
4170 		h->ip_sum = 0;
4171 		h->ip_src.s_addr = saddr->v4.s_addr;
4172 		h->ip_dst.s_addr = daddr->v4.s_addr;
4173 
4174 		th = (struct tcphdr *)((caddr_t)h + sizeof(struct ip));
4175 		th->th_sum = in_pseudo(h->ip_src.s_addr, h->ip_dst.s_addr,
4176 		    htons(len - sizeof(struct ip) + IPPROTO_TCP));
4177 		break;
4178 #endif /* INET */
4179 #ifdef INET6
4180 	case AF_INET6:
4181 		m->m_pkthdr.csum_flags |= CSUM_TCP_IPV6;
4182 		m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
4183 
4184 		h6 = mtod(m, struct ip6_hdr *);
4185 
4186 		/* IP header fields included in the TCP checksum */
4187 		h6->ip6_nxt = IPPROTO_TCP;
4188 		h6->ip6_plen = htons(tlen);
4189 		h6->ip6_vfc |= IPV6_VERSION;
4190 		h6->ip6_hlim = V_ip6_defhlim;
4191 		memcpy(&h6->ip6_src, &saddr->v6, sizeof(struct in6_addr));
4192 		memcpy(&h6->ip6_dst, &daddr->v6, sizeof(struct in6_addr));
4193 
4194 		th = (struct tcphdr *)((caddr_t)h6 + sizeof(struct ip6_hdr));
4195 		th->th_sum = in6_cksum_pseudo(h6, len - sizeof(struct ip6_hdr),
4196 		    IPPROTO_TCP, 0);
4197 		break;
4198 #endif /* INET6 */
4199 	}
4200 
4201 	/* TCP header */
4202 	th->th_sport = sport;
4203 	th->th_dport = dport;
4204 	th->th_seq = htonl(seq);
4205 	th->th_ack = htonl(ack);
4206 	th->th_off = tlen >> 2;
4207 	tcp_set_flags(th, tcp_flags);
4208 	th->th_win = htons(win);
4209 
4210 	opt = (char *)(th + 1);
4211 	if (mss) {
4212 		opt = (char *)(th + 1);
4213 		opt[0] = TCPOPT_MAXSEG;
4214 		opt[1] = 4;
4215 		mss = htons(mss);
4216 		memcpy((opt + 2), &mss, 2);
4217 		opt += 4;
4218 	}
4219 	if (sack) {
4220 		opt[0] = TCPOPT_SACK_PERMITTED;
4221 		opt[1] = 2;
4222 		opt += 2;
4223 	}
4224 
4225 	return (m);
4226 }
4227 
4228 static void
4229 pf_send_sctp_abort(sa_family_t af, struct pf_pdesc *pd,
4230     uint8_t ttl, int rtableid)
4231 {
4232 	struct mbuf		*m;
4233 #ifdef INET
4234 	struct ip		*h = NULL;
4235 #endif /* INET */
4236 #ifdef INET6
4237 	struct ip6_hdr		*h6 = NULL;
4238 #endif /* INET6 */
4239 	struct sctphdr		*hdr;
4240 	struct sctp_chunkhdr	*chunk;
4241 	struct pf_send_entry	*pfse;
4242 	int			 off = 0;
4243 
4244 	MPASS(af == pd->af);
4245 
4246 	m = m_gethdr(M_NOWAIT, MT_DATA);
4247 	if (m == NULL)
4248 		return;
4249 
4250 	m->m_data += max_linkhdr;
4251 	m->m_flags |= M_SKIP_FIREWALL;
4252 	/* The rest of the stack assumes a rcvif, so provide one.
4253 	 * This is a locally generated packet, so .. close enough. */
4254 	m->m_pkthdr.rcvif = V_loif;
4255 
4256 	/* IPv4|6 header */
4257 	switch (af) {
4258 #ifdef INET
4259 	case AF_INET:
4260 		bzero(m->m_data, sizeof(struct ip) + sizeof(*hdr) + sizeof(*chunk));
4261 
4262 		h = mtod(m, struct ip *);
4263 
4264 		/* IP header fields included in the TCP checksum */
4265 
4266 		h->ip_p = IPPROTO_SCTP;
4267 		h->ip_len = htons(sizeof(*h) + sizeof(*hdr) + sizeof(*chunk));
4268 		h->ip_ttl = ttl ? ttl : V_ip_defttl;
4269 		h->ip_src = pd->dst->v4;
4270 		h->ip_dst = pd->src->v4;
4271 
4272 		off += sizeof(struct ip);
4273 		break;
4274 #endif /* INET */
4275 #ifdef INET6
4276 	case AF_INET6:
4277 		bzero(m->m_data, sizeof(struct ip6_hdr) + sizeof(*hdr) + sizeof(*chunk));
4278 
4279 		h6 = mtod(m, struct ip6_hdr *);
4280 
4281 		/* IP header fields included in the TCP checksum */
4282 		h6->ip6_vfc |= IPV6_VERSION;
4283 		h6->ip6_nxt = IPPROTO_SCTP;
4284 		h6->ip6_plen = htons(sizeof(*h6) + sizeof(*hdr) + sizeof(*chunk));
4285 		h6->ip6_hlim = ttl ? ttl : V_ip6_defhlim;
4286 		memcpy(&h6->ip6_src, &pd->dst->v6, sizeof(struct in6_addr));
4287 		memcpy(&h6->ip6_dst, &pd->src->v6, sizeof(struct in6_addr));
4288 
4289 		off += sizeof(struct ip6_hdr);
4290 		break;
4291 #endif /* INET6 */
4292 	default:
4293 		unhandled_af(af);
4294 	}
4295 
4296 	/* SCTP header */
4297 	hdr = mtodo(m, off);
4298 
4299 	hdr->src_port = pd->hdr.sctp.dest_port;
4300 	hdr->dest_port = pd->hdr.sctp.src_port;
4301 	hdr->v_tag = pd->sctp_initiate_tag;
4302 	hdr->checksum = 0;
4303 
4304 	/* Abort chunk. */
4305 	off += sizeof(struct sctphdr);
4306 	chunk = mtodo(m, off);
4307 
4308 	chunk->chunk_type = SCTP_ABORT_ASSOCIATION;
4309 	chunk->chunk_length = htons(sizeof(*chunk));
4310 
4311 	/* SCTP checksum */
4312 	off += sizeof(*chunk);
4313 	m->m_pkthdr.len = m->m_len = off;
4314 
4315 	pf_sctp_checksum(m, off - sizeof(*hdr) - sizeof(*chunk));
4316 
4317 	if (rtableid >= 0)
4318 		M_SETFIB(m, rtableid);
4319 
4320 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
4321 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
4322 	if (pfse == NULL) {
4323 		m_freem(m);
4324 		return;
4325 	}
4326 
4327 	switch (af) {
4328 #ifdef INET
4329 	case AF_INET:
4330 		pfse->pfse_type = PFSE_IP;
4331 		break;
4332 #endif /* INET */
4333 #ifdef INET6
4334 	case AF_INET6:
4335 		pfse->pfse_type = PFSE_IP6;
4336 		break;
4337 #endif /* INET6 */
4338 	}
4339 
4340 	pfse->pfse_m = m;
4341 	pf_send(pfse);
4342 }
4343 
4344 void
4345 pf_send_tcp(const struct pf_krule *r, sa_family_t af,
4346     const struct pf_addr *saddr, const struct pf_addr *daddr,
4347     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
4348     u_int8_t tcp_flags, u_int16_t win, u_int16_t mss, u_int8_t ttl,
4349     int mbuf_flags, u_int16_t mtag_tag, u_int16_t mtag_flags, int rtableid)
4350 {
4351 	struct pf_send_entry *pfse;
4352 	struct mbuf	*m;
4353 
4354 	m = pf_build_tcp(r, af, saddr, daddr, sport, dport, seq, ack, tcp_flags,
4355 	    win, mss, ttl, mbuf_flags, mtag_tag, mtag_flags, 0, rtableid);
4356 	if (m == NULL)
4357 		return;
4358 
4359 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
4360 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
4361 	if (pfse == NULL) {
4362 		m_freem(m);
4363 		return;
4364 	}
4365 
4366 	switch (af) {
4367 #ifdef INET
4368 	case AF_INET:
4369 		pfse->pfse_type = PFSE_IP;
4370 		break;
4371 #endif /* INET */
4372 #ifdef INET6
4373 	case AF_INET6:
4374 		pfse->pfse_type = PFSE_IP6;
4375 		break;
4376 #endif /* INET6 */
4377 	default:
4378 		unhandled_af(af);
4379 	}
4380 
4381 	pfse->pfse_m = m;
4382 	pf_send(pfse);
4383 }
4384 
4385 static void
4386 pf_undo_nat(struct pf_krule *nr, struct pf_pdesc *pd, uint16_t bip_sum)
4387 {
4388 	/* undo NAT changes, if they have taken place */
4389 	if (nr != NULL) {
4390 		pf_addrcpy(pd->src, &pd->osrc, pd->af);
4391 		pf_addrcpy(pd->dst, &pd->odst, pd->af);
4392 		if (pd->sport)
4393 			*pd->sport = pd->osport;
4394 		if (pd->dport)
4395 			*pd->dport = pd->odport;
4396 		if (pd->ip_sum)
4397 			*pd->ip_sum = bip_sum;
4398 		m_copyback(pd->m, pd->off, pd->hdrlen, pd->hdr.any);
4399 	}
4400 }
4401 
4402 static void
4403 pf_return(struct pf_krule *r, struct pf_krule *nr, struct pf_pdesc *pd,
4404     struct tcphdr *th, u_int16_t bproto_sum, u_int16_t bip_sum,
4405     u_short *reason, int rtableid)
4406 {
4407 	pf_undo_nat(nr, pd, bip_sum);
4408 
4409 	if (pd->proto == IPPROTO_TCP &&
4410 	    ((r->rule_flag & PFRULE_RETURNRST) ||
4411 	    (r->rule_flag & PFRULE_RETURN)) &&
4412 	    !(tcp_get_flags(th) & TH_RST)) {
4413 		u_int32_t	 ack = ntohl(th->th_seq) + pd->p_len;
4414 
4415 		if (pf_check_proto_cksum(pd->m, pd->off, pd->tot_len - pd->off,
4416 		    IPPROTO_TCP, pd->af))
4417 			REASON_SET(reason, PFRES_PROTCKSUM);
4418 		else {
4419 			if (tcp_get_flags(th) & TH_SYN)
4420 				ack++;
4421 			if (tcp_get_flags(th) & TH_FIN)
4422 				ack++;
4423 			pf_send_tcp(r, pd->af, pd->dst,
4424 				pd->src, th->th_dport, th->th_sport,
4425 				ntohl(th->th_ack), ack, TH_RST|TH_ACK, 0, 0,
4426 				r->return_ttl, M_SKIP_FIREWALL, 0, 0, rtableid);
4427 		}
4428 	} else if (pd->proto == IPPROTO_SCTP &&
4429 	    (r->rule_flag & PFRULE_RETURN)) {
4430 		pf_send_sctp_abort(pd->af, pd, r->return_ttl, rtableid);
4431 	} else if (pd->proto != IPPROTO_ICMP && pd->af == AF_INET &&
4432 		r->return_icmp)
4433 		pf_send_icmp(pd->m, r->return_icmp >> 8,
4434 			r->return_icmp & 255, 0, pd->af, r, rtableid);
4435 	else if (pd->proto != IPPROTO_ICMPV6 && pd->af == AF_INET6 &&
4436 		r->return_icmp6)
4437 		pf_send_icmp(pd->m, r->return_icmp6 >> 8,
4438 			r->return_icmp6 & 255, 0, pd->af, r, rtableid);
4439 }
4440 
4441 static int
4442 pf_match_ieee8021q_pcp(u_int8_t prio, struct mbuf *m)
4443 {
4444 	struct m_tag *mtag;
4445 	u_int8_t mpcp;
4446 
4447 	mtag = m_tag_locate(m, MTAG_8021Q, MTAG_8021Q_PCP_IN, NULL);
4448 	if (mtag == NULL)
4449 		return (0);
4450 
4451 	if (prio == PF_PRIO_ZERO)
4452 		prio = 0;
4453 
4454 	mpcp = *(uint8_t *)(mtag + 1);
4455 
4456 	return (mpcp == prio);
4457 }
4458 
4459 static int
4460 pf_icmp_to_bandlim(uint8_t type)
4461 {
4462 	switch (type) {
4463 		case ICMP_ECHO:
4464 		case ICMP_ECHOREPLY:
4465 			return (BANDLIM_ICMP_ECHO);
4466 		case ICMP_TSTAMP:
4467 		case ICMP_TSTAMPREPLY:
4468 			return (BANDLIM_ICMP_TSTAMP);
4469 		case ICMP_UNREACH:
4470 		default:
4471 			return (BANDLIM_ICMP_UNREACH);
4472 	}
4473 }
4474 
4475 static void
4476 pf_send_challenge_ack(struct pf_pdesc *pd, struct pf_kstate *s,
4477     struct pf_state_peer *src, struct pf_state_peer *dst)
4478 {
4479 	/*
4480 	 * We are sending challenge ACK as a response to SYN packet, which
4481 	 * matches existing state (modulo TCP window check). Therefore packet
4482 	 * must be sent on behalf of destination.
4483 	 *
4484 	 * We expect sender to remain either silent, or send RST packet
4485 	 * so both, firewall and remote peer, can purge dead state from
4486 	 * memory.
4487 	 */
4488 	pf_send_tcp(s->rule, pd->af, pd->dst, pd->src,
4489 	    pd->hdr.tcp.th_dport, pd->hdr.tcp.th_sport, dst->seqlo,
4490 	    src->seqlo, TH_ACK, 0, 0, s->rule->return_ttl, 0, 0, 0,
4491 	    s->rule->rtableid);
4492 }
4493 
4494 static void
4495 pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, int mtu,
4496     sa_family_t af, struct pf_krule *r, int rtableid)
4497 {
4498 	struct pf_send_entry *pfse;
4499 	struct mbuf *m0;
4500 	struct pf_mtag *pf_mtag;
4501 
4502 	/* ICMP packet rate limitation. */
4503 	switch (af) {
4504 #ifdef INET6
4505 	case AF_INET6:
4506 		if (icmp6_ratelimit(NULL, type, code))
4507 			return;
4508 		break;
4509 #endif /* INET6 */
4510 #ifdef INET
4511 	case AF_INET:
4512 		if (badport_bandlim(pf_icmp_to_bandlim(type)) != 0)
4513 			return;
4514 		break;
4515 #endif /* INET */
4516 	}
4517 
4518 	/* Allocate outgoing queue entry, mbuf and mbuf tag. */
4519 	pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT);
4520 	if (pfse == NULL)
4521 		return;
4522 
4523 	if ((m0 = m_copypacket(m, M_NOWAIT)) == NULL) {
4524 		free(pfse, M_PFTEMP);
4525 		return;
4526 	}
4527 
4528 	if ((pf_mtag = pf_get_mtag(m0)) == NULL) {
4529 		free(pfse, M_PFTEMP);
4530 		return;
4531 	}
4532 	/* XXX: revisit */
4533 	m0->m_flags |= M_SKIP_FIREWALL;
4534 
4535 	if (rtableid >= 0)
4536 		M_SETFIB(m0, rtableid);
4537 
4538 #ifdef ALTQ
4539 	if (r->qid) {
4540 		pf_mtag->qid = r->qid;
4541 		/* add hints for ecn */
4542 		pf_mtag->hdr = mtod(m0, struct ip *);
4543 	}
4544 #endif /* ALTQ */
4545 
4546 	switch (af) {
4547 #ifdef INET
4548 	case AF_INET:
4549 		pfse->pfse_type = PFSE_ICMP;
4550 		break;
4551 #endif /* INET */
4552 #ifdef INET6
4553 	case AF_INET6:
4554 		pfse->pfse_type = PFSE_ICMP6;
4555 		break;
4556 #endif /* INET6 */
4557 	}
4558 	pfse->pfse_m = m0;
4559 	pfse->icmpopts.type = type;
4560 	pfse->icmpopts.code = code;
4561 	pfse->icmpopts.mtu = mtu;
4562 	pf_send(pfse);
4563 }
4564 
4565 /*
4566  * Return ((n = 0) == (a = b [with mask m]))
4567  * Note: n != 0 => returns (a != b [with mask m])
4568  */
4569 int
4570 pf_match_addr(u_int8_t n, const struct pf_addr *a, const struct pf_addr *m,
4571     const struct pf_addr *b, sa_family_t af)
4572 {
4573 	switch (af) {
4574 #ifdef INET
4575 	case AF_INET:
4576 		if (IN_ARE_MASKED_ADDR_EQUAL(a->v4, b->v4, m->v4))
4577 			return (n == 0);
4578 		break;
4579 #endif /* INET */
4580 #ifdef INET6
4581 	case AF_INET6:
4582 		if (IN6_ARE_MASKED_ADDR_EQUAL(&a->v6, &b->v6, &m->v6))
4583 			return (n == 0);
4584 		break;
4585 #endif /* INET6 */
4586 	}
4587 
4588 	return (n != 0);
4589 }
4590 
4591 /*
4592  * Return 1 if b <= a <= e, otherwise return 0.
4593  */
4594 int
4595 pf_match_addr_range(const struct pf_addr *b, const struct pf_addr *e,
4596     const struct pf_addr *a, sa_family_t af)
4597 {
4598 	switch (af) {
4599 #ifdef INET
4600 	case AF_INET:
4601 		if ((ntohl(a->addr32[0]) < ntohl(b->addr32[0])) ||
4602 		    (ntohl(a->addr32[0]) > ntohl(e->addr32[0])))
4603 			return (0);
4604 		break;
4605 #endif /* INET */
4606 #ifdef INET6
4607 	case AF_INET6: {
4608 		int	i;
4609 
4610 		/* check a >= b */
4611 		for (i = 0; i < 4; ++i)
4612 			if (ntohl(a->addr32[i]) > ntohl(b->addr32[i]))
4613 				break;
4614 			else if (ntohl(a->addr32[i]) < ntohl(b->addr32[i]))
4615 				return (0);
4616 		/* check a <= e */
4617 		for (i = 0; i < 4; ++i)
4618 			if (ntohl(a->addr32[i]) < ntohl(e->addr32[i]))
4619 				break;
4620 			else if (ntohl(a->addr32[i]) > ntohl(e->addr32[i]))
4621 				return (0);
4622 		break;
4623 	}
4624 #endif /* INET6 */
4625 	}
4626 	return (1);
4627 }
4628 
4629 static int
4630 pf_match(u_int8_t op, u_int32_t a1, u_int32_t a2, u_int32_t p)
4631 {
4632 	switch (op) {
4633 	case PF_OP_IRG:
4634 		return ((p > a1) && (p < a2));
4635 	case PF_OP_XRG:
4636 		return ((p < a1) || (p > a2));
4637 	case PF_OP_RRG:
4638 		return ((p >= a1) && (p <= a2));
4639 	case PF_OP_EQ:
4640 		return (p == a1);
4641 	case PF_OP_NE:
4642 		return (p != a1);
4643 	case PF_OP_LT:
4644 		return (p < a1);
4645 	case PF_OP_LE:
4646 		return (p <= a1);
4647 	case PF_OP_GT:
4648 		return (p > a1);
4649 	case PF_OP_GE:
4650 		return (p >= a1);
4651 	}
4652 	return (0); /* never reached */
4653 }
4654 
4655 int
4656 pf_match_port(u_int8_t op, u_int16_t a1, u_int16_t a2, u_int16_t p)
4657 {
4658 	return (pf_match(op, ntohs(a1), ntohs(a2), ntohs(p)));
4659 }
4660 
4661 static int
4662 pf_match_uid(u_int8_t op, uid_t a1, uid_t a2, uid_t u)
4663 {
4664 	if (u == -1 && op != PF_OP_EQ && op != PF_OP_NE)
4665 		return (0);
4666 	return (pf_match(op, a1, a2, u));
4667 }
4668 
4669 static int
4670 pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g)
4671 {
4672 	if (g == -1 && op != PF_OP_EQ && op != PF_OP_NE)
4673 		return (0);
4674 	return (pf_match(op, a1, a2, g));
4675 }
4676 
4677 int
4678 pf_match_tag(struct mbuf *m, struct pf_krule *r, int *tag, int mtag)
4679 {
4680 	if (*tag == -1)
4681 		*tag = mtag;
4682 
4683 	return ((!r->match_tag_not && r->match_tag == *tag) ||
4684 	    (r->match_tag_not && r->match_tag != *tag));
4685 }
4686 
4687 static int
4688 pf_match_rcvif(struct mbuf *m, struct pf_krule *r)
4689 {
4690 	struct ifnet *ifp = m->m_pkthdr.rcvif;
4691 	struct pfi_kkif *kif;
4692 
4693 	if (ifp == NULL)
4694 		return (0);
4695 
4696 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
4697 
4698 	if (kif == NULL) {
4699 		DPFPRINTF(PF_DEBUG_URGENT,
4700 		    "%s: kif == NULL, @%d via %s", __func__, r->nr,
4701 			r->rcv_ifname);
4702 		return (0);
4703 	}
4704 
4705 	return (pfi_kkif_match(r->rcv_kif, kif));
4706 }
4707 
4708 int
4709 pf_tag_packet(struct pf_pdesc *pd, int tag)
4710 {
4711 
4712 	KASSERT(tag > 0, ("%s: tag %d", __func__, tag));
4713 
4714 	if (pd->pf_mtag == NULL && ((pd->pf_mtag = pf_get_mtag(pd->m)) == NULL))
4715 		return (ENOMEM);
4716 
4717 	pd->pf_mtag->tag = tag;
4718 
4719 	return (0);
4720 }
4721 
4722 /*
4723  * XXX: We rely on malloc(9) returning pointer aligned addresses.
4724  */
4725 #define	PF_ANCHORSTACK_MATCH	0x00000001
4726 #define	PF_ANCHORSTACK_MASK	(PF_ANCHORSTACK_MATCH)
4727 
4728 #define	PF_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
4729 #define	PF_ANCHOR_RULE(f)	(struct pf_krule *)			\
4730 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
4731 #define	PF_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 			\
4732 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
4733 } while (0)
4734 
4735 enum pf_test_status
4736 pf_step_into_anchor(struct pf_test_ctx *ctx, struct pf_krule *r)
4737 {
4738 	enum pf_test_status	rv;
4739 
4740 	PF_RULES_RASSERT();
4741 
4742 	if (ctx->depth >= PF_ANCHOR_STACK_MAX) {
4743 		printf("%s: anchor stack overflow on %s\n",
4744 		    __func__, r->anchor->name);
4745 		return (PF_TEST_FAIL);
4746 	}
4747 
4748 	ctx->depth++;
4749 
4750 	if (r->anchor_wildcard) {
4751 		struct pf_kanchor *child;
4752 		rv = PF_TEST_OK;
4753 		RB_FOREACH(child, pf_kanchor_node, &r->anchor->children) {
4754 			rv = pf_match_rule(ctx, &child->ruleset);
4755 			if ((rv == PF_TEST_QUICK) || (rv == PF_TEST_FAIL)) {
4756 				/*
4757 				 * we either hit a rule with quick action
4758 				 * (more likely), or hit some runtime
4759 				 * error (e.g. pool_get() failure).
4760 				 */
4761 				break;
4762 			}
4763 		}
4764 	} else {
4765 		rv = pf_match_rule(ctx, &r->anchor->ruleset);
4766 		/*
4767 		 * Unless errors occured, stop iff any rule matched
4768 		 * within quick anchors.
4769 		 */
4770 		if (rv != PF_TEST_FAIL && r->quick == PF_TEST_QUICK &&
4771 		    *ctx->am == r)
4772 			rv = PF_TEST_QUICK;
4773 	}
4774 
4775 	ctx->depth--;
4776 
4777 	return (rv);
4778 }
4779 
4780 struct pf_keth_anchor_stackframe {
4781 	struct pf_keth_ruleset	*rs;
4782 	struct pf_keth_rule	*r;	/* XXX: + match bit */
4783 	struct pf_keth_anchor	*child;
4784 };
4785 
4786 #define	PF_ETH_ANCHOR_MATCH(f)	((uintptr_t)(f)->r & PF_ANCHORSTACK_MATCH)
4787 #define	PF_ETH_ANCHOR_RULE(f)	(struct pf_keth_rule *)			\
4788 				((uintptr_t)(f)->r & ~PF_ANCHORSTACK_MASK)
4789 #define	PF_ETH_ANCHOR_SET_MATCH(f)	do { (f)->r = (void *) 		\
4790 				((uintptr_t)(f)->r | PF_ANCHORSTACK_MATCH);  \
4791 } while (0)
4792 
4793 void
4794 pf_step_into_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
4795     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
4796     struct pf_keth_rule **a, int *match)
4797 {
4798 	struct pf_keth_anchor_stackframe	*f;
4799 
4800 	NET_EPOCH_ASSERT();
4801 
4802 	if (match)
4803 		*match = 0;
4804 	if (*depth >= PF_ANCHOR_STACK_MAX) {
4805 		printf("%s: anchor stack overflow on %s\n",
4806 		    __func__, (*r)->anchor->name);
4807 		*r = TAILQ_NEXT(*r, entries);
4808 		return;
4809 	} else if (*depth == 0 && a != NULL)
4810 		*a = *r;
4811 	f = stack + (*depth)++;
4812 	f->rs = *rs;
4813 	f->r = *r;
4814 	if ((*r)->anchor_wildcard) {
4815 		struct pf_keth_anchor_node *parent = &(*r)->anchor->children;
4816 
4817 		if ((f->child = RB_MIN(pf_keth_anchor_node, parent)) == NULL) {
4818 			*r = NULL;
4819 			return;
4820 		}
4821 		*rs = &f->child->ruleset;
4822 	} else {
4823 		f->child = NULL;
4824 		*rs = &(*r)->anchor->ruleset;
4825 	}
4826 	*r = TAILQ_FIRST((*rs)->active.rules);
4827 }
4828 
4829 int
4830 pf_step_out_of_keth_anchor(struct pf_keth_anchor_stackframe *stack, int *depth,
4831     struct pf_keth_ruleset **rs, struct pf_keth_rule **r,
4832     struct pf_keth_rule **a, int *match)
4833 {
4834 	struct pf_keth_anchor_stackframe	*f;
4835 	struct pf_keth_rule *fr;
4836 	int quick = 0;
4837 
4838 	NET_EPOCH_ASSERT();
4839 
4840 	do {
4841 		if (*depth <= 0)
4842 			break;
4843 		f = stack + *depth - 1;
4844 		fr = PF_ETH_ANCHOR_RULE(f);
4845 		if (f->child != NULL) {
4846 			/*
4847 			 * This block traverses through
4848 			 * a wildcard anchor.
4849 			 */
4850 			if (match != NULL && *match) {
4851 				/*
4852 				 * If any of "*" matched, then
4853 				 * "foo/ *" matched, mark frame
4854 				 * appropriately.
4855 				 */
4856 				PF_ETH_ANCHOR_SET_MATCH(f);
4857 				*match = 0;
4858 			}
4859 			f->child = RB_NEXT(pf_keth_anchor_node,
4860 			    &fr->anchor->children, f->child);
4861 			if (f->child != NULL) {
4862 				*rs = &f->child->ruleset;
4863 				*r = TAILQ_FIRST((*rs)->active.rules);
4864 				if (*r == NULL)
4865 					continue;
4866 				else
4867 					break;
4868 			}
4869 		}
4870 		(*depth)--;
4871 		if (*depth == 0 && a != NULL)
4872 			*a = NULL;
4873 		*rs = f->rs;
4874 		if (PF_ETH_ANCHOR_MATCH(f) || (match != NULL && *match))
4875 			quick = fr->quick;
4876 		*r = TAILQ_NEXT(fr, entries);
4877 	} while (*r == NULL);
4878 
4879 	return (quick);
4880 }
4881 
4882 void
4883 pf_poolmask(struct pf_addr *naddr, struct pf_addr *raddr,
4884     struct pf_addr *rmask, struct pf_addr *saddr, sa_family_t af)
4885 {
4886 	switch (af) {
4887 #ifdef INET
4888 	case AF_INET:
4889 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
4890 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
4891 		break;
4892 #endif /* INET */
4893 #ifdef INET6
4894 	case AF_INET6:
4895 		naddr->addr32[0] = (raddr->addr32[0] & rmask->addr32[0]) |
4896 		((rmask->addr32[0] ^ 0xffffffff ) & saddr->addr32[0]);
4897 		naddr->addr32[1] = (raddr->addr32[1] & rmask->addr32[1]) |
4898 		((rmask->addr32[1] ^ 0xffffffff ) & saddr->addr32[1]);
4899 		naddr->addr32[2] = (raddr->addr32[2] & rmask->addr32[2]) |
4900 		((rmask->addr32[2] ^ 0xffffffff ) & saddr->addr32[2]);
4901 		naddr->addr32[3] = (raddr->addr32[3] & rmask->addr32[3]) |
4902 		((rmask->addr32[3] ^ 0xffffffff ) & saddr->addr32[3]);
4903 		break;
4904 #endif /* INET6 */
4905 	}
4906 }
4907 
4908 void
4909 pf_addr_inc(struct pf_addr *addr, sa_family_t af)
4910 {
4911 	switch (af) {
4912 #ifdef INET
4913 	case AF_INET:
4914 		addr->addr32[0] = htonl(ntohl(addr->addr32[0]) + 1);
4915 		break;
4916 #endif /* INET */
4917 #ifdef INET6
4918 	case AF_INET6:
4919 		if (addr->addr32[3] == 0xffffffff) {
4920 			addr->addr32[3] = 0;
4921 			if (addr->addr32[2] == 0xffffffff) {
4922 				addr->addr32[2] = 0;
4923 				if (addr->addr32[1] == 0xffffffff) {
4924 					addr->addr32[1] = 0;
4925 					addr->addr32[0] =
4926 					    htonl(ntohl(addr->addr32[0]) + 1);
4927 				} else
4928 					addr->addr32[1] =
4929 					    htonl(ntohl(addr->addr32[1]) + 1);
4930 			} else
4931 				addr->addr32[2] =
4932 				    htonl(ntohl(addr->addr32[2]) + 1);
4933 		} else
4934 			addr->addr32[3] =
4935 			    htonl(ntohl(addr->addr32[3]) + 1);
4936 		break;
4937 #endif /* INET6 */
4938 	}
4939 }
4940 
4941 void
4942 pf_rule_to_actions(struct pf_krule *r, struct pf_rule_actions *a)
4943 {
4944 	/*
4945 	 * Modern rules use the same flags in rules as they do in states.
4946 	 */
4947 	a->flags |= (r->scrub_flags & (PFSTATE_NODF|PFSTATE_RANDOMID|
4948 	    PFSTATE_SCRUB_TCP|PFSTATE_SETPRIO));
4949 
4950 	/*
4951 	 * Old-style scrub rules have different flags which need to be translated.
4952 	 */
4953 	if (r->rule_flag & PFRULE_RANDOMID)
4954 		a->flags |= PFSTATE_RANDOMID;
4955 	if (r->scrub_flags & PFSTATE_SETTOS || r->rule_flag & PFRULE_SET_TOS ) {
4956 		a->flags |= PFSTATE_SETTOS;
4957 		a->set_tos = r->set_tos;
4958 	}
4959 
4960 	if (r->qid)
4961 		a->qid = r->qid;
4962 	if (r->pqid)
4963 		a->pqid = r->pqid;
4964 	if (r->rtableid >= 0)
4965 		a->rtableid = r->rtableid;
4966 	a->log |= r->log;
4967 	if (r->min_ttl)
4968 		a->min_ttl = r->min_ttl;
4969 	if (r->max_mss)
4970 		a->max_mss = r->max_mss;
4971 	if (r->dnpipe)
4972 		a->dnpipe = r->dnpipe;
4973 	if (r->dnrpipe)
4974 		a->dnrpipe = r->dnrpipe;
4975 	if (r->dnpipe || r->dnrpipe) {
4976 		if (r->free_flags & PFRULE_DN_IS_PIPE)
4977 			a->flags |= PFSTATE_DN_IS_PIPE;
4978 		else
4979 			a->flags &= ~PFSTATE_DN_IS_PIPE;
4980 	}
4981 	if (r->scrub_flags & PFSTATE_SETPRIO) {
4982 		a->set_prio[0] = r->set_prio[0];
4983 		a->set_prio[1] = r->set_prio[1];
4984 	}
4985 	if (r->allow_opts)
4986 		a->allow_opts = r->allow_opts;
4987 	if (r->max_pkt_size)
4988 		a->max_pkt_size = r->max_pkt_size;
4989 }
4990 
4991 int
4992 pf_socket_lookup(struct pf_pdesc *pd)
4993 {
4994 	struct pf_addr		*saddr, *daddr;
4995 	u_int16_t		 sport, dport;
4996 	struct inpcbinfo	*pi;
4997 	struct inpcb		*inp;
4998 
4999 	pd->lookup.uid = -1;
5000 	pd->lookup.gid = -1;
5001 
5002 	switch (pd->proto) {
5003 	case IPPROTO_TCP:
5004 		sport = pd->hdr.tcp.th_sport;
5005 		dport = pd->hdr.tcp.th_dport;
5006 		pi = &V_tcbinfo;
5007 		break;
5008 	case IPPROTO_UDP:
5009 		sport = pd->hdr.udp.uh_sport;
5010 		dport = pd->hdr.udp.uh_dport;
5011 		pi = &V_udbinfo;
5012 		break;
5013 	default:
5014 		return (-1);
5015 	}
5016 	if (pd->dir == PF_IN) {
5017 		saddr = pd->src;
5018 		daddr = pd->dst;
5019 	} else {
5020 		u_int16_t	p;
5021 
5022 		p = sport;
5023 		sport = dport;
5024 		dport = p;
5025 		saddr = pd->dst;
5026 		daddr = pd->src;
5027 	}
5028 	switch (pd->af) {
5029 #ifdef INET
5030 	case AF_INET:
5031 		inp = in_pcblookup_mbuf(pi, saddr->v4, sport, daddr->v4,
5032 		    dport, INPLOOKUP_RLOCKPCB, NULL, pd->m);
5033 		if (inp == NULL) {
5034 			inp = in_pcblookup_mbuf(pi, saddr->v4, sport,
5035 			   daddr->v4, dport, INPLOOKUP_WILDCARD |
5036 			   INPLOOKUP_RLOCKPCB, NULL, pd->m);
5037 			if (inp == NULL)
5038 				return (-1);
5039 		}
5040 		break;
5041 #endif /* INET */
5042 #ifdef INET6
5043 	case AF_INET6:
5044 		inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport, &daddr->v6,
5045 		    dport, INPLOOKUP_RLOCKPCB, NULL, pd->m);
5046 		if (inp == NULL) {
5047 			inp = in6_pcblookup_mbuf(pi, &saddr->v6, sport,
5048 			    &daddr->v6, dport, INPLOOKUP_WILDCARD |
5049 			    INPLOOKUP_RLOCKPCB, NULL, pd->m);
5050 			if (inp == NULL)
5051 				return (-1);
5052 		}
5053 		break;
5054 #endif /* INET6 */
5055 	default:
5056 		unhandled_af(pd->af);
5057 	}
5058 	INP_RLOCK_ASSERT(inp);
5059 	pd->lookup.uid = inp->inp_cred->cr_uid;
5060 	pd->lookup.gid = inp->inp_cred->cr_gid;
5061 	INP_RUNLOCK(inp);
5062 
5063 	return (1);
5064 }
5065 
5066 /* post: r  => (r[0] == type /\ r[1] >= min_typelen >= 2  "validity"
5067  *                      /\ (eoh - r) >= min_typelen >= 2  "safety"  )
5068  *
5069  * warning: r + r[1] may exceed opts bounds for r[1] > min_typelen
5070  */
5071 uint8_t*
5072 pf_find_tcpopt(u_int8_t *opt, u_int8_t *opts, size_t hlen, u_int8_t type,
5073     u_int8_t min_typelen)
5074 {
5075 	uint8_t	*eoh = opts + hlen;
5076 
5077 	if (min_typelen < 2)
5078 		return (NULL);
5079 
5080 	while ((eoh - opt) >= min_typelen) {
5081 		switch (*opt) {
5082 		case TCPOPT_EOL:
5083 			/* FALLTHROUGH - Workaround the failure of some
5084 			 systems to NOP-pad their bzero'd option buffers,
5085 			 producing spurious EOLs */
5086 		case TCPOPT_NOP:
5087 			opt++;
5088 			continue;
5089 		default:
5090 		if (opt[0] == type &&
5091 			    opt[1] >= min_typelen)
5092 			return (opt);
5093 		}
5094 
5095 		opt += MAX(opt[1], 2); /* evade infinite loops */
5096 	}
5097 
5098 	return (NULL);
5099 }
5100 
5101 u_int8_t
5102 pf_get_wscale(struct pf_pdesc *pd)
5103 {
5104 	int	 olen;
5105 	uint8_t	 opts[MAX_TCPOPTLEN], *opt;
5106 	uint8_t	 wscale = 0;
5107 
5108 	olen = (pd->hdr.tcp.th_off << 2) - sizeof(struct tcphdr);
5109 	if (olen < TCPOLEN_WINDOW || !pf_pull_hdr(pd->m,
5110 	    pd->off + sizeof(struct tcphdr), opts, olen, NULL, NULL, pd->af))
5111 		return (0);
5112 
5113 	opt = opts;
5114 	while ((opt = pf_find_tcpopt(opt, opts, olen,
5115 		    TCPOPT_WINDOW, TCPOLEN_WINDOW)) != NULL) {
5116 		wscale = opt[2];
5117 		wscale = MIN(wscale, TCP_MAX_WINSHIFT);
5118 		wscale |= PF_WSCALE_FLAG;
5119 
5120 		opt += opt[1];
5121 	}
5122 
5123 	return (wscale);
5124 }
5125 
5126 u_int16_t
5127 pf_get_mss(struct pf_pdesc *pd)
5128 {
5129 	int		 olen;
5130 	uint8_t		 opts[MAX_TCPOPTLEN], *opt;
5131 	u_int16_t	 mss = V_tcp_mssdflt;
5132 
5133 	olen = (pd->hdr.tcp.th_off << 2) - sizeof(struct tcphdr);
5134 	if (olen < TCPOLEN_MAXSEG || !pf_pull_hdr(pd->m,
5135 	    pd->off + sizeof(struct tcphdr), opts, olen, NULL, NULL, pd->af))
5136 		return (0);
5137 
5138 	opt = opts;
5139 	while ((opt = pf_find_tcpopt(opt, opts, olen,
5140 	    TCPOPT_MAXSEG, TCPOLEN_MAXSEG)) != NULL) {
5141 		memcpy(&mss, (opt + 2), 2);
5142 		mss = ntohs(mss);
5143 		opt += opt[1];
5144 	}
5145 
5146 	return (mss);
5147 }
5148 
5149 static u_int16_t
5150 pf_calc_mss(struct pf_addr *addr, sa_family_t af, int rtableid, u_int16_t offer)
5151 {
5152 	struct nhop_object *nh;
5153 #ifdef INET6
5154 	struct in6_addr		dst6;
5155 	uint32_t		scopeid;
5156 #endif /* INET6 */
5157 	int			 hlen = 0;
5158 	uint16_t		 mss = 0;
5159 
5160 	NET_EPOCH_ASSERT();
5161 
5162 	switch (af) {
5163 #ifdef INET
5164 	case AF_INET:
5165 		hlen = sizeof(struct ip);
5166 		nh = fib4_lookup(rtableid, addr->v4, 0, 0, 0);
5167 		if (nh != NULL)
5168 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
5169 		break;
5170 #endif /* INET */
5171 #ifdef INET6
5172 	case AF_INET6:
5173 		hlen = sizeof(struct ip6_hdr);
5174 		in6_splitscope(&addr->v6, &dst6, &scopeid);
5175 		nh = fib6_lookup(rtableid, &dst6, scopeid, 0, 0);
5176 		if (nh != NULL)
5177 			mss = nh->nh_mtu - hlen - sizeof(struct tcphdr);
5178 		break;
5179 #endif /* INET6 */
5180 	}
5181 
5182 	mss = max(V_tcp_mssdflt, mss);
5183 	mss = min(mss, offer);
5184 	mss = max(mss, 64);		/* sanity - at least max opt space */
5185 	return (mss);
5186 }
5187 
5188 static u_int32_t
5189 pf_tcp_iss(struct pf_pdesc *pd)
5190 {
5191 	SHA512_CTX ctx;
5192 	union {
5193 		uint8_t bytes[SHA512_DIGEST_LENGTH];
5194 		uint32_t words[1];
5195 	} digest;
5196 
5197 	if (V_pf_tcp_secret_init == 0) {
5198 		arc4random_buf(&V_pf_tcp_secret, sizeof(V_pf_tcp_secret));
5199 		SHA512_Init(&V_pf_tcp_secret_ctx);
5200 		SHA512_Update(&V_pf_tcp_secret_ctx, V_pf_tcp_secret,
5201 		    sizeof(V_pf_tcp_secret));
5202 		V_pf_tcp_secret_init = 1;
5203 	}
5204 
5205 	ctx = V_pf_tcp_secret_ctx;
5206 
5207 	SHA512_Update(&ctx, &pd->hdr.tcp.th_sport, sizeof(u_short));
5208 	SHA512_Update(&ctx, &pd->hdr.tcp.th_dport, sizeof(u_short));
5209 	switch (pd->af) {
5210 	case AF_INET6:
5211 		SHA512_Update(&ctx, &pd->src->v6, sizeof(struct in6_addr));
5212 		SHA512_Update(&ctx, &pd->dst->v6, sizeof(struct in6_addr));
5213 		break;
5214 	case AF_INET:
5215 		SHA512_Update(&ctx, &pd->src->v4, sizeof(struct in_addr));
5216 		SHA512_Update(&ctx, &pd->dst->v4, sizeof(struct in_addr));
5217 		break;
5218 	}
5219 	SHA512_Final(digest.bytes, &ctx);
5220 	V_pf_tcp_iss_off += 4096;
5221 #define	ISN_RANDOM_INCREMENT (4096 - 1)
5222 	return (digest.words[0] + (arc4random() & ISN_RANDOM_INCREMENT) +
5223 	    V_pf_tcp_iss_off);
5224 #undef	ISN_RANDOM_INCREMENT
5225 }
5226 
5227 static bool
5228 pf_match_eth_addr(const uint8_t *a, const struct pf_keth_rule_addr *r)
5229 {
5230 	bool match = true;
5231 
5232 	/* Always matches if not set */
5233 	if (! r->isset)
5234 		return (!r->neg);
5235 
5236 	for (int i = 0; i < ETHER_ADDR_LEN; i++) {
5237 		if ((a[i] & r->mask[i]) != (r->addr[i] & r->mask[i])) {
5238 			match = false;
5239 			break;
5240 		}
5241 	}
5242 
5243 	return (match ^ r->neg);
5244 }
5245 
5246 static int
5247 pf_match_eth_tag(struct mbuf *m, struct pf_keth_rule *r, int *tag, int mtag)
5248 {
5249 	if (*tag == -1)
5250 		*tag = mtag;
5251 
5252 	return ((!r->match_tag_not && r->match_tag == *tag) ||
5253 	    (r->match_tag_not && r->match_tag != *tag));
5254 }
5255 
5256 static void
5257 pf_bridge_to(struct ifnet *ifp, struct mbuf *m)
5258 {
5259 	/* If we don't have the interface drop the packet. */
5260 	if (ifp == NULL) {
5261 		m_freem(m);
5262 		return;
5263 	}
5264 
5265 	switch (ifp->if_type) {
5266 	case IFT_ETHER:
5267 	case IFT_XETHER:
5268 	case IFT_L2VLAN:
5269 	case IFT_BRIDGE:
5270 	case IFT_IEEE8023ADLAG:
5271 		break;
5272 	default:
5273 		m_freem(m);
5274 		return;
5275 	}
5276 
5277 	ifp->if_transmit(ifp, m);
5278 }
5279 
5280 static int
5281 pf_test_eth_rule(int dir, struct pfi_kkif *kif, struct mbuf **m0)
5282 {
5283 #ifdef INET
5284 	struct ip ip;
5285 #endif /* INET */
5286 #ifdef INET6
5287 	struct ip6_hdr ip6;
5288 #endif /* INET6 */
5289 	struct mbuf *m = *m0;
5290 	struct ether_header *e;
5291 	struct pf_keth_rule *r, *rm, *a = NULL;
5292 	struct pf_keth_ruleset *ruleset = NULL;
5293 	struct pf_mtag *mtag;
5294 	struct pf_keth_ruleq *rules;
5295 	struct pf_addr *src = NULL, *dst = NULL;
5296 	struct pfi_kkif *bridge_to;
5297 	sa_family_t af = 0;
5298 	uint16_t proto;
5299 	int asd = 0, match = 0;
5300 	int tag = -1;
5301 	uint8_t action;
5302 	struct pf_keth_anchor_stackframe	anchor_stack[PF_ANCHOR_STACK_MAX];
5303 
5304 	MPASS(kif->pfik_ifp->if_vnet == curvnet);
5305 	NET_EPOCH_ASSERT();
5306 
5307 	PF_RULES_RLOCK_TRACKER;
5308 
5309 	SDT_PROBE3(pf, eth, test_rule, entry, dir, kif->pfik_ifp, m);
5310 
5311 	mtag = pf_find_mtag(m);
5312 	if (mtag != NULL && mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
5313 		/* Dummynet re-injects packets after they've
5314 		 * completed their delay. We've already
5315 		 * processed them, so pass unconditionally. */
5316 
5317 		/* But only once. We may see the packet multiple times (e.g.
5318 		 * PFIL_IN/PFIL_OUT). */
5319 		pf_dummynet_flag_remove(m, mtag);
5320 
5321 		return (PF_PASS);
5322 	}
5323 
5324 	if (__predict_false(m->m_len < sizeof(struct ether_header)) &&
5325 	    (m = *m0 = m_pullup(*m0, sizeof(struct ether_header))) == NULL) {
5326 		DPFPRINTF(PF_DEBUG_URGENT,
5327 		    "%s: m_len < sizeof(struct ether_header)"
5328 		     ", pullup failed", __func__);
5329 		return (PF_DROP);
5330 	}
5331 	e = mtod(m, struct ether_header *);
5332 	proto = ntohs(e->ether_type);
5333 
5334 	switch (proto) {
5335 #ifdef INET
5336 	case ETHERTYPE_IP: {
5337 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
5338 		    sizeof(ip)))
5339 			return (PF_DROP);
5340 
5341 		af = AF_INET;
5342 		m_copydata(m, sizeof(struct ether_header), sizeof(ip),
5343 		    (caddr_t)&ip);
5344 		src = (struct pf_addr *)&ip.ip_src;
5345 		dst = (struct pf_addr *)&ip.ip_dst;
5346 		break;
5347 	}
5348 #endif /* INET */
5349 #ifdef INET6
5350 	case ETHERTYPE_IPV6: {
5351 		if (m_length(m, NULL) < (sizeof(struct ether_header) +
5352 		    sizeof(ip6)))
5353 			return (PF_DROP);
5354 
5355 		af = AF_INET6;
5356 		m_copydata(m, sizeof(struct ether_header), sizeof(ip6),
5357 		    (caddr_t)&ip6);
5358 		src = (struct pf_addr *)&ip6.ip6_src;
5359 		dst = (struct pf_addr *)&ip6.ip6_dst;
5360 		break;
5361 	}
5362 #endif /* INET6 */
5363 	}
5364 
5365 	PF_RULES_RLOCK();
5366 
5367 	ruleset = V_pf_keth;
5368 	rules = atomic_load_ptr(&ruleset->active.rules);
5369 	for (r = TAILQ_FIRST(rules), rm = NULL; r != NULL;) {
5370 		counter_u64_add(r->evaluations, 1);
5371 		SDT_PROBE2(pf, eth, test_rule, test, r->nr, r);
5372 
5373 		if (pfi_kkif_match(r->kif, kif) == r->ifnot) {
5374 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5375 			    "kif");
5376 			r = r->skip[PFE_SKIP_IFP].ptr;
5377 		}
5378 		else if (r->direction && r->direction != dir) {
5379 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5380 			    "dir");
5381 			r = r->skip[PFE_SKIP_DIR].ptr;
5382 		}
5383 		else if (r->proto && r->proto != proto) {
5384 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5385 			    "proto");
5386 			r = r->skip[PFE_SKIP_PROTO].ptr;
5387 		}
5388 		else if (! pf_match_eth_addr(e->ether_shost, &r->src)) {
5389 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5390 			    "src");
5391 			r = r->skip[PFE_SKIP_SRC_ADDR].ptr;
5392 		}
5393 		else if (! pf_match_eth_addr(e->ether_dhost, &r->dst)) {
5394 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5395 			    "dst");
5396 			r = r->skip[PFE_SKIP_DST_ADDR].ptr;
5397 		}
5398 		else if (src != NULL && PF_MISMATCHAW(&r->ipsrc.addr, src, af,
5399 		    r->ipsrc.neg, kif, M_GETFIB(m))) {
5400 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5401 			    "ip_src");
5402 			r = r->skip[PFE_SKIP_SRC_IP_ADDR].ptr;
5403 		}
5404 		else if (dst != NULL && PF_MISMATCHAW(&r->ipdst.addr, dst, af,
5405 		    r->ipdst.neg, kif, M_GETFIB(m))) {
5406 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5407 			    "ip_dst");
5408 			r = r->skip[PFE_SKIP_DST_IP_ADDR].ptr;
5409 		}
5410 		else if (r->match_tag && !pf_match_eth_tag(m, r, &tag,
5411 		    mtag ? mtag->tag : 0)) {
5412 			SDT_PROBE3(pf, eth, test_rule, mismatch, r->nr, r,
5413 			    "match_tag");
5414 			r = TAILQ_NEXT(r, entries);
5415 		}
5416 		else {
5417 			if (r->tag)
5418 				tag = r->tag;
5419 			if (r->anchor == NULL) {
5420 				/* Rule matches */
5421 				rm = r;
5422 
5423 				SDT_PROBE2(pf, eth, test_rule, match, r->nr, r);
5424 
5425 				if (r->quick)
5426 					break;
5427 
5428 				r = TAILQ_NEXT(r, entries);
5429 			} else {
5430 				pf_step_into_keth_anchor(anchor_stack, &asd,
5431 				    &ruleset, &r, &a, &match);
5432 			}
5433 		}
5434 		if (r == NULL && pf_step_out_of_keth_anchor(anchor_stack, &asd,
5435 		    &ruleset, &r, &a, &match))
5436 			break;
5437 	}
5438 
5439 	r = rm;
5440 
5441 	SDT_PROBE2(pf, eth, test_rule, final_match, (r != NULL ? r->nr : -1), r);
5442 
5443 	/* Default to pass. */
5444 	if (r == NULL) {
5445 		PF_RULES_RUNLOCK();
5446 		return (PF_PASS);
5447 	}
5448 
5449 	/* Execute action. */
5450 	counter_u64_add(r->packets[dir == PF_OUT], 1);
5451 	counter_u64_add(r->bytes[dir == PF_OUT], m_length(m, NULL));
5452 	pf_update_timestamp(r);
5453 
5454 	/* Shortcut. Don't tag if we're just going to drop anyway. */
5455 	if (r->action == PF_DROP) {
5456 		PF_RULES_RUNLOCK();
5457 		return (PF_DROP);
5458 	}
5459 
5460 	if (tag > 0) {
5461 		if (mtag == NULL)
5462 			mtag = pf_get_mtag(m);
5463 		if (mtag == NULL) {
5464 			PF_RULES_RUNLOCK();
5465 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
5466 			return (PF_DROP);
5467 		}
5468 		mtag->tag = tag;
5469 	}
5470 
5471 	if (r->qid != 0) {
5472 		if (mtag == NULL)
5473 			mtag = pf_get_mtag(m);
5474 		if (mtag == NULL) {
5475 			PF_RULES_RUNLOCK();
5476 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
5477 			return (PF_DROP);
5478 		}
5479 		mtag->qid = r->qid;
5480 	}
5481 
5482 	action = r->action;
5483 	bridge_to = r->bridge_to;
5484 
5485 	/* Dummynet */
5486 	if (r->dnpipe) {
5487 		struct ip_fw_args dnflow;
5488 
5489 		/* Drop packet if dummynet is not loaded. */
5490 		if (ip_dn_io_ptr == NULL) {
5491 			PF_RULES_RUNLOCK();
5492 			m_freem(m);
5493 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
5494 			return (PF_DROP);
5495 		}
5496 		if (mtag == NULL)
5497 			mtag = pf_get_mtag(m);
5498 		if (mtag == NULL) {
5499 			PF_RULES_RUNLOCK();
5500 			counter_u64_add(V_pf_status.counters[PFRES_MEMORY], 1);
5501 			return (PF_DROP);
5502 		}
5503 
5504 		bzero(&dnflow, sizeof(dnflow));
5505 
5506 		/* We don't have port numbers here, so we set 0.  That means
5507 		 * that we'll be somewhat limited in distinguishing flows (i.e.
5508 		 * only based on IP addresses, not based on port numbers), but
5509 		 * it's better than nothing. */
5510 		dnflow.f_id.dst_port = 0;
5511 		dnflow.f_id.src_port = 0;
5512 		dnflow.f_id.proto = 0;
5513 
5514 		dnflow.rule.info = r->dnpipe;
5515 		dnflow.rule.info |= IPFW_IS_DUMMYNET;
5516 		if (r->dnflags & PFRULE_DN_IS_PIPE)
5517 			dnflow.rule.info |= IPFW_IS_PIPE;
5518 
5519 		dnflow.f_id.extra = dnflow.rule.info;
5520 
5521 		dnflow.flags = dir == PF_IN ? IPFW_ARGS_IN : IPFW_ARGS_OUT;
5522 		dnflow.flags |= IPFW_ARGS_ETHER;
5523 		dnflow.ifp = kif->pfik_ifp;
5524 
5525 		switch (af) {
5526 		case AF_INET:
5527 			dnflow.f_id.addr_type = 4;
5528 			dnflow.f_id.src_ip = src->v4.s_addr;
5529 			dnflow.f_id.dst_ip = dst->v4.s_addr;
5530 			break;
5531 		case AF_INET6:
5532 			dnflow.flags |= IPFW_ARGS_IP6;
5533 			dnflow.f_id.addr_type = 6;
5534 			dnflow.f_id.src_ip6 = src->v6;
5535 			dnflow.f_id.dst_ip6 = dst->v6;
5536 			break;
5537 		}
5538 
5539 		PF_RULES_RUNLOCK();
5540 
5541 		mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
5542 		ip_dn_io_ptr(m0, &dnflow);
5543 		if (*m0 != NULL)
5544 			pf_dummynet_flag_remove(m, mtag);
5545 	} else {
5546 		PF_RULES_RUNLOCK();
5547 	}
5548 
5549 	if (action == PF_PASS && bridge_to) {
5550 		pf_bridge_to(bridge_to->pfik_ifp, *m0);
5551 		*m0 = NULL; /* We've eaten the packet. */
5552 	}
5553 
5554 	return (action);
5555 }
5556 
5557 #define PF_TEST_ATTRIB(t, a)		\
5558 	if (t) {			\
5559 		r = a;			\
5560 		continue;		\
5561 	} else do {			\
5562 	} while (0)
5563 
5564 static __inline u_short
5565 pf_rule_apply_nat(struct pf_test_ctx *ctx, struct pf_krule *r)
5566 {
5567 	struct pf_pdesc	*pd = ctx->pd;
5568 	u_short		 transerror;
5569 	u_int8_t	 nat_action;
5570 
5571 	if (r->rule_flag & PFRULE_AFTO) {
5572 		/* Don't translate if there was an old style NAT rule */
5573 		if (ctx->nr != NULL)
5574 			return (PFRES_TRANSLATE);
5575 
5576 		/* pass af-to rules, unsupported on match rules */
5577 		KASSERT(r->action != PF_MATCH, ("%s: af-to on match rule", __func__));
5578 		/* XXX I can imagine scenarios where we have both NAT and RDR source tracking */
5579 		ctx->nat_pool = &(r->nat);
5580 		ctx->nr = r;
5581 		pd->naf = r->naf;
5582 		if (pf_get_transaddr_af(ctx->nr, pd) == -1) {
5583 			return (PFRES_TRANSLATE);
5584 		}
5585 		return (PFRES_MATCH);
5586 	} else if (r->rdr.cur || r->nat.cur) {
5587 		/* Don't translate if there was an old style NAT rule */
5588 		if (ctx->nr != NULL)
5589 			return (PFRES_TRANSLATE);
5590 
5591 		/* match/pass nat-to/rdr-to rules */
5592 		ctx->nr = r;
5593 		if (r->nat.cur) {
5594 			nat_action = PF_NAT;
5595 			ctx->nat_pool = &(r->nat);
5596 		} else {
5597 			nat_action = PF_RDR;
5598 			ctx->nat_pool = &(r->rdr);
5599 		}
5600 
5601 		transerror = pf_get_transaddr(ctx, ctx->nr,
5602 		    nat_action, ctx->nat_pool);
5603 		if (transerror == PFRES_MATCH) {
5604 			ctx->rewrite += pf_translate_compat(ctx);
5605 			return(PFRES_MATCH);
5606 		}
5607 		return (transerror);
5608 	}
5609 
5610 	return (PFRES_MAX);
5611 }
5612 
5613 enum pf_test_status
5614 pf_match_rule(struct pf_test_ctx *ctx, struct pf_kruleset *ruleset)
5615 {
5616 	struct pf_krule_item	*ri;
5617 	struct pf_krule		*r;
5618 	struct pf_krule		*save_a;
5619 	struct pf_kruleset	*save_aruleset;
5620 	struct pf_pdesc		*pd = ctx->pd;
5621 	u_short			 transerror;
5622 
5623 	r = TAILQ_FIRST(ruleset->rules[PF_RULESET_FILTER].active.ptr);
5624 	while (r != NULL) {
5625 		if (ctx->pd->related_rule) {
5626 			*ctx->rm = ctx->pd->related_rule;
5627 			break;
5628 		}
5629 		pf_counter_u64_add(&r->evaluations, 1);
5630 		PF_TEST_ATTRIB(pfi_kkif_match(r->kif, pd->kif) == r->ifnot,
5631 			r->skip[PF_SKIP_IFP]);
5632 		PF_TEST_ATTRIB(r->direction && r->direction != pd->dir,
5633 			r->skip[PF_SKIP_DIR]);
5634 		PF_TEST_ATTRIB(r->af && r->af != pd->af,
5635 			r->skip[PF_SKIP_AF]);
5636 		PF_TEST_ATTRIB(r->proto && r->proto != pd->proto,
5637 			r->skip[PF_SKIP_PROTO]);
5638 		PF_TEST_ATTRIB(PF_MISMATCHAW(&r->src.addr, &pd->nsaddr, pd->naf,
5639 		    r->src.neg, pd->kif, M_GETFIB(pd->m)),
5640 			r->skip[PF_SKIP_SRC_ADDR]);
5641 		PF_TEST_ATTRIB(PF_MISMATCHAW(&r->dst.addr, &pd->ndaddr, pd->af,
5642 		    r->dst.neg, NULL, M_GETFIB(pd->m)),
5643 			r->skip[PF_SKIP_DST_ADDR]);
5644 		switch (pd->virtual_proto) {
5645 		case PF_VPROTO_FRAGMENT:
5646 			/* tcp/udp only. port_op always 0 in other cases */
5647 			PF_TEST_ATTRIB((r->src.port_op || r->dst.port_op),
5648 				TAILQ_NEXT(r, entries));
5649 			PF_TEST_ATTRIB((pd->proto == IPPROTO_TCP && r->flagset),
5650 				TAILQ_NEXT(r, entries));
5651 			/* icmp only. type/code always 0 in other cases */
5652 			PF_TEST_ATTRIB((r->type || r->code),
5653 				TAILQ_NEXT(r, entries));
5654 			/* tcp/udp only. {uid|gid}.op always 0 in other cases */
5655 			PF_TEST_ATTRIB((r->gid.op || r->uid.op),
5656 				TAILQ_NEXT(r, entries));
5657 			break;
5658 
5659 		case IPPROTO_TCP:
5660 			PF_TEST_ATTRIB((r->flagset & tcp_get_flags(ctx->th))
5661 			    != r->flags,
5662 				TAILQ_NEXT(r, entries));
5663 			/* FALLTHROUGH */
5664 		case IPPROTO_SCTP:
5665 		case IPPROTO_UDP:
5666 			/* tcp/udp only. port_op always 0 in other cases */
5667 			PF_TEST_ATTRIB(r->src.port_op && !pf_match_port(r->src.port_op,
5668 			    r->src.port[0], r->src.port[1], pd->nsport),
5669 				r->skip[PF_SKIP_SRC_PORT]);
5670 			/* tcp/udp only. port_op always 0 in other cases */
5671 			PF_TEST_ATTRIB(r->dst.port_op && !pf_match_port(r->dst.port_op,
5672 			    r->dst.port[0], r->dst.port[1], pd->ndport),
5673 				r->skip[PF_SKIP_DST_PORT]);
5674 			/* tcp/udp only. uid.op always 0 in other cases */
5675 			PF_TEST_ATTRIB(r->uid.op && (pd->lookup.done || (pd->lookup.done =
5676 			    pf_socket_lookup(pd), 1)) &&
5677 			    !pf_match_uid(r->uid.op, r->uid.uid[0], r->uid.uid[1],
5678 			    pd->lookup.uid),
5679 				TAILQ_NEXT(r, entries));
5680 			/* tcp/udp only. gid.op always 0 in other cases */
5681 			PF_TEST_ATTRIB(r->gid.op && (pd->lookup.done || (pd->lookup.done =
5682 			    pf_socket_lookup(pd), 1)) &&
5683 			    !pf_match_gid(r->gid.op, r->gid.gid[0], r->gid.gid[1],
5684 			    pd->lookup.gid),
5685 				TAILQ_NEXT(r, entries));
5686 			break;
5687 
5688 		case IPPROTO_ICMP:
5689 		case IPPROTO_ICMPV6:
5690 			/* icmp only. type always 0 in other cases */
5691 			PF_TEST_ATTRIB(r->type && r->type != ctx->icmptype + 1,
5692 				TAILQ_NEXT(r, entries));
5693 			/* icmp only. type always 0 in other cases */
5694 			PF_TEST_ATTRIB(r->code && r->code != ctx->icmpcode + 1,
5695 				TAILQ_NEXT(r, entries));
5696 			break;
5697 
5698 		default:
5699 			break;
5700 		}
5701 		PF_TEST_ATTRIB(r->tos && !(r->tos == pd->tos),
5702 			TAILQ_NEXT(r, entries));
5703 		PF_TEST_ATTRIB(r->prio &&
5704 		    !pf_match_ieee8021q_pcp(r->prio, pd->m),
5705 			TAILQ_NEXT(r, entries));
5706 		PF_TEST_ATTRIB(r->prob &&
5707 		    r->prob <= arc4random(),
5708 			TAILQ_NEXT(r, entries));
5709 		PF_TEST_ATTRIB(r->match_tag && !pf_match_tag(pd->m, r,
5710 		    &ctx->tag, pd->pf_mtag ? pd->pf_mtag->tag : 0),
5711 			TAILQ_NEXT(r, entries));
5712 		PF_TEST_ATTRIB((r->rcv_kif && pf_match_rcvif(pd->m, r) ==
5713 		   r->rcvifnot),
5714 			TAILQ_NEXT(r, entries));
5715 		PF_TEST_ATTRIB((r->rule_flag & PFRULE_FRAGMENT &&
5716 		    pd->virtual_proto != PF_VPROTO_FRAGMENT),
5717 			TAILQ_NEXT(r, entries));
5718 		PF_TEST_ATTRIB(r->os_fingerprint != PF_OSFP_ANY &&
5719 		    (pd->virtual_proto != IPPROTO_TCP || !pf_osfp_match(
5720 		    pf_osfp_fingerprint(pd, ctx->th),
5721 		    r->os_fingerprint)),
5722 			TAILQ_NEXT(r, entries));
5723 		/* must be last! */
5724 		if (r->pktrate.limit) {
5725 			PF_TEST_ATTRIB((pf_check_threshold(&r->pktrate)),
5726 			    TAILQ_NEXT(r, entries));
5727 		}
5728 		/* FALLTHROUGH */
5729 		if (r->tag)
5730 			ctx->tag = r->tag;
5731 		if (r->anchor == NULL) {
5732 			if (r->action == PF_MATCH) {
5733 				/*
5734 				 * Apply translations before increasing counters,
5735 				 * in case it fails.
5736 				 */
5737 				transerror = pf_rule_apply_nat(ctx, r);
5738 				switch (transerror) {
5739 				case PFRES_MATCH:
5740 					/* Translation action found in rule and applied successfully */
5741 				case PFRES_MAX:
5742 					/* No translation action found in rule */
5743 					break;
5744 				default:
5745 					/* Translation action found in rule but failed to apply */
5746 					REASON_SET(&ctx->reason, transerror);
5747 					return (PF_TEST_FAIL);
5748 				}
5749 				ri = malloc(sizeof(struct pf_krule_item), M_PF_RULE_ITEM, M_NOWAIT | M_ZERO);
5750 				if (ri == NULL) {
5751 					REASON_SET(&ctx->reason, PFRES_MEMORY);
5752 					return (PF_TEST_FAIL);
5753 				}
5754 				ri->r = r;
5755 				SLIST_INSERT_HEAD(&ctx->rules, ri, entry);
5756 				pf_counter_u64_critical_enter();
5757 				pf_counter_u64_add_protected(&r->packets[pd->dir == PF_OUT], 1);
5758 				pf_counter_u64_add_protected(&r->bytes[pd->dir == PF_OUT], pd->tot_len);
5759 				pf_counter_u64_critical_exit();
5760 				pf_rule_to_actions(r, &pd->act);
5761 				if (r->log)
5762 					PFLOG_PACKET(r->action, PFRES_MATCH, r,
5763 					    ctx->a, ruleset, pd, 1, NULL);
5764 			} else {
5765 				/*
5766 				 * found matching r
5767 				 */
5768 				*ctx->rm = r;
5769 				/*
5770 				 * anchor, with ruleset, where r belongs to
5771 				 */
5772 				*ctx->am = ctx->a;
5773 				/*
5774 				 * ruleset where r belongs to
5775 				 */
5776 				*ctx->rsm = ruleset;
5777 				/*
5778 				 * ruleset, where anchor belongs to.
5779 				 */
5780 				ctx->arsm = ctx->aruleset;
5781 			}
5782 			if (pd->act.log & PF_LOG_MATCHES)
5783 				pf_log_matches(pd, r, ctx->a, ruleset, &ctx->rules);
5784 			if (r->quick) {
5785 				ctx->test_status = PF_TEST_QUICK;
5786 				break;
5787 			}
5788 		} else {
5789 			save_a = ctx->a;
5790 			save_aruleset = ctx->aruleset;
5791 
5792 			ctx->a = r;			/* remember anchor */
5793 			ctx->aruleset = ruleset;	/* and its ruleset */
5794 			if (ctx->a->quick)
5795 				ctx->test_status = PF_TEST_QUICK;
5796 			/*
5797 			 * Note: we don't need to restore if we are not going
5798 			 * to continue with ruleset evaluation.
5799 			 */
5800 			if (pf_step_into_anchor(ctx, r) != PF_TEST_OK) {
5801 				break;
5802 			}
5803 			ctx->a = save_a;
5804 			ctx->aruleset = save_aruleset;
5805 		}
5806 		r = TAILQ_NEXT(r, entries);
5807 	}
5808 
5809 	return (ctx->test_status);
5810 }
5811 
5812 static int
5813 pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm,
5814     struct pf_pdesc *pd, struct pf_krule **am,
5815     struct pf_kruleset **rsm, u_short *reason, struct inpcb *inp)
5816 {
5817 	struct pf_krule		*r = NULL;
5818 	struct pf_kruleset	*ruleset = NULL;
5819 	struct pf_krule_item	*ri;
5820 	struct pf_test_ctx	 ctx;
5821 	u_short			 transerror;
5822 	int			 action = PF_PASS;
5823 	u_int16_t		 bproto_sum = 0, bip_sum = 0;
5824 	enum pf_test_status	 rv;
5825 
5826 	PF_RULES_RASSERT();
5827 
5828 	bzero(&ctx, sizeof(ctx));
5829 	ctx.tag = -1;
5830 	ctx.pd = pd;
5831 	ctx.rm = rm;
5832 	ctx.am = am;
5833 	ctx.rsm = rsm;
5834 	ctx.th = &pd->hdr.tcp;
5835 	ctx.reason = *reason;
5836 	SLIST_INIT(&ctx.rules);
5837 
5838 	pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
5839 	pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
5840 
5841 	if (inp != NULL) {
5842 		INP_LOCK_ASSERT(inp);
5843 		pd->lookup.uid = inp->inp_cred->cr_uid;
5844 		pd->lookup.gid = inp->inp_cred->cr_gid;
5845 		pd->lookup.done = 1;
5846 	}
5847 
5848 	if (pd->ip_sum)
5849 		bip_sum = *pd->ip_sum;
5850 
5851 	switch (pd->virtual_proto) {
5852 	case IPPROTO_TCP:
5853 		bproto_sum = ctx.th->th_sum;
5854 		pd->nsport = ctx.th->th_sport;
5855 		pd->ndport = ctx.th->th_dport;
5856 		break;
5857 	case IPPROTO_UDP:
5858 		bproto_sum = pd->hdr.udp.uh_sum;
5859 		pd->nsport = pd->hdr.udp.uh_sport;
5860 		pd->ndport = pd->hdr.udp.uh_dport;
5861 		break;
5862 	case IPPROTO_SCTP:
5863 		pd->nsport = pd->hdr.sctp.src_port;
5864 		pd->ndport = pd->hdr.sctp.dest_port;
5865 		break;
5866 #ifdef INET
5867 	case IPPROTO_ICMP:
5868 		MPASS(pd->af == AF_INET);
5869 		ctx.icmptype = pd->hdr.icmp.icmp_type;
5870 		ctx.icmpcode = pd->hdr.icmp.icmp_code;
5871 		ctx.state_icmp = pf_icmp_mapping(pd, ctx.icmptype,
5872 		    &ctx.icmp_dir, &ctx.virtual_id, &ctx.virtual_type);
5873 		if (ctx.icmp_dir == PF_IN) {
5874 			pd->nsport = ctx.virtual_id;
5875 			pd->ndport = ctx.virtual_type;
5876 		} else {
5877 			pd->nsport = ctx.virtual_type;
5878 			pd->ndport = ctx.virtual_id;
5879 		}
5880 		break;
5881 #endif /* INET */
5882 #ifdef INET6
5883 	case IPPROTO_ICMPV6:
5884 		MPASS(pd->af == AF_INET6);
5885 		ctx.icmptype = pd->hdr.icmp6.icmp6_type;
5886 		ctx.icmpcode = pd->hdr.icmp6.icmp6_code;
5887 		ctx.state_icmp = pf_icmp_mapping(pd, ctx.icmptype,
5888 		    &ctx.icmp_dir, &ctx.virtual_id, &ctx.virtual_type);
5889 		if (ctx.icmp_dir == PF_IN) {
5890 			pd->nsport = ctx.virtual_id;
5891 			pd->ndport = ctx.virtual_type;
5892 		} else {
5893 			pd->nsport = ctx.virtual_type;
5894 			pd->ndport = ctx.virtual_id;
5895 		}
5896 
5897 		break;
5898 #endif /* INET6 */
5899 	default:
5900 		pd->nsport = pd->ndport = 0;
5901 		break;
5902 	}
5903 	pd->osport = pd->nsport;
5904 	pd->odport = pd->ndport;
5905 
5906 	/* check packet for BINAT/NAT/RDR */
5907 	transerror = pf_get_translation(&ctx);
5908 	switch (transerror) {
5909 	default:
5910 		/* A translation error occurred. */
5911 		REASON_SET(&ctx.reason, transerror);
5912 		goto cleanup;
5913 	case PFRES_MAX:
5914 		/* No match. */
5915 		break;
5916 	case PFRES_MATCH:
5917 		KASSERT(ctx.sk != NULL, ("%s: null sk", __func__));
5918 		KASSERT(ctx.nk != NULL, ("%s: null nk", __func__));
5919 		if (ctx.nr->log) {
5920 			PFLOG_PACKET(ctx.nr->action, PFRES_MATCH, ctx.nr, ctx.a,
5921 			    ruleset, pd, 1, NULL);
5922 		}
5923 
5924 		ctx.rewrite += pf_translate_compat(&ctx);
5925 		ctx.nat_pool = &(ctx.nr->rdr);
5926 	}
5927 
5928 	ruleset = &pf_main_ruleset;
5929 	rv = pf_match_rule(&ctx, ruleset);
5930 	if (rv == PF_TEST_FAIL) {
5931 		/*
5932 		 * Reason has been set in pf_match_rule() already.
5933 		 */
5934 		goto cleanup;
5935 	}
5936 
5937 	r = *ctx.rm;			/* matching rule */
5938 	ctx.a = *ctx.am;		/* rule that defines an anchor containing 'r' */
5939 	ruleset = *ctx.rsm;		/* ruleset of the anchor defined by the rule 'a' */
5940 	ctx.aruleset = ctx.arsm;	/* ruleset of the 'a' rule itself */
5941 
5942 	REASON_SET(&ctx.reason, PFRES_MATCH);
5943 
5944 	/* apply actions for last matching pass/block rule */
5945 	pf_rule_to_actions(r, &pd->act);
5946 	transerror = pf_rule_apply_nat(&ctx, r);
5947 	switch (transerror) {
5948 	case PFRES_MATCH:
5949 		/* Translation action found in rule and applied successfully */
5950 	case PFRES_MAX:
5951 		/* No translation action found in rule */
5952 		break;
5953 	default:
5954 		/* Translation action found in rule but failed to apply */
5955 		REASON_SET(&ctx.reason, transerror);
5956 		goto cleanup;
5957 	}
5958 
5959 	if (r->log) {
5960 		if (ctx.rewrite)
5961 			m_copyback(pd->m, pd->off, pd->hdrlen, pd->hdr.any);
5962 		PFLOG_PACKET(r->action, ctx.reason, r, ctx.a, ruleset, pd, 1, NULL);
5963 	}
5964 	if (pd->act.log & PF_LOG_MATCHES)
5965 		pf_log_matches(pd, r, ctx.a, ruleset, &ctx.rules);
5966 	if (pd->virtual_proto != PF_VPROTO_FRAGMENT &&
5967 	   (r->action == PF_DROP) &&
5968 	    ((r->rule_flag & PFRULE_RETURNRST) ||
5969 	    (r->rule_flag & PFRULE_RETURNICMP) ||
5970 	    (r->rule_flag & PFRULE_RETURN))) {
5971 		pf_return(r, ctx.nr, pd, ctx.th, bproto_sum,
5972 		    bip_sum, &ctx.reason, r->rtableid);
5973 	}
5974 
5975 	if (r->action == PF_DROP)
5976 		goto cleanup;
5977 
5978 	if (ctx.tag > 0 && pf_tag_packet(pd, ctx.tag)) {
5979 		REASON_SET(&ctx.reason, PFRES_MEMORY);
5980 		goto cleanup;
5981 	}
5982 	if (pd->act.rtableid >= 0)
5983 		M_SETFIB(pd->m, pd->act.rtableid);
5984 
5985 	if (r->rt) {
5986 		/*
5987 		 * Set act.rt here instead of in pf_rule_to_actions() because
5988 		 * it is applied only from the last pass rule. For rules
5989 		 * with the prefer-ipv6-nexthop option act.rt_af is a hint
5990 		 * about AF of the forwarded packet and might be changed.
5991 		 */
5992 		pd->act.rt = r->rt;
5993 		if (r->rt == PF_REPLYTO)
5994 			pd->act.rt_af = pd->af;
5995 		else
5996 			pd->act.rt_af = pd->naf;
5997 		if ((transerror = pf_map_addr_sn(pd->af, r, pd->src,
5998 		    &pd->act.rt_addr, &pd->act.rt_af, &pd->act.rt_kif, NULL,
5999 		    &(r->route), PF_SN_ROUTE)) != PFRES_MATCH) {
6000 			REASON_SET(&ctx.reason, transerror);
6001 			goto cleanup;
6002 		}
6003 	}
6004 
6005 	if (pd->virtual_proto != PF_VPROTO_FRAGMENT &&
6006 	   (!ctx.state_icmp && (r->keep_state || ctx.nr != NULL ||
6007 	    (pd->flags & PFDESC_TCP_NORM)))) {
6008 		bool nat64;
6009 
6010 		action = pf_create_state(r, &ctx, sm, bproto_sum, bip_sum);
6011 		ctx.sk = ctx.nk = NULL;
6012 		if (action != PF_PASS) {
6013 			pf_udp_mapping_release(ctx.udp_mapping);
6014 			if (r->log || (ctx.nr != NULL && ctx.nr->log) ||
6015 			    ctx.reason == PFRES_MEMORY)
6016 				pd->act.log |= PF_LOG_FORCE;
6017 			if (action == PF_DROP &&
6018 			    (r->rule_flag & PFRULE_RETURN))
6019 				pf_return(r, ctx.nr, pd, ctx.th,
6020 				    bproto_sum, bip_sum, &ctx.reason,
6021 				    pd->act.rtableid);
6022 			*reason = ctx.reason;
6023 			return (action);
6024 		}
6025 
6026 		nat64 = pd->af != pd->naf;
6027 		if (nat64) {
6028 			int			 ret;
6029 
6030 			if (ctx.sk == NULL)
6031 				ctx.sk = (*sm)->key[pd->dir == PF_IN ? PF_SK_STACK : PF_SK_WIRE];
6032 			if (ctx.nk == NULL)
6033 				ctx.nk = (*sm)->key[pd->dir == PF_IN ? PF_SK_WIRE : PF_SK_STACK];
6034 
6035 			if (pd->dir == PF_IN) {
6036 				ret = pf_translate(pd, &ctx.sk->addr[pd->didx],
6037 				    ctx.sk->port[pd->didx], &ctx.sk->addr[pd->sidx],
6038 				    ctx.sk->port[pd->sidx], ctx.virtual_type,
6039 				    ctx.icmp_dir);
6040 			} else {
6041 				ret = pf_translate(pd, &ctx.sk->addr[pd->sidx],
6042 				    ctx.sk->port[pd->sidx], &ctx.sk->addr[pd->didx],
6043 				    ctx.sk->port[pd->didx], ctx.virtual_type,
6044 				    ctx.icmp_dir);
6045 			}
6046 
6047 			if (ret < 0)
6048 				goto cleanup;
6049 
6050 			ctx.rewrite += ret;
6051 
6052 			if (ctx.rewrite && ctx.sk->af != ctx.nk->af)
6053 				action = PF_AFRT;
6054 		}
6055 	} else {
6056 		while ((ri = SLIST_FIRST(&ctx.rules))) {
6057 			SLIST_REMOVE_HEAD(&ctx.rules, entry);
6058 			free(ri, M_PF_RULE_ITEM);
6059 		}
6060 
6061 		uma_zfree(V_pf_state_key_z, ctx.sk);
6062 		uma_zfree(V_pf_state_key_z, ctx.nk);
6063 		ctx.sk = ctx.nk = NULL;
6064 		pf_udp_mapping_release(ctx.udp_mapping);
6065 	}
6066 
6067 	/* copy back packet headers if we performed NAT operations */
6068 	if (ctx.rewrite)
6069 		m_copyback(pd->m, pd->off, pd->hdrlen, pd->hdr.any);
6070 
6071 	if (*sm != NULL && !((*sm)->state_flags & PFSTATE_NOSYNC) &&
6072 	    pd->dir == PF_OUT &&
6073 	    V_pfsync_defer_ptr != NULL && V_pfsync_defer_ptr(*sm, pd->m)) {
6074 		/*
6075 		 * We want the state created, but we dont
6076 		 * want to send this in case a partner
6077 		 * firewall has to know about it to allow
6078 		 * replies through it.
6079 		 */
6080 		*reason = ctx.reason;
6081 		return (PF_DEFER);
6082 	}
6083 
6084 	*reason = ctx.reason;
6085 	return (action);
6086 
6087 cleanup:
6088 	while ((ri = SLIST_FIRST(&ctx.rules))) {
6089 		SLIST_REMOVE_HEAD(&ctx.rules, entry);
6090 		free(ri, M_PF_RULE_ITEM);
6091 	}
6092 
6093 	uma_zfree(V_pf_state_key_z, ctx.sk);
6094 	uma_zfree(V_pf_state_key_z, ctx.nk);
6095 	pf_udp_mapping_release(ctx.udp_mapping);
6096 	*reason = ctx.reason;
6097 
6098 	return (PF_DROP);
6099 }
6100 
6101 static int
6102 pf_create_state(struct pf_krule *r, struct pf_test_ctx *ctx,
6103     struct pf_kstate **sm, u_int16_t bproto_sum, u_int16_t bip_sum)
6104 {
6105 	struct pf_pdesc		*pd = ctx->pd;
6106 	struct pf_kstate	*s = NULL;
6107 	struct pf_ksrc_node	*sns[PF_SN_MAX] = { NULL };
6108 	/*
6109 	 * XXXKS: The hash for PF_SN_LIMIT and PF_SN_ROUTE should be the same
6110 	 *        but for PF_SN_NAT it is different. Don't try optimizing it,
6111 	 *        just store all 3 hashes.
6112 	 */
6113 	struct pf_srchash	*snhs[PF_SN_MAX] = { NULL };
6114 	struct tcphdr		*th = &pd->hdr.tcp;
6115 	u_int16_t		 mss = V_tcp_mssdflt;
6116 	u_short			 sn_reason;
6117 	struct pf_krule_item	*ri;
6118 
6119 	/* check maximums */
6120 	if (r->max_states &&
6121 	    (counter_u64_fetch(r->states_cur) >= r->max_states)) {
6122 		counter_u64_add(V_pf_status.lcounters[LCNT_STATES], 1);
6123 		REASON_SET(&ctx->reason, PFRES_MAXSTATES);
6124 		goto csfailed;
6125 	}
6126 	/* src node for limits */
6127 	if ((r->rule_flag & PFRULE_SRCTRACK) &&
6128 	    (sn_reason = pf_insert_src_node(sns, snhs, r, pd->src, pd->af,
6129 	    NULL, NULL, pd->af, PF_SN_LIMIT)) != 0) {
6130 		REASON_SET(&ctx->reason, sn_reason);
6131 		goto csfailed;
6132 	}
6133 	/* src node for route-to rule */
6134 	if (r->rt) {
6135 		if ((r->route.opts & PF_POOL_STICKYADDR) &&
6136 		    (sn_reason = pf_insert_src_node(sns, snhs, r, pd->src,
6137 		    pd->af, &pd->act.rt_addr, pd->act.rt_kif, pd->act.rt_af,
6138 		    PF_SN_ROUTE)) != 0) {
6139 			REASON_SET(&ctx->reason, sn_reason);
6140 			goto csfailed;
6141 		}
6142 	}
6143 	/* src node for translation rule */
6144 	if (ctx->nr != NULL) {
6145 		KASSERT(ctx->nat_pool != NULL, ("%s: nat_pool is NULL", __func__));
6146 		/*
6147 		 * The NAT addresses are chosen during ruleset parsing.
6148 		 * The new afto code stores post-nat addresses in nsaddr.
6149 		 * The old nat code (also used for new nat-to rules) creates
6150 		 * state keys and stores addresses in them.
6151 		 */
6152 		if ((ctx->nat_pool->opts & PF_POOL_STICKYADDR) &&
6153 		    (sn_reason = pf_insert_src_node(sns, snhs, ctx->nr,
6154 		    ctx->sk ? &(ctx->sk->addr[pd->sidx]) : pd->src, pd->af,
6155 		    ctx->nk ? &(ctx->nk->addr[1]) : &(pd->nsaddr), NULL,
6156 		    pd->naf, PF_SN_NAT)) != 0 ) {
6157 			REASON_SET(&ctx->reason, sn_reason);
6158 			goto csfailed;
6159 		}
6160 	}
6161 	s = pf_alloc_state(M_NOWAIT);
6162 	if (s == NULL) {
6163 		REASON_SET(&ctx->reason, PFRES_MEMORY);
6164 		goto csfailed;
6165 	}
6166 	s->rule = r;
6167 	s->nat_rule = ctx->nr;
6168 	s->anchor = ctx->a;
6169 	memcpy(&s->match_rules, &ctx->rules, sizeof(s->match_rules));
6170 	memcpy(&s->act, &pd->act, sizeof(struct pf_rule_actions));
6171 
6172 	if (pd->act.allow_opts)
6173 		s->state_flags |= PFSTATE_ALLOWOPTS;
6174 	if (r->rule_flag & PFRULE_STATESLOPPY)
6175 		s->state_flags |= PFSTATE_SLOPPY;
6176 	if (pd->flags & PFDESC_TCP_NORM) /* Set by old-style scrub rules */
6177 		s->state_flags |= PFSTATE_SCRUB_TCP;
6178 	if ((r->rule_flag & PFRULE_PFLOW) ||
6179 	    (ctx->nr != NULL && ctx->nr->rule_flag & PFRULE_PFLOW))
6180 		s->state_flags |= PFSTATE_PFLOW;
6181 
6182 	s->act.log = pd->act.log & PF_LOG_ALL;
6183 	s->sync_state = PFSYNC_S_NONE;
6184 	s->state_flags |= pd->act.flags; /* Only needed for pfsync and state export */
6185 
6186 	if (ctx->nr != NULL)
6187 		s->act.log |= ctx->nr->log & PF_LOG_ALL;
6188 	switch (pd->proto) {
6189 	case IPPROTO_TCP:
6190 		s->src.seqlo = ntohl(th->th_seq);
6191 		s->src.seqhi = s->src.seqlo + pd->p_len + 1;
6192 		if ((tcp_get_flags(th) & (TH_SYN|TH_ACK)) == TH_SYN &&
6193 		    r->keep_state == PF_STATE_MODULATE) {
6194 			/* Generate sequence number modulator */
6195 			if ((s->src.seqdiff = pf_tcp_iss(pd) - s->src.seqlo) ==
6196 			    0)
6197 				s->src.seqdiff = 1;
6198 			pf_change_proto_a(pd->m, &th->th_seq, &th->th_sum,
6199 			    htonl(s->src.seqlo + s->src.seqdiff), 0);
6200 			ctx->rewrite = 1;
6201 		} else
6202 			s->src.seqdiff = 0;
6203 		if (tcp_get_flags(th) & TH_SYN) {
6204 			s->src.seqhi++;
6205 			s->src.wscale = pf_get_wscale(pd);
6206 		}
6207 		s->src.max_win = MAX(ntohs(th->th_win), 1);
6208 		if (s->src.wscale & PF_WSCALE_MASK) {
6209 			/* Remove scale factor from initial window */
6210 			int win = s->src.max_win;
6211 			win += 1 << (s->src.wscale & PF_WSCALE_MASK);
6212 			s->src.max_win = (win - 1) >>
6213 			    (s->src.wscale & PF_WSCALE_MASK);
6214 		}
6215 		if (tcp_get_flags(th) & TH_FIN)
6216 			s->src.seqhi++;
6217 		s->dst.seqhi = 1;
6218 		s->dst.max_win = 1;
6219 		pf_set_protostate(s, PF_PEER_SRC, TCPS_SYN_SENT);
6220 		pf_set_protostate(s, PF_PEER_DST, TCPS_CLOSED);
6221 		s->timeout = PFTM_TCP_FIRST_PACKET;
6222 		atomic_add_32(&V_pf_status.states_halfopen, 1);
6223 		break;
6224 	case IPPROTO_UDP:
6225 		pf_set_protostate(s, PF_PEER_SRC, PFUDPS_SINGLE);
6226 		pf_set_protostate(s, PF_PEER_DST, PFUDPS_NO_TRAFFIC);
6227 		s->timeout = PFTM_UDP_FIRST_PACKET;
6228 		break;
6229 	case IPPROTO_SCTP:
6230 		pf_set_protostate(s, PF_PEER_SRC, SCTP_COOKIE_WAIT);
6231 		pf_set_protostate(s, PF_PEER_DST, SCTP_CLOSED);
6232 		s->timeout = PFTM_SCTP_FIRST_PACKET;
6233 		break;
6234 	case IPPROTO_ICMP:
6235 #ifdef INET6
6236 	case IPPROTO_ICMPV6:
6237 #endif /* INET6 */
6238 		s->timeout = PFTM_ICMP_FIRST_PACKET;
6239 		break;
6240 	default:
6241 		pf_set_protostate(s, PF_PEER_SRC, PFOTHERS_SINGLE);
6242 		pf_set_protostate(s, PF_PEER_DST, PFOTHERS_NO_TRAFFIC);
6243 		s->timeout = PFTM_OTHER_FIRST_PACKET;
6244 	}
6245 
6246 	s->creation = s->expire = pf_get_uptime();
6247 
6248 	if (pd->proto == IPPROTO_TCP) {
6249 		if (s->state_flags & PFSTATE_SCRUB_TCP &&
6250 		    pf_normalize_tcp_init(pd, th, &s->src)) {
6251 			REASON_SET(&ctx->reason, PFRES_MEMORY);
6252 			goto csfailed;
6253 		}
6254 		if (s->state_flags & PFSTATE_SCRUB_TCP && s->src.scrub &&
6255 		    pf_normalize_tcp_stateful(pd, &ctx->reason, th, s,
6256 		    &s->src, &s->dst, &ctx->rewrite)) {
6257 			/* This really shouldn't happen!!! */
6258 			DPFPRINTF(PF_DEBUG_URGENT,
6259 			    "%s: tcp normalize failed on first "
6260 			     "pkt", __func__);
6261 			goto csfailed;
6262 		}
6263 	} else if (pd->proto == IPPROTO_SCTP) {
6264 		if (pf_normalize_sctp_init(pd, &s->src, &s->dst))
6265 			goto csfailed;
6266 		if (! (pd->sctp_flags & (PFDESC_SCTP_INIT | PFDESC_SCTP_ADD_IP)))
6267 			goto csfailed;
6268 	}
6269 	s->direction = pd->dir;
6270 
6271 	/*
6272 	 * sk/nk could already been setup by pf_get_translation().
6273 	 */
6274 	if (ctx->sk == NULL && ctx->nk == NULL) {
6275 		MPASS(pd->sport == NULL || (pd->osport == *pd->sport));
6276 		MPASS(pd->dport == NULL || (pd->odport == *pd->dport));
6277 		if (pf_state_key_setup(pd, pd->nsport, pd->ndport,
6278 		    &ctx->sk, &ctx->nk)) {
6279 			goto csfailed;
6280 		}
6281 	} else
6282 		KASSERT((ctx->sk != NULL && ctx->nk != NULL), ("%s: nr %p sk %p, nk %p",
6283 		    __func__, ctx->nr, ctx->sk, ctx->nk));
6284 
6285 	/* Swap sk/nk for PF_OUT. */
6286 	if (pf_state_insert(BOUND_IFACE(s, pd), pd->kif,
6287 	    (pd->dir == PF_IN) ? ctx->sk : ctx->nk,
6288 	    (pd->dir == PF_IN) ? ctx->nk : ctx->sk, s)) {
6289 		REASON_SET(&ctx->reason, PFRES_STATEINS);
6290 		goto drop;
6291 	} else
6292 		*sm = s;
6293 	ctx->sk = ctx->nk = NULL;
6294 
6295 	STATE_INC_COUNTERS(s);
6296 
6297 	/*
6298 	 * Lock order is important: first state, then source node.
6299 	 */
6300 	for (pf_sn_types_t sn_type=0; sn_type<PF_SN_MAX; sn_type++) {
6301 		if (pf_src_node_exists(&sns[sn_type], snhs[sn_type])) {
6302 			s->sns[sn_type] = sns[sn_type];
6303 			PF_HASHROW_UNLOCK(snhs[sn_type]);
6304 		}
6305 	}
6306 
6307 	if (ctx->tag > 0)
6308 		s->tag = ctx->tag;
6309 	if (pd->proto == IPPROTO_TCP && (tcp_get_flags(th) & (TH_SYN|TH_ACK)) ==
6310 	    TH_SYN && r->keep_state == PF_STATE_SYNPROXY && pd->dir == PF_IN) {
6311 		pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_SRC);
6312 		pf_undo_nat(ctx->nr, pd, bip_sum);
6313 		s->src.seqhi = arc4random();
6314 		/* Find mss option */
6315 		int rtid = M_GETFIB(pd->m);
6316 		mss = pf_get_mss(pd);
6317 		mss = pf_calc_mss(pd->src, pd->af, rtid, mss);
6318 		mss = pf_calc_mss(pd->dst, pd->af, rtid, mss);
6319 		s->src.mss = mss;
6320 		pf_send_tcp(r, pd->af, pd->dst, pd->src, th->th_dport,
6321 		    th->th_sport, s->src.seqhi, ntohl(th->th_seq) + 1,
6322 		    TH_SYN|TH_ACK, 0, s->src.mss, 0, M_SKIP_FIREWALL, 0, 0,
6323 		    pd->act.rtableid);
6324 		REASON_SET(&ctx->reason, PFRES_SYNPROXY);
6325 		return (PF_SYNPROXY_DROP);
6326 	}
6327 
6328 	s->udp_mapping = ctx->udp_mapping;
6329 
6330 	return (PF_PASS);
6331 
6332 csfailed:
6333 	while ((ri = SLIST_FIRST(&ctx->rules))) {
6334 		SLIST_REMOVE_HEAD(&ctx->rules, entry);
6335 		free(ri, M_PF_RULE_ITEM);
6336 	}
6337 
6338 	uma_zfree(V_pf_state_key_z, ctx->sk);
6339 	uma_zfree(V_pf_state_key_z, ctx->nk);
6340 
6341 	for (pf_sn_types_t sn_type=0; sn_type<PF_SN_MAX; sn_type++) {
6342 		if (pf_src_node_exists(&sns[sn_type], snhs[sn_type])) {
6343 			if (--sns[sn_type]->states == 0 &&
6344 			    sns[sn_type]->expire == 0) {
6345 				pf_unlink_src_node(sns[sn_type]);
6346 				pf_free_src_node(sns[sn_type]);
6347 				counter_u64_add(
6348 				    V_pf_status.scounters[SCNT_SRC_NODE_REMOVALS], 1);
6349 			}
6350 			PF_HASHROW_UNLOCK(snhs[sn_type]);
6351 		}
6352 	}
6353 
6354 drop:
6355 	if (s != NULL) {
6356 		pf_src_tree_remove_state(s);
6357 		s->timeout = PFTM_UNLINKED;
6358 		pf_free_state(s);
6359 	}
6360 
6361 	return (PF_DROP);
6362 }
6363 
6364 int
6365 pf_translate(struct pf_pdesc *pd, struct pf_addr *saddr, u_int16_t sport,
6366     struct pf_addr *daddr, u_int16_t dport, u_int16_t virtual_type,
6367     int icmp_dir)
6368 {
6369 	/*
6370 	 * pf_translate() implements OpenBSD's "new" NAT approach.
6371 	 * We don't follow it, because it involves a breaking syntax change
6372 	 * (removing nat/rdr rules, moving it into regular pf rules.)
6373 	 * It also moves NAT processing to be done after normal rules evaluation
6374 	 * whereas in FreeBSD that's done before rules processing.
6375 	 *
6376 	 * We adopt the function only for nat64, and keep other NAT processing
6377 	 * before rules processing.
6378 	 */
6379 	int	rewrite = 0;
6380 	int	afto = pd->af != pd->naf;
6381 
6382 	MPASS(afto);
6383 
6384 	switch (pd->proto) {
6385 	case IPPROTO_TCP:
6386 	case IPPROTO_UDP:
6387 	case IPPROTO_SCTP:
6388 		if (afto || *pd->sport != sport) {
6389 			pf_change_ap(pd, pd->src, pd->sport,
6390 			    saddr, sport);
6391 			rewrite = 1;
6392 		}
6393 		if (afto || *pd->dport != dport) {
6394 			pf_change_ap(pd, pd->dst, pd->dport,
6395 			    daddr, dport);
6396 			rewrite = 1;
6397 		}
6398 		break;
6399 
6400 #ifdef INET
6401 	case IPPROTO_ICMP:
6402 		/* pf_translate() is also used when logging invalid packets */
6403 		if (pd->af != AF_INET)
6404 			return (0);
6405 
6406 		if (afto) {
6407 			if (pf_translate_icmp_af(AF_INET6, &pd->hdr.icmp))
6408 				return (-1);
6409 			pd->proto = IPPROTO_ICMPV6;
6410 			rewrite = 1;
6411 		}
6412 		if (virtual_type == htons(ICMP_ECHO)) {
6413 			u_int16_t icmpid = (icmp_dir == PF_IN) ? sport : dport;
6414 
6415 			if (icmpid != pd->hdr.icmp.icmp_id) {
6416 				pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
6417 				    pd->hdr.icmp.icmp_cksum,
6418 				    pd->hdr.icmp.icmp_id, icmpid, 0);
6419 				pd->hdr.icmp.icmp_id = icmpid;
6420 				/* XXX TODO copyback. */
6421 				rewrite = 1;
6422 			}
6423 		}
6424 		break;
6425 #endif /* INET */
6426 
6427 #ifdef INET6
6428 	case IPPROTO_ICMPV6:
6429 		/* pf_translate() is also used when logging invalid packets */
6430 		if (pd->af != AF_INET6)
6431 			return (0);
6432 
6433 		if (afto) {
6434 			/* ip_sum will be recalculated in pf_translate_af */
6435 			if (pf_translate_icmp_af(AF_INET, &pd->hdr.icmp6))
6436 				return (0);
6437 			pd->proto = IPPROTO_ICMP;
6438 			rewrite = 1;
6439 		}
6440 		break;
6441 #endif /* INET6 */
6442 
6443 	default:
6444 		break;
6445 	}
6446 
6447 	return (rewrite);
6448 }
6449 
6450 int
6451 pf_translate_compat(struct pf_test_ctx *ctx)
6452 {
6453 	struct pf_pdesc		*pd = ctx->pd;
6454 	struct pf_state_key	*nk = ctx->nk;
6455 	struct tcphdr		*th = &pd->hdr.tcp;
6456 	int 			 rewrite = 0;
6457 
6458 	KASSERT(ctx->sk != NULL, ("%s: null sk", __func__));
6459 	KASSERT(ctx->nk != NULL, ("%s: null nk", __func__));
6460 
6461 	switch (pd->virtual_proto) {
6462 	case IPPROTO_TCP:
6463 		if (PF_ANEQ(&pd->nsaddr, &nk->addr[pd->sidx], pd->af) ||
6464 		    nk->port[pd->sidx] != pd->nsport) {
6465 			pf_change_ap(pd, pd->src, &th->th_sport,
6466 			    &nk->addr[pd->sidx], nk->port[pd->sidx]);
6467 			pd->sport = &th->th_sport;
6468 			pd->nsport = th->th_sport;
6469 			pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
6470 		}
6471 
6472 		if (PF_ANEQ(&pd->ndaddr, &nk->addr[pd->didx], pd->af) ||
6473 		    nk->port[pd->didx] != pd->ndport) {
6474 			pf_change_ap(pd, pd->dst, &th->th_dport,
6475 			    &nk->addr[pd->didx], nk->port[pd->didx]);
6476 			pd->dport = &th->th_dport;
6477 			pd->ndport = th->th_dport;
6478 			pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
6479 		}
6480 		rewrite++;
6481 		break;
6482 	case IPPROTO_UDP:
6483 		if (PF_ANEQ(&pd->nsaddr, &nk->addr[pd->sidx], pd->af) ||
6484 		    nk->port[pd->sidx] != pd->nsport) {
6485 			pf_change_ap(pd, pd->src,
6486 			    &pd->hdr.udp.uh_sport,
6487 			    &nk->addr[pd->sidx],
6488 			    nk->port[pd->sidx]);
6489 			pd->sport = &pd->hdr.udp.uh_sport;
6490 			pd->nsport = pd->hdr.udp.uh_sport;
6491 			pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
6492 		}
6493 
6494 		if (PF_ANEQ(&pd->ndaddr, &nk->addr[pd->didx], pd->af) ||
6495 		    nk->port[pd->didx] != pd->ndport) {
6496 			pf_change_ap(pd, pd->dst,
6497 			    &pd->hdr.udp.uh_dport,
6498 			    &nk->addr[pd->didx],
6499 			    nk->port[pd->didx]);
6500 			pd->dport = &pd->hdr.udp.uh_dport;
6501 			pd->ndport = pd->hdr.udp.uh_dport;
6502 			pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
6503 		}
6504 		rewrite++;
6505 		break;
6506 	case IPPROTO_SCTP: {
6507 		if (PF_ANEQ(&pd->nsaddr, &nk->addr[pd->sidx], pd->af) ||
6508 		    nk->port[pd->sidx] != pd->nsport) {
6509 			pf_change_ap(pd, pd->src,
6510 			    &pd->hdr.sctp.src_port,
6511 			    &nk->addr[pd->sidx],
6512 			    nk->port[pd->sidx]);
6513 			pd->sport = &pd->hdr.sctp.src_port;
6514 			pd->nsport = pd->hdr.sctp.src_port;
6515 			pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
6516 		}
6517 		if (PF_ANEQ(&pd->ndaddr, &nk->addr[pd->didx], pd->af) ||
6518 		    nk->port[pd->didx] != pd->ndport) {
6519 			pf_change_ap(pd, pd->dst,
6520 			    &pd->hdr.sctp.dest_port,
6521 			    &nk->addr[pd->didx],
6522 			    nk->port[pd->didx]);
6523 			pd->dport = &pd->hdr.sctp.dest_port;
6524 			pd->ndport = pd->hdr.sctp.dest_port;
6525 			pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
6526 		}
6527 		break;
6528 	}
6529 #ifdef INET
6530 	case IPPROTO_ICMP:
6531 		if (PF_ANEQ(&pd->nsaddr, &nk->addr[pd->sidx], AF_INET)) {
6532 			pf_change_a(&pd->src->v4.s_addr, pd->ip_sum,
6533 			    nk->addr[pd->sidx].v4.s_addr, 0);
6534 			pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
6535 		}
6536 
6537 		if (PF_ANEQ(&pd->ndaddr, &nk->addr[pd->didx], AF_INET)) {
6538 			pf_change_a(&pd->dst->v4.s_addr, pd->ip_sum,
6539 			    nk->addr[pd->didx].v4.s_addr, 0);
6540 			pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
6541 		}
6542 
6543 		if (ctx->virtual_type == htons(ICMP_ECHO) &&
6544 		    nk->port[pd->sidx] != pd->hdr.icmp.icmp_id) {
6545 			pd->hdr.icmp.icmp_cksum = pf_cksum_fixup(
6546 			    pd->hdr.icmp.icmp_cksum, pd->nsport,
6547 			    nk->port[pd->sidx], 0);
6548 			pd->hdr.icmp.icmp_id = nk->port[pd->sidx];
6549 			pd->sport = &pd->hdr.icmp.icmp_id;
6550 		}
6551 		m_copyback(pd->m, pd->off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
6552 		break;
6553 #endif /* INET */
6554 #ifdef INET6
6555 	case IPPROTO_ICMPV6:
6556 		if (PF_ANEQ(&pd->nsaddr, &nk->addr[pd->sidx], AF_INET6)) {
6557 			pf_change_a6(pd->src, &pd->hdr.icmp6.icmp6_cksum,
6558 			    &nk->addr[pd->sidx], 0);
6559 			pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
6560 		}
6561 
6562 		if (PF_ANEQ(&pd->ndaddr, &nk->addr[pd->didx], AF_INET6)) {
6563 			pf_change_a6(pd->dst, &pd->hdr.icmp6.icmp6_cksum,
6564 			    &nk->addr[pd->didx], 0);
6565 			pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
6566 		}
6567 		rewrite++;
6568 		break;
6569 #endif /* INET */
6570 	default:
6571 		switch (pd->af) {
6572 #ifdef INET
6573 		case AF_INET:
6574 			if (PF_ANEQ(&pd->nsaddr,
6575 				&nk->addr[pd->sidx], AF_INET)) {
6576 				pf_change_a(&pd->src->v4.s_addr,
6577 				    pd->ip_sum,
6578 				    nk->addr[pd->sidx].v4.s_addr, 0);
6579 				pf_addrcpy(&pd->nsaddr, pd->src, pd->af);
6580 			}
6581 
6582 			if (PF_ANEQ(&pd->ndaddr,
6583 				&nk->addr[pd->didx], AF_INET)) {
6584 				pf_change_a(&pd->dst->v4.s_addr,
6585 				    pd->ip_sum,
6586 				    nk->addr[pd->didx].v4.s_addr, 0);
6587 				pf_addrcpy(&pd->ndaddr, pd->dst, pd->af);
6588 			}
6589 			break;
6590 #endif /* INET */
6591 #ifdef INET6
6592 		case AF_INET6:
6593 			if (PF_ANEQ(&pd->nsaddr,
6594 				&nk->addr[pd->sidx], AF_INET6)) {
6595 				pf_addrcpy(&pd->nsaddr, &nk->addr[pd->sidx],
6596 				    pd->af);
6597 				pf_addrcpy(pd->src, &nk->addr[pd->sidx], pd->af);
6598 			}
6599 
6600 			if (PF_ANEQ(&pd->ndaddr,
6601 				&nk->addr[pd->didx], AF_INET6)) {
6602 				pf_addrcpy(&pd->ndaddr, &nk->addr[pd->didx],
6603 				    pd->af);
6604 				pf_addrcpy(pd->dst, &nk->addr[pd->didx],
6605 				    pd->af);
6606 			}
6607 			break;
6608 #endif /* INET6 */
6609 		}
6610 		break;
6611 	}
6612 	return (rewrite);
6613 }
6614 
6615 static int
6616 pf_tcp_track_full(struct pf_kstate *state, struct pf_pdesc *pd,
6617     u_short *reason, int *copyback, struct pf_state_peer *src,
6618     struct pf_state_peer *dst, u_int8_t psrc, u_int8_t pdst)
6619 {
6620 	struct tcphdr		*th = &pd->hdr.tcp;
6621 	u_int16_t		 win = ntohs(th->th_win);
6622 	u_int32_t		 ack, end, data_end, seq, orig_seq;
6623 	u_int8_t		 sws, dws;
6624 	int			 ackskew;
6625 
6626 	if (src->wscale && dst->wscale && !(tcp_get_flags(th) & TH_SYN)) {
6627 		sws = src->wscale & PF_WSCALE_MASK;
6628 		dws = dst->wscale & PF_WSCALE_MASK;
6629 	} else
6630 		sws = dws = 0;
6631 
6632 	/*
6633 	 * Sequence tracking algorithm from Guido van Rooij's paper:
6634 	 *   http://www.madison-gurkha.com/publications/tcp_filtering/
6635 	 *	tcp_filtering.ps
6636 	 */
6637 
6638 	orig_seq = seq = ntohl(th->th_seq);
6639 	if (src->seqlo == 0) {
6640 		/* First packet from this end. Set its state */
6641 
6642 		if ((state->state_flags & PFSTATE_SCRUB_TCP || dst->scrub) &&
6643 		    src->scrub == NULL) {
6644 			if (pf_normalize_tcp_init(pd, th, src)) {
6645 				REASON_SET(reason, PFRES_MEMORY);
6646 				return (PF_DROP);
6647 			}
6648 		}
6649 
6650 		/* Deferred generation of sequence number modulator */
6651 		if (dst->seqdiff && !src->seqdiff) {
6652 			/* use random iss for the TCP server */
6653 			while ((src->seqdiff = arc4random() - seq) == 0)
6654 				;
6655 			ack = ntohl(th->th_ack) - dst->seqdiff;
6656 			pf_change_proto_a(pd->m, &th->th_seq, &th->th_sum, htonl(seq +
6657 			    src->seqdiff), 0);
6658 			pf_change_proto_a(pd->m, &th->th_ack, &th->th_sum, htonl(ack), 0);
6659 			*copyback = 1;
6660 		} else {
6661 			ack = ntohl(th->th_ack);
6662 		}
6663 
6664 		end = seq + pd->p_len;
6665 		if (tcp_get_flags(th) & TH_SYN) {
6666 			end++;
6667 			if (dst->wscale & PF_WSCALE_FLAG) {
6668 				src->wscale = pf_get_wscale(pd);
6669 				if (src->wscale & PF_WSCALE_FLAG) {
6670 					/* Remove scale factor from initial
6671 					 * window */
6672 					sws = src->wscale & PF_WSCALE_MASK;
6673 					win = ((u_int32_t)win + (1 << sws) - 1)
6674 					    >> sws;
6675 					dws = dst->wscale & PF_WSCALE_MASK;
6676 				} else {
6677 					/* fixup other window */
6678 					dst->max_win = MIN(TCP_MAXWIN,
6679 					    (u_int32_t)dst->max_win <<
6680 					    (dst->wscale & PF_WSCALE_MASK));
6681 					/* in case of a retrans SYN|ACK */
6682 					dst->wscale = 0;
6683 				}
6684 			}
6685 		}
6686 		data_end = end;
6687 		if (tcp_get_flags(th) & TH_FIN)
6688 			end++;
6689 
6690 		src->seqlo = seq;
6691 		if (src->state < TCPS_SYN_SENT)
6692 			pf_set_protostate(state, psrc, TCPS_SYN_SENT);
6693 
6694 		/*
6695 		 * May need to slide the window (seqhi may have been set by
6696 		 * the crappy stack check or if we picked up the connection
6697 		 * after establishment)
6698 		 */
6699 		if (src->seqhi == 1 ||
6700 		    SEQ_GEQ(end + MAX(1, dst->max_win << dws), src->seqhi))
6701 			src->seqhi = end + MAX(1, dst->max_win << dws);
6702 		if (win > src->max_win)
6703 			src->max_win = win;
6704 
6705 	} else {
6706 		ack = ntohl(th->th_ack) - dst->seqdiff;
6707 		if (src->seqdiff) {
6708 			/* Modulate sequence numbers */
6709 			pf_change_proto_a(pd->m, &th->th_seq, &th->th_sum, htonl(seq +
6710 			    src->seqdiff), 0);
6711 			pf_change_proto_a(pd->m, &th->th_ack, &th->th_sum, htonl(ack), 0);
6712 			*copyback = 1;
6713 		}
6714 		end = seq + pd->p_len;
6715 		if (tcp_get_flags(th) & TH_SYN)
6716 			end++;
6717 		data_end = end;
6718 		if (tcp_get_flags(th) & TH_FIN)
6719 			end++;
6720 	}
6721 
6722 	if ((tcp_get_flags(th) & TH_ACK) == 0) {
6723 		/* Let it pass through the ack skew check */
6724 		ack = dst->seqlo;
6725 	} else if ((ack == 0 &&
6726 	    (tcp_get_flags(th) & (TH_ACK|TH_RST)) == (TH_ACK|TH_RST)) ||
6727 	    /* broken tcp stacks do not set ack */
6728 	    (dst->state < TCPS_SYN_SENT)) {
6729 		/*
6730 		 * Many stacks (ours included) will set the ACK number in an
6731 		 * FIN|ACK if the SYN times out -- no sequence to ACK.
6732 		 */
6733 		ack = dst->seqlo;
6734 	}
6735 
6736 	if (seq == end) {
6737 		/* Ease sequencing restrictions on no data packets */
6738 		seq = src->seqlo;
6739 		data_end = end = seq;
6740 	}
6741 
6742 	ackskew = dst->seqlo - ack;
6743 
6744 	/*
6745 	 * Need to demodulate the sequence numbers in any TCP SACK options
6746 	 * (Selective ACK). We could optionally validate the SACK values
6747 	 * against the current ACK window, either forwards or backwards, but
6748 	 * I'm not confident that SACK has been implemented properly
6749 	 * everywhere. It wouldn't surprise me if several stacks accidentally
6750 	 * SACK too far backwards of previously ACKed data. There really aren't
6751 	 * any security implications of bad SACKing unless the target stack
6752 	 * doesn't validate the option length correctly. Someone trying to
6753 	 * spoof into a TCP connection won't bother blindly sending SACK
6754 	 * options anyway.
6755 	 */
6756 	if (dst->seqdiff && (th->th_off << 2) > sizeof(struct tcphdr)) {
6757 		if (pf_modulate_sack(pd, th, dst))
6758 			*copyback = 1;
6759 	}
6760 
6761 #define	MAXACKWINDOW (0xffff + 1500)	/* 1500 is an arbitrary fudge factor */
6762 	if (SEQ_GEQ(src->seqhi, data_end) &&
6763 	    /* Last octet inside other's window space */
6764 	    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) &&
6765 	    /* Retrans: not more than one window back */
6766 	    (ackskew >= -MAXACKWINDOW) &&
6767 	    /* Acking not more than one reassembled fragment backwards */
6768 	    (ackskew <= (MAXACKWINDOW << sws)) &&
6769 	    /* Acking not more than one window forward */
6770 	    ((tcp_get_flags(th) & TH_RST) == 0 || orig_seq == src->seqlo ||
6771 	    (orig_seq == src->seqlo + 1) || (orig_seq + 1 == src->seqlo))) {
6772 	    /* Require an exact/+1 sequence match on resets when possible */
6773 
6774 		if (dst->scrub || src->scrub) {
6775 			if (pf_normalize_tcp_stateful(pd, reason, th,
6776 			    state, src, dst, copyback))
6777 				return (PF_DROP);
6778 		}
6779 
6780 		/* update max window */
6781 		if (src->max_win < win)
6782 			src->max_win = win;
6783 		/* synchronize sequencing */
6784 		if (SEQ_GT(end, src->seqlo))
6785 			src->seqlo = end;
6786 		/* slide the window of what the other end can send */
6787 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
6788 			dst->seqhi = ack + MAX((win << sws), 1);
6789 
6790 		/* update states */
6791 		if (tcp_get_flags(th) & TH_SYN)
6792 			if (src->state < TCPS_SYN_SENT)
6793 				pf_set_protostate(state, psrc, TCPS_SYN_SENT);
6794 		if (tcp_get_flags(th) & TH_FIN)
6795 			if (src->state < TCPS_CLOSING)
6796 				pf_set_protostate(state, psrc, TCPS_CLOSING);
6797 		if (tcp_get_flags(th) & TH_ACK) {
6798 			if (dst->state == TCPS_SYN_SENT) {
6799 				pf_set_protostate(state, pdst,
6800 				    TCPS_ESTABLISHED);
6801 				if (src->state == TCPS_ESTABLISHED &&
6802 				    state->sns[PF_SN_LIMIT] != NULL &&
6803 				    pf_src_connlimit(state)) {
6804 					REASON_SET(reason, PFRES_SRCLIMIT);
6805 					return (PF_DROP);
6806 				}
6807 			} else if (dst->state == TCPS_CLOSING)
6808 				pf_set_protostate(state, pdst,
6809 				    TCPS_FIN_WAIT_2);
6810 		}
6811 		if (tcp_get_flags(th) & TH_RST)
6812 			pf_set_protostate(state, PF_PEER_BOTH, TCPS_TIME_WAIT);
6813 
6814 		/* update expire time */
6815 		state->expire = pf_get_uptime();
6816 		if (src->state >= TCPS_FIN_WAIT_2 &&
6817 		    dst->state >= TCPS_FIN_WAIT_2)
6818 			state->timeout = PFTM_TCP_CLOSED;
6819 		else if (src->state >= TCPS_CLOSING &&
6820 		    dst->state >= TCPS_CLOSING)
6821 			state->timeout = PFTM_TCP_FIN_WAIT;
6822 		else if (src->state < TCPS_ESTABLISHED ||
6823 		    dst->state < TCPS_ESTABLISHED)
6824 			state->timeout = PFTM_TCP_OPENING;
6825 		else if (src->state >= TCPS_CLOSING ||
6826 		    dst->state >= TCPS_CLOSING)
6827 			state->timeout = PFTM_TCP_CLOSING;
6828 		else
6829 			state->timeout = PFTM_TCP_ESTABLISHED;
6830 
6831 		/* Fall through to PASS packet */
6832 
6833 	} else if ((dst->state < TCPS_SYN_SENT ||
6834 		dst->state >= TCPS_FIN_WAIT_2 ||
6835 		src->state >= TCPS_FIN_WAIT_2) &&
6836 	    SEQ_GEQ(src->seqhi + MAXACKWINDOW, data_end) &&
6837 	    /* Within a window forward of the originating packet */
6838 	    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW)) {
6839 	    /* Within a window backward of the originating packet */
6840 
6841 		/*
6842 		 * This currently handles three situations:
6843 		 *  1) Stupid stacks will shotgun SYNs before their peer
6844 		 *     replies.
6845 		 *  2) When PF catches an already established stream (the
6846 		 *     firewall rebooted, the state table was flushed, routes
6847 		 *     changed...)
6848 		 *  3) Packets get funky immediately after the connection
6849 		 *     closes (this should catch Solaris spurious ACK|FINs
6850 		 *     that web servers like to spew after a close)
6851 		 *
6852 		 * This must be a little more careful than the above code
6853 		 * since packet floods will also be caught here. We don't
6854 		 * update the TTL here to mitigate the damage of a packet
6855 		 * flood and so the same code can handle awkward establishment
6856 		 * and a loosened connection close.
6857 		 * In the establishment case, a correct peer response will
6858 		 * validate the connection, go through the normal state code
6859 		 * and keep updating the state TTL.
6860 		 */
6861 
6862 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
6863 			printf("pf: loose state match: ");
6864 			pf_print_state(state);
6865 			pf_print_flags(tcp_get_flags(th));
6866 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
6867 			    "pkts=%llu:%llu dir=%s,%s\n", seq, orig_seq, ack,
6868 			    pd->p_len, ackskew, (unsigned long long)state->packets[0],
6869 			    (unsigned long long)state->packets[1],
6870 			    pd->dir == PF_IN ? "in" : "out",
6871 			    pd->dir == state->direction ? "fwd" : "rev");
6872 		}
6873 
6874 		if (dst->scrub || src->scrub) {
6875 			if (pf_normalize_tcp_stateful(pd, reason, th,
6876 			    state, src, dst, copyback))
6877 				return (PF_DROP);
6878 		}
6879 
6880 		/* update max window */
6881 		if (src->max_win < win)
6882 			src->max_win = win;
6883 		/* synchronize sequencing */
6884 		if (SEQ_GT(end, src->seqlo))
6885 			src->seqlo = end;
6886 		/* slide the window of what the other end can send */
6887 		if (SEQ_GEQ(ack + (win << sws), dst->seqhi))
6888 			dst->seqhi = ack + MAX((win << sws), 1);
6889 
6890 		/*
6891 		 * Cannot set dst->seqhi here since this could be a shotgunned
6892 		 * SYN and not an already established connection.
6893 		 */
6894 
6895 		if (tcp_get_flags(th) & TH_FIN)
6896 			if (src->state < TCPS_CLOSING)
6897 				pf_set_protostate(state, psrc, TCPS_CLOSING);
6898 		if (tcp_get_flags(th) & TH_RST)
6899 			pf_set_protostate(state, PF_PEER_BOTH, TCPS_TIME_WAIT);
6900 
6901 		/* Fall through to PASS packet */
6902 
6903 	} else {
6904 		if (state->dst.state == TCPS_SYN_SENT &&
6905 		    state->src.state == TCPS_SYN_SENT) {
6906 			/* Send RST for state mismatches during handshake */
6907 			if (!(tcp_get_flags(th) & TH_RST))
6908 				pf_send_tcp(state->rule, pd->af,
6909 				    pd->dst, pd->src, th->th_dport,
6910 				    th->th_sport, ntohl(th->th_ack), 0,
6911 				    TH_RST, 0, 0,
6912 				    state->rule->return_ttl, M_SKIP_FIREWALL,
6913 				    0, 0, state->act.rtableid);
6914 			src->seqlo = 0;
6915 			src->seqhi = 1;
6916 			src->max_win = 1;
6917 		} else if (V_pf_status.debug >= PF_DEBUG_MISC) {
6918 			printf("pf: BAD state: ");
6919 			pf_print_state(state);
6920 			pf_print_flags(tcp_get_flags(th));
6921 			printf(" seq=%u (%u) ack=%u len=%u ackskew=%d "
6922 			    "pkts=%llu:%llu dir=%s,%s\n",
6923 			    seq, orig_seq, ack, pd->p_len, ackskew,
6924 			    (unsigned long long)state->packets[0],
6925 			    (unsigned long long)state->packets[1],
6926 			    pd->dir == PF_IN ? "in" : "out",
6927 			    pd->dir == state->direction ? "fwd" : "rev");
6928 			printf("pf: State failure on: %c %c %c %c | %c %c\n",
6929 			    SEQ_GEQ(src->seqhi, data_end) ? ' ' : '1',
6930 			    SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)) ?
6931 			    ' ': '2',
6932 			    (ackskew >= -MAXACKWINDOW) ? ' ' : '3',
6933 			    (ackskew <= (MAXACKWINDOW << sws)) ? ' ' : '4',
6934 			    SEQ_GEQ(src->seqhi + MAXACKWINDOW, data_end) ?' ' :'5',
6935 			    SEQ_GEQ(seq, src->seqlo - MAXACKWINDOW) ?' ' :'6');
6936 		}
6937 		REASON_SET(reason, PFRES_BADSTATE);
6938 		return (PF_DROP);
6939 	}
6940 
6941 	return (PF_PASS);
6942 }
6943 
6944 static int
6945 pf_tcp_track_sloppy(struct pf_kstate *state, struct pf_pdesc *pd,
6946     u_short *reason, struct pf_state_peer *src, struct pf_state_peer *dst,
6947     u_int8_t psrc, u_int8_t pdst)
6948 {
6949 	struct tcphdr		*th = &pd->hdr.tcp;
6950 
6951 	if (tcp_get_flags(th) & TH_SYN)
6952 		if (src->state < TCPS_SYN_SENT)
6953 			pf_set_protostate(state, psrc, TCPS_SYN_SENT);
6954 	if (tcp_get_flags(th) & TH_FIN)
6955 		if (src->state < TCPS_CLOSING)
6956 			pf_set_protostate(state, psrc, TCPS_CLOSING);
6957 	if (tcp_get_flags(th) & TH_ACK) {
6958 		if (dst->state == TCPS_SYN_SENT) {
6959 			pf_set_protostate(state, pdst, TCPS_ESTABLISHED);
6960 			if (src->state == TCPS_ESTABLISHED &&
6961 			    state->sns[PF_SN_LIMIT] != NULL &&
6962 			    pf_src_connlimit(state)) {
6963 				REASON_SET(reason, PFRES_SRCLIMIT);
6964 				return (PF_DROP);
6965 			}
6966 		} else if (dst->state == TCPS_CLOSING) {
6967 			pf_set_protostate(state, pdst, TCPS_FIN_WAIT_2);
6968 		} else if (src->state == TCPS_SYN_SENT &&
6969 		    dst->state < TCPS_SYN_SENT) {
6970 			/*
6971 			 * Handle a special sloppy case where we only see one
6972 			 * half of the connection. If there is a ACK after
6973 			 * the initial SYN without ever seeing a packet from
6974 			 * the destination, set the connection to established.
6975 			 */
6976 			pf_set_protostate(state, PF_PEER_BOTH,
6977 			    TCPS_ESTABLISHED);
6978 			dst->state = src->state = TCPS_ESTABLISHED;
6979 			if (state->sns[PF_SN_LIMIT] != NULL &&
6980 			    pf_src_connlimit(state)) {
6981 				REASON_SET(reason, PFRES_SRCLIMIT);
6982 				return (PF_DROP);
6983 			}
6984 		} else if (src->state == TCPS_CLOSING &&
6985 		    dst->state == TCPS_ESTABLISHED &&
6986 		    dst->seqlo == 0) {
6987 			/*
6988 			 * Handle the closing of half connections where we
6989 			 * don't see the full bidirectional FIN/ACK+ACK
6990 			 * handshake.
6991 			 */
6992 			pf_set_protostate(state, pdst, TCPS_CLOSING);
6993 		}
6994 	}
6995 	if (tcp_get_flags(th) & TH_RST)
6996 		pf_set_protostate(state, PF_PEER_BOTH, TCPS_TIME_WAIT);
6997 
6998 	/* update expire time */
6999 	state->expire = pf_get_uptime();
7000 	if (src->state >= TCPS_FIN_WAIT_2 &&
7001 	    dst->state >= TCPS_FIN_WAIT_2)
7002 		state->timeout = PFTM_TCP_CLOSED;
7003 	else if (src->state >= TCPS_CLOSING &&
7004 	    dst->state >= TCPS_CLOSING)
7005 		state->timeout = PFTM_TCP_FIN_WAIT;
7006 	else if (src->state < TCPS_ESTABLISHED ||
7007 	    dst->state < TCPS_ESTABLISHED)
7008 		state->timeout = PFTM_TCP_OPENING;
7009 	else if (src->state >= TCPS_CLOSING ||
7010 	    dst->state >= TCPS_CLOSING)
7011 		state->timeout = PFTM_TCP_CLOSING;
7012 	else
7013 		state->timeout = PFTM_TCP_ESTABLISHED;
7014 
7015 	return (PF_PASS);
7016 }
7017 
7018 static int
7019 pf_synproxy(struct pf_pdesc *pd, struct pf_kstate *state, u_short *reason)
7020 {
7021 	struct pf_state_key	*sk = state->key[pd->didx];
7022 	struct tcphdr		*th = &pd->hdr.tcp;
7023 
7024 	if (state->src.state == PF_TCPS_PROXY_SRC) {
7025 		if (pd->dir != state->direction) {
7026 			REASON_SET(reason, PFRES_SYNPROXY);
7027 			return (PF_SYNPROXY_DROP);
7028 		}
7029 		if (tcp_get_flags(th) & TH_SYN) {
7030 			if (ntohl(th->th_seq) != state->src.seqlo) {
7031 				REASON_SET(reason, PFRES_SYNPROXY);
7032 				return (PF_DROP);
7033 			}
7034 			pf_send_tcp(state->rule, pd->af, pd->dst,
7035 			    pd->src, th->th_dport, th->th_sport,
7036 			    state->src.seqhi, ntohl(th->th_seq) + 1,
7037 			    TH_SYN|TH_ACK, 0, state->src.mss, 0,
7038 			    M_SKIP_FIREWALL, 0, 0, state->act.rtableid);
7039 			REASON_SET(reason, PFRES_SYNPROXY);
7040 			return (PF_SYNPROXY_DROP);
7041 		} else if ((tcp_get_flags(th) & (TH_ACK|TH_RST|TH_FIN)) != TH_ACK ||
7042 		    (ntohl(th->th_ack) != state->src.seqhi + 1) ||
7043 		    (ntohl(th->th_seq) != state->src.seqlo + 1)) {
7044 			REASON_SET(reason, PFRES_SYNPROXY);
7045 			return (PF_DROP);
7046 		} else if (state->sns[PF_SN_LIMIT] != NULL &&
7047 		    pf_src_connlimit(state)) {
7048 			REASON_SET(reason, PFRES_SRCLIMIT);
7049 			return (PF_DROP);
7050 		} else
7051 			pf_set_protostate(state, PF_PEER_SRC,
7052 			    PF_TCPS_PROXY_DST);
7053 	}
7054 	if (state->src.state == PF_TCPS_PROXY_DST) {
7055 		if (pd->dir == state->direction) {
7056 			if (((tcp_get_flags(th) & (TH_SYN|TH_ACK)) != TH_ACK) ||
7057 			    (ntohl(th->th_ack) != state->src.seqhi + 1) ||
7058 			    (ntohl(th->th_seq) != state->src.seqlo + 1)) {
7059 				REASON_SET(reason, PFRES_SYNPROXY);
7060 				return (PF_DROP);
7061 			}
7062 			state->src.max_win = MAX(ntohs(th->th_win), 1);
7063 			if (state->dst.seqhi == 1)
7064 				state->dst.seqhi = arc4random();
7065 			pf_send_tcp(state->rule, pd->af,
7066 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
7067 			    sk->port[pd->sidx], sk->port[pd->didx],
7068 			    state->dst.seqhi, 0, TH_SYN, 0,
7069 			    state->src.mss, 0,
7070 			    state->orig_kif->pfik_ifp == V_loif ? M_LOOP : 0,
7071 			    state->tag, 0, state->act.rtableid);
7072 			REASON_SET(reason, PFRES_SYNPROXY);
7073 			return (PF_SYNPROXY_DROP);
7074 		} else if (((tcp_get_flags(th) & (TH_SYN|TH_ACK)) !=
7075 		    (TH_SYN|TH_ACK)) ||
7076 		    (ntohl(th->th_ack) != state->dst.seqhi + 1)) {
7077 			REASON_SET(reason, PFRES_SYNPROXY);
7078 			return (PF_DROP);
7079 		} else {
7080 			state->dst.max_win = MAX(ntohs(th->th_win), 1);
7081 			state->dst.seqlo = ntohl(th->th_seq);
7082 			pf_send_tcp(state->rule, pd->af, pd->dst,
7083 			    pd->src, th->th_dport, th->th_sport,
7084 			    ntohl(th->th_ack), ntohl(th->th_seq) + 1,
7085 			    TH_ACK, state->src.max_win, 0, 0, 0,
7086 			    state->tag, 0, state->act.rtableid);
7087 			pf_send_tcp(state->rule, pd->af,
7088 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
7089 			    sk->port[pd->sidx], sk->port[pd->didx],
7090 			    state->src.seqhi + 1, state->src.seqlo + 1,
7091 			    TH_ACK, state->dst.max_win, 0, 0,
7092 			    M_SKIP_FIREWALL, 0, 0, state->act.rtableid);
7093 			state->src.seqdiff = state->dst.seqhi -
7094 			    state->src.seqlo;
7095 			state->dst.seqdiff = state->src.seqhi -
7096 			    state->dst.seqlo;
7097 			state->src.seqhi = state->src.seqlo +
7098 			    state->dst.max_win;
7099 			state->dst.seqhi = state->dst.seqlo +
7100 			    state->src.max_win;
7101 			state->src.wscale = state->dst.wscale = 0;
7102 			pf_set_protostate(state, PF_PEER_BOTH,
7103 			    TCPS_ESTABLISHED);
7104 			REASON_SET(reason, PFRES_SYNPROXY);
7105 			return (PF_SYNPROXY_DROP);
7106 		}
7107 	}
7108 
7109 	return (PF_PASS);
7110 }
7111 
7112 static int
7113 pf_test_state(struct pf_kstate **state, struct pf_pdesc *pd, u_short *reason)
7114 {
7115 	struct pf_state_key_cmp	 key;
7116 	int			 copyback = 0;
7117 	struct pf_state_peer	*src, *dst;
7118 	uint8_t			 psrc, pdst;
7119 	int			 action;
7120 
7121 	bzero(&key, sizeof(key));
7122 	key.af = pd->af;
7123 	key.proto = pd->virtual_proto;
7124 	pf_addrcpy(&key.addr[pd->sidx], pd->src, key.af);
7125 	pf_addrcpy(&key.addr[pd->didx], pd->dst, key.af);
7126 	key.port[pd->sidx] = pd->osport;
7127 	key.port[pd->didx] = pd->odport;
7128 
7129 	action = pf_find_state(pd, &key, state);
7130 	if (action != PF_MATCH)
7131 		return (action);
7132 
7133 	action = PF_PASS;
7134 	if (pd->dir == (*state)->direction) {
7135 		if (PF_REVERSED_KEY(*state, pd->af)) {
7136 			src = &(*state)->dst;
7137 			dst = &(*state)->src;
7138 			psrc = PF_PEER_DST;
7139 			pdst = PF_PEER_SRC;
7140 		} else {
7141 			src = &(*state)->src;
7142 			dst = &(*state)->dst;
7143 			psrc = PF_PEER_SRC;
7144 			pdst = PF_PEER_DST;
7145 		}
7146 	} else {
7147 		if (PF_REVERSED_KEY(*state, pd->af)) {
7148 			src = &(*state)->src;
7149 			dst = &(*state)->dst;
7150 			psrc = PF_PEER_SRC;
7151 			pdst = PF_PEER_DST;
7152 		} else {
7153 			src = &(*state)->dst;
7154 			dst = &(*state)->src;
7155 			psrc = PF_PEER_DST;
7156 			pdst = PF_PEER_SRC;
7157 		}
7158 	}
7159 
7160 	switch (pd->virtual_proto) {
7161 	case IPPROTO_TCP: {
7162 		struct tcphdr		*th = &pd->hdr.tcp;
7163 
7164 		if ((action = pf_synproxy(pd, *state, reason)) != PF_PASS)
7165 			return (action);
7166 		if (((tcp_get_flags(th) & (TH_SYN | TH_ACK)) == TH_SYN) ||
7167 		    ((th->th_flags & (TH_SYN | TH_ACK | TH_RST)) == TH_ACK &&
7168 		    pf_syncookie_check(pd) && pd->dir == PF_IN)) {
7169 			if ((*state)->src.state >= TCPS_FIN_WAIT_2 &&
7170 			    (*state)->dst.state >= TCPS_FIN_WAIT_2) {
7171 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
7172 					printf("pf: state reuse ");
7173 					pf_print_state(*state);
7174 					pf_print_flags(tcp_get_flags(th));
7175 					printf("\n");
7176 				}
7177 				/* XXX make sure it's the same direction ?? */
7178 				pf_set_protostate(*state, PF_PEER_BOTH, TCPS_CLOSED);
7179 				pf_remove_state(*state);
7180 				*state = NULL;
7181 				return (PF_DROP);
7182 			} else if ((*state)->src.state >= TCPS_ESTABLISHED &&
7183 			    (*state)->dst.state >= TCPS_ESTABLISHED) {
7184 				/*
7185 				 * SYN matches existing state???
7186 				 * Typically happens when sender boots up after
7187 				 * sudden panic. Certain protocols (NFSv3) are
7188 				 * always using same port numbers. Challenge
7189 				 * ACK enables all parties (firewall and peers)
7190 				 * to get in sync again.
7191 				 */
7192 				pf_send_challenge_ack(pd, *state, src, dst);
7193 				return (PF_DROP);
7194 			}
7195 		}
7196 		if ((*state)->state_flags & PFSTATE_SLOPPY) {
7197 			if (pf_tcp_track_sloppy(*state, pd, reason, src, dst,
7198 			    psrc, pdst) == PF_DROP)
7199 				return (PF_DROP);
7200 		} else {
7201 			int	 ret;
7202 
7203 			ret = pf_tcp_track_full(*state, pd, reason,
7204 			    &copyback, src, dst, psrc, pdst);
7205 			if (ret == PF_DROP)
7206 				return (PF_DROP);
7207 		}
7208 		break;
7209 	}
7210 	case IPPROTO_UDP:
7211 		/* update states */
7212 		if (src->state < PFUDPS_SINGLE)
7213 			pf_set_protostate(*state, psrc, PFUDPS_SINGLE);
7214 		if (dst->state == PFUDPS_SINGLE)
7215 			pf_set_protostate(*state, pdst, PFUDPS_MULTIPLE);
7216 
7217 		/* update expire time */
7218 		(*state)->expire = pf_get_uptime();
7219 		if (src->state == PFUDPS_MULTIPLE && dst->state == PFUDPS_MULTIPLE)
7220 			(*state)->timeout = PFTM_UDP_MULTIPLE;
7221 		else
7222 			(*state)->timeout = PFTM_UDP_SINGLE;
7223 		break;
7224 	case IPPROTO_SCTP:
7225 		if ((src->state >= SCTP_SHUTDOWN_SENT || src->state == SCTP_CLOSED) &&
7226 		    (dst->state >= SCTP_SHUTDOWN_SENT || dst->state == SCTP_CLOSED) &&
7227 		    pd->sctp_flags & PFDESC_SCTP_INIT) {
7228 			pf_set_protostate(*state, PF_PEER_BOTH, SCTP_CLOSED);
7229 			pf_remove_state(*state);
7230 			*state = NULL;
7231 			return (PF_DROP);
7232 		}
7233 
7234 		if (pf_sctp_track(*state, pd, reason) != PF_PASS)
7235 			return (PF_DROP);
7236 
7237 		/* Track state. */
7238 		if (pd->sctp_flags & PFDESC_SCTP_INIT) {
7239 			if (src->state < SCTP_COOKIE_WAIT) {
7240 				pf_set_protostate(*state, psrc, SCTP_COOKIE_WAIT);
7241 				(*state)->timeout = PFTM_SCTP_OPENING;
7242 			}
7243 		}
7244 		if (pd->sctp_flags & PFDESC_SCTP_INIT_ACK) {
7245 			MPASS(dst->scrub != NULL);
7246 			if (dst->scrub->pfss_v_tag == 0)
7247 				dst->scrub->pfss_v_tag = pd->sctp_initiate_tag;
7248 		}
7249 
7250 		/*
7251 		 * Bind to the correct interface if we're if-bound. For multihomed
7252 		 * extra associations we don't know which interface that will be until
7253 		 * here, so we've inserted the state on V_pf_all. Fix that now.
7254 		 */
7255 		if ((*state)->kif == V_pfi_all &&
7256 		    (*state)->rule->rule_flag & PFRULE_IFBOUND)
7257 			(*state)->kif = pd->kif;
7258 
7259 		if (pd->sctp_flags & (PFDESC_SCTP_COOKIE | PFDESC_SCTP_HEARTBEAT_ACK)) {
7260 			if (src->state < SCTP_ESTABLISHED) {
7261 				pf_set_protostate(*state, psrc, SCTP_ESTABLISHED);
7262 				(*state)->timeout = PFTM_SCTP_ESTABLISHED;
7263 			}
7264 		}
7265 		if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN |
7266 		    PFDESC_SCTP_SHUTDOWN_COMPLETE)) {
7267 			if (src->state < SCTP_SHUTDOWN_PENDING) {
7268 				pf_set_protostate(*state, psrc, SCTP_SHUTDOWN_PENDING);
7269 				(*state)->timeout = PFTM_SCTP_CLOSING;
7270 			}
7271 		}
7272 		if (pd->sctp_flags & (PFDESC_SCTP_SHUTDOWN_COMPLETE | PFDESC_SCTP_ABORT)) {
7273 			pf_set_protostate(*state, psrc, SCTP_CLOSED);
7274 			(*state)->timeout = PFTM_SCTP_CLOSED;
7275 		}
7276 
7277 		(*state)->expire = pf_get_uptime();
7278 		break;
7279 	default:
7280 		/* update states */
7281 		if (src->state < PFOTHERS_SINGLE)
7282 			pf_set_protostate(*state, psrc, PFOTHERS_SINGLE);
7283 		if (dst->state == PFOTHERS_SINGLE)
7284 			pf_set_protostate(*state, pdst, PFOTHERS_MULTIPLE);
7285 
7286 		/* update expire time */
7287 		(*state)->expire = pf_get_uptime();
7288 		if (src->state == PFOTHERS_MULTIPLE && dst->state == PFOTHERS_MULTIPLE)
7289 			(*state)->timeout = PFTM_OTHER_MULTIPLE;
7290 		else
7291 			(*state)->timeout = PFTM_OTHER_SINGLE;
7292 		break;
7293 	}
7294 
7295 	/* translate source/destination address, if necessary */
7296 	if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
7297 		struct pf_state_key	*nk;
7298 		int			 afto, sidx, didx;
7299 
7300 		if (PF_REVERSED_KEY(*state, pd->af))
7301 			nk = (*state)->key[pd->sidx];
7302 		else
7303 			nk = (*state)->key[pd->didx];
7304 
7305 		afto = pd->af != nk->af;
7306 
7307 		if (afto && (*state)->direction == PF_IN) {
7308 			sidx = pd->didx;
7309 			didx = pd->sidx;
7310 		} else {
7311 			sidx = pd->sidx;
7312 			didx = pd->didx;
7313 		}
7314 
7315 		if (afto) {
7316 			pf_addrcpy(&pd->nsaddr, &nk->addr[sidx], nk->af);
7317 			pf_addrcpy(&pd->ndaddr, &nk->addr[didx], nk->af);
7318 			pd->naf = nk->af;
7319 			action = PF_AFRT;
7320 		}
7321 
7322 		if (afto || PF_ANEQ(pd->src, &nk->addr[sidx], pd->af) ||
7323 		    nk->port[sidx] != pd->osport)
7324 			pf_change_ap(pd, pd->src, pd->sport,
7325 			    &nk->addr[sidx], nk->port[sidx]);
7326 
7327 		if (afto || PF_ANEQ(pd->dst, &nk->addr[didx], pd->af) ||
7328 		    nk->port[didx] != pd->odport)
7329 			pf_change_ap(pd, pd->dst, pd->dport,
7330 			    &nk->addr[didx], nk->port[didx]);
7331 
7332 		copyback = 1;
7333 	}
7334 
7335 	if (copyback && pd->hdrlen > 0)
7336 		m_copyback(pd->m, pd->off, pd->hdrlen, pd->hdr.any);
7337 
7338 	return (action);
7339 }
7340 
7341 static int
7342 pf_sctp_track(struct pf_kstate *state, struct pf_pdesc *pd,
7343     u_short *reason)
7344 {
7345 	struct pf_state_peer	*src;
7346 	if (pd->dir == state->direction) {
7347 		if (PF_REVERSED_KEY(state, pd->af))
7348 			src = &state->dst;
7349 		else
7350 			src = &state->src;
7351 	} else {
7352 		if (PF_REVERSED_KEY(state, pd->af))
7353 			src = &state->src;
7354 		else
7355 			src = &state->dst;
7356 	}
7357 
7358 	if (src->scrub != NULL) {
7359 		if (src->scrub->pfss_v_tag == 0)
7360 			src->scrub->pfss_v_tag = pd->hdr.sctp.v_tag;
7361 		else  if (src->scrub->pfss_v_tag != pd->hdr.sctp.v_tag)
7362 			return (PF_DROP);
7363 	}
7364 
7365 	return (PF_PASS);
7366 }
7367 
7368 static void
7369 pf_sctp_multihome_detach_addr(const struct pf_kstate *s)
7370 {
7371 	struct pf_sctp_endpoint key;
7372 	struct pf_sctp_endpoint *ep;
7373 	struct pf_state_key *sks = s->key[PF_SK_STACK];
7374 	struct pf_sctp_source *i, *tmp;
7375 
7376 	if (sks == NULL || sks->proto != IPPROTO_SCTP || s->dst.scrub == NULL)
7377 		return;
7378 
7379 	PF_SCTP_ENDPOINTS_LOCK();
7380 
7381 	key.v_tag = s->dst.scrub->pfss_v_tag;
7382 	ep  = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
7383 	if (ep != NULL) {
7384 		TAILQ_FOREACH_SAFE(i, &ep->sources, entry, tmp) {
7385 			if (pf_addr_cmp(&i->addr,
7386 			    &s->key[PF_SK_WIRE]->addr[s->direction == PF_OUT],
7387 			    s->key[PF_SK_WIRE]->af) == 0) {
7388 				SDT_PROBE3(pf, sctp, multihome, remove,
7389 				    key.v_tag, s, i);
7390 				TAILQ_REMOVE(&ep->sources, i, entry);
7391 				free(i, M_PFTEMP);
7392 				break;
7393 			}
7394 		}
7395 
7396 		if (TAILQ_EMPTY(&ep->sources)) {
7397 			RB_REMOVE(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
7398 			free(ep, M_PFTEMP);
7399 		}
7400 	}
7401 
7402 	/* Other direction. */
7403 	key.v_tag = s->src.scrub->pfss_v_tag;
7404 	ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
7405 	if (ep != NULL) {
7406 		TAILQ_FOREACH_SAFE(i, &ep->sources, entry, tmp) {
7407 			if (pf_addr_cmp(&i->addr,
7408 			    &s->key[PF_SK_WIRE]->addr[s->direction == PF_IN],
7409 			    s->key[PF_SK_WIRE]->af) == 0) {
7410 				SDT_PROBE3(pf, sctp, multihome, remove,
7411 				    key.v_tag, s, i);
7412 				TAILQ_REMOVE(&ep->sources, i, entry);
7413 				free(i, M_PFTEMP);
7414 				break;
7415 			}
7416 		}
7417 
7418 		if (TAILQ_EMPTY(&ep->sources)) {
7419 			RB_REMOVE(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
7420 			free(ep, M_PFTEMP);
7421 		}
7422 	}
7423 
7424 	PF_SCTP_ENDPOINTS_UNLOCK();
7425 }
7426 
7427 static void
7428 pf_sctp_multihome_add_addr(struct pf_pdesc *pd, struct pf_addr *a, uint32_t v_tag)
7429 {
7430 	struct pf_sctp_endpoint key = {
7431 		.v_tag = v_tag,
7432 	};
7433 	struct pf_sctp_source *i;
7434 	struct pf_sctp_endpoint *ep;
7435 	int count;
7436 
7437 	PF_SCTP_ENDPOINTS_LOCK();
7438 
7439 	ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
7440 	if (ep == NULL) {
7441 		ep = malloc(sizeof(struct pf_sctp_endpoint),
7442 		    M_PFTEMP, M_NOWAIT);
7443 		if (ep == NULL) {
7444 			PF_SCTP_ENDPOINTS_UNLOCK();
7445 			return;
7446 		}
7447 
7448 		ep->v_tag = v_tag;
7449 		TAILQ_INIT(&ep->sources);
7450 		RB_INSERT(pf_sctp_endpoints, &V_pf_sctp_endpoints, ep);
7451 	}
7452 
7453 	/* Avoid inserting duplicates. */
7454 	count = 0;
7455 	TAILQ_FOREACH(i, &ep->sources, entry) {
7456 		count++;
7457 		if (pf_addr_cmp(&i->addr, a, pd->af) == 0) {
7458 			PF_SCTP_ENDPOINTS_UNLOCK();
7459 			return;
7460 		}
7461 	}
7462 
7463 	/* Limit the number of addresses per endpoint. */
7464 	if (count >= PF_SCTP_MAX_ENDPOINTS) {
7465 		PF_SCTP_ENDPOINTS_UNLOCK();
7466 		return;
7467 	}
7468 
7469 	i = malloc(sizeof(*i), M_PFTEMP, M_NOWAIT);
7470 	if (i == NULL) {
7471 		PF_SCTP_ENDPOINTS_UNLOCK();
7472 		return;
7473 	}
7474 
7475 	i->af = pd->af;
7476 	memcpy(&i->addr, a, sizeof(*a));
7477 	TAILQ_INSERT_TAIL(&ep->sources, i, entry);
7478 	SDT_PROBE2(pf, sctp, multihome, add, v_tag, i);
7479 
7480 	PF_SCTP_ENDPOINTS_UNLOCK();
7481 }
7482 
7483 static void
7484 pf_sctp_multihome_delayed(struct pf_pdesc *pd, struct pfi_kkif *kif,
7485     struct pf_kstate *s, int action)
7486 {
7487 	struct pf_sctp_multihome_job	*j, *tmp;
7488 	struct pf_sctp_source		*i;
7489 	int			 ret;
7490 	struct pf_kstate	*sm = NULL;
7491 	struct pf_krule		*ra = NULL;
7492 	struct pf_krule		*r = &V_pf_default_rule;
7493 	struct pf_kruleset	*rs = NULL;
7494 	u_short			 reason;
7495 	bool do_extra = true;
7496 
7497 	PF_RULES_RLOCK_TRACKER;
7498 
7499 again:
7500 	TAILQ_FOREACH_SAFE(j, &pd->sctp_multihome_jobs, next, tmp) {
7501 		if (s == NULL || action != PF_PASS)
7502 			goto free;
7503 
7504 		/* Confirm we don't recurse here. */
7505 		MPASS(! (pd->sctp_flags & PFDESC_SCTP_ADD_IP));
7506 
7507 		switch (j->op) {
7508 		case  SCTP_ADD_IP_ADDRESS: {
7509 			uint32_t v_tag = pd->sctp_initiate_tag;
7510 
7511 			if (v_tag == 0) {
7512 				if (s->direction == pd->dir)
7513 					v_tag = s->src.scrub->pfss_v_tag;
7514 				else
7515 					v_tag = s->dst.scrub->pfss_v_tag;
7516 			}
7517 
7518 			/*
7519 			 * Avoid duplicating states. We'll already have
7520 			 * created a state based on the source address of
7521 			 * the packet, but SCTP endpoints may also list this
7522 			 * address again in the INIT(_ACK) parameters.
7523 			 */
7524 			if (pf_addr_cmp(&j->src, pd->src, pd->af) == 0) {
7525 				break;
7526 			}
7527 
7528 			j->pd.sctp_flags |= PFDESC_SCTP_ADD_IP;
7529 			PF_RULES_RLOCK();
7530 			sm = NULL;
7531 			if (s->rule->rule_flag & PFRULE_ALLOW_RELATED) {
7532 				j->pd.related_rule = s->rule;
7533 			}
7534 			ret = pf_test_rule(&r, &sm,
7535 			    &j->pd, &ra, &rs, &reason, NULL);
7536 			PF_RULES_RUNLOCK();
7537 			SDT_PROBE4(pf, sctp, multihome, test, kif, r, j->pd.m, ret);
7538 			if (ret != PF_DROP && sm != NULL) {
7539 				/* Inherit v_tag values. */
7540 				if (sm->direction == s->direction) {
7541 					sm->src.scrub->pfss_v_tag = s->src.scrub->pfss_v_tag;
7542 					sm->dst.scrub->pfss_v_tag = s->dst.scrub->pfss_v_tag;
7543 				} else {
7544 					sm->src.scrub->pfss_v_tag = s->dst.scrub->pfss_v_tag;
7545 					sm->dst.scrub->pfss_v_tag = s->src.scrub->pfss_v_tag;
7546 				}
7547 				PF_STATE_UNLOCK(sm);
7548 			} else {
7549 				/* If we try duplicate inserts? */
7550 				break;
7551 			}
7552 
7553 			/* Only add the address if we've actually allowed the state. */
7554 			pf_sctp_multihome_add_addr(pd, &j->src, v_tag);
7555 
7556 			if (! do_extra) {
7557 				break;
7558 			}
7559 			/*
7560 			 * We need to do this for each of our source addresses.
7561 			 * Find those based on the verification tag.
7562 			 */
7563 			struct pf_sctp_endpoint key = {
7564 				.v_tag = pd->hdr.sctp.v_tag,
7565 			};
7566 			struct pf_sctp_endpoint *ep;
7567 
7568 			PF_SCTP_ENDPOINTS_LOCK();
7569 			ep = RB_FIND(pf_sctp_endpoints, &V_pf_sctp_endpoints, &key);
7570 			if (ep == NULL) {
7571 				PF_SCTP_ENDPOINTS_UNLOCK();
7572 				break;
7573 			}
7574 			MPASS(ep != NULL);
7575 
7576 			TAILQ_FOREACH(i, &ep->sources, entry) {
7577 				struct pf_sctp_multihome_job *nj;
7578 
7579 				/* SCTP can intermingle IPv4 and IPv6. */
7580 				if (i->af != pd->af)
7581 					continue;
7582 
7583 				nj = malloc(sizeof(*nj), M_PFTEMP, M_NOWAIT | M_ZERO);
7584 				if (! nj) {
7585 					continue;
7586 				}
7587 				memcpy(&nj->pd, &j->pd, sizeof(j->pd));
7588 				memcpy(&nj->src, &j->src, sizeof(nj->src));
7589 				nj->pd.src = &nj->src;
7590 				// New destination address!
7591 				memcpy(&nj->dst, &i->addr, sizeof(nj->dst));
7592 				nj->pd.dst = &nj->dst;
7593 				nj->pd.m = j->pd.m;
7594 				nj->op = j->op;
7595 
7596 				TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, nj, next);
7597 			}
7598 			PF_SCTP_ENDPOINTS_UNLOCK();
7599 
7600 			break;
7601 		}
7602 		case SCTP_DEL_IP_ADDRESS: {
7603 			struct pf_state_key_cmp key;
7604 			uint8_t psrc;
7605 			int action;
7606 
7607 			bzero(&key, sizeof(key));
7608 			key.af = j->pd.af;
7609 			key.proto = IPPROTO_SCTP;
7610 			if (j->pd.dir == PF_IN)	{	/* wire side, straight */
7611 				pf_addrcpy(&key.addr[0], j->pd.src, key.af);
7612 				pf_addrcpy(&key.addr[1], j->pd.dst, key.af);
7613 				key.port[0] = j->pd.hdr.sctp.src_port;
7614 				key.port[1] = j->pd.hdr.sctp.dest_port;
7615 			} else {			/* stack side, reverse */
7616 				pf_addrcpy(&key.addr[1], j->pd.src, key.af);
7617 				pf_addrcpy(&key.addr[0], j->pd.dst, key.af);
7618 				key.port[1] = j->pd.hdr.sctp.src_port;
7619 				key.port[0] = j->pd.hdr.sctp.dest_port;
7620 			}
7621 
7622 			action = pf_find_state(&j->pd, &key, &sm);
7623 			if (action == PF_MATCH) {
7624 				PF_STATE_LOCK_ASSERT(sm);
7625 				if (j->pd.dir == sm->direction) {
7626 					psrc = PF_PEER_SRC;
7627 				} else {
7628 					psrc = PF_PEER_DST;
7629 				}
7630 				pf_set_protostate(sm, psrc, SCTP_SHUTDOWN_PENDING);
7631 				sm->timeout = PFTM_SCTP_CLOSING;
7632 				PF_STATE_UNLOCK(sm);
7633 			}
7634 			break;
7635 		default:
7636 			panic("Unknown op %#x", j->op);
7637 		}
7638 	}
7639 
7640 	free:
7641 		TAILQ_REMOVE(&pd->sctp_multihome_jobs, j, next);
7642 		free(j, M_PFTEMP);
7643 	}
7644 
7645 	/* We may have inserted extra work while processing the list. */
7646 	if (! TAILQ_EMPTY(&pd->sctp_multihome_jobs)) {
7647 		do_extra = false;
7648 		goto again;
7649 	}
7650 }
7651 
7652 static int
7653 pf_multihome_scan(int start, int len, struct pf_pdesc *pd, int op)
7654 {
7655 	int			 off = 0;
7656 	struct pf_sctp_multihome_job	*job;
7657 
7658 	SDT_PROBE4(pf, sctp, multihome_scan, entry, start, len, pd, op);
7659 
7660 	while (off < len) {
7661 		struct sctp_paramhdr h;
7662 
7663 		if (!pf_pull_hdr(pd->m, start + off, &h, sizeof(h), NULL, NULL,
7664 		    pd->af))
7665 			return (PF_DROP);
7666 
7667 		/* Parameters are at least 4 bytes. */
7668 		if (ntohs(h.param_length) < 4)
7669 			return (PF_DROP);
7670 
7671 		SDT_PROBE2(pf, sctp, multihome_scan, param, ntohs(h.param_type),
7672 		    ntohs(h.param_length));
7673 
7674 		switch (ntohs(h.param_type)) {
7675 		case  SCTP_IPV4_ADDRESS: {
7676 			struct in_addr t;
7677 
7678 			if (ntohs(h.param_length) !=
7679 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
7680 				return (PF_DROP);
7681 
7682 			if (!pf_pull_hdr(pd->m, start + off + sizeof(h), &t, sizeof(t),
7683 			    NULL, NULL, pd->af))
7684 				return (PF_DROP);
7685 
7686 			if (in_nullhost(t))
7687 				t.s_addr = pd->src->v4.s_addr;
7688 
7689 			/*
7690 			 * We hold the state lock (idhash) here, which means
7691 			 * that we can't acquire the keyhash, or we'll get a
7692 			 * LOR (and potentially double-lock things too). We also
7693 			 * can't release the state lock here, so instead we'll
7694 			 * enqueue this for async handling.
7695 			 * There's a relatively small race here, in that a
7696 			 * packet using the new addresses could arrive already,
7697 			 * but that's just though luck for it.
7698 			 */
7699 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
7700 			if (! job)
7701 				return (PF_DROP);
7702 
7703 			SDT_PROBE2(pf, sctp, multihome_scan, ipv4, &t, op);
7704 
7705 			memcpy(&job->pd, pd, sizeof(*pd));
7706 
7707 			// New source address!
7708 			memcpy(&job->src, &t, sizeof(t));
7709 			job->pd.src = &job->src;
7710 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
7711 			job->pd.dst = &job->dst;
7712 			job->pd.m = pd->m;
7713 			job->op = op;
7714 
7715 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
7716 			break;
7717 		}
7718 #ifdef INET6
7719 		case SCTP_IPV6_ADDRESS: {
7720 			struct in6_addr t;
7721 
7722 			if (ntohs(h.param_length) !=
7723 			    (sizeof(struct sctp_paramhdr) + sizeof(t)))
7724 				return (PF_DROP);
7725 
7726 			if (!pf_pull_hdr(pd->m, start + off + sizeof(h), &t, sizeof(t),
7727 			    NULL, NULL, pd->af))
7728 				return (PF_DROP);
7729 			if (memcmp(&t, &pd->src->v6, sizeof(t)) == 0)
7730 				break;
7731 			if (memcmp(&t, &in6addr_any, sizeof(t)) == 0)
7732 				memcpy(&t, &pd->src->v6, sizeof(t));
7733 
7734 			job = malloc(sizeof(*job), M_PFTEMP, M_NOWAIT | M_ZERO);
7735 			if (! job)
7736 				return (PF_DROP);
7737 
7738 			SDT_PROBE2(pf, sctp, multihome_scan, ipv6, &t, op);
7739 
7740 			memcpy(&job->pd, pd, sizeof(*pd));
7741 			memcpy(&job->src, &t, sizeof(t));
7742 			job->pd.src = &job->src;
7743 			memcpy(&job->dst, pd->dst, sizeof(job->dst));
7744 			job->pd.dst = &job->dst;
7745 			job->pd.m = pd->m;
7746 			job->op = op;
7747 
7748 			TAILQ_INSERT_TAIL(&pd->sctp_multihome_jobs, job, next);
7749 			break;
7750 		}
7751 #endif /* INET6 */
7752 		case SCTP_ADD_IP_ADDRESS: {
7753 			int ret;
7754 			struct sctp_asconf_paramhdr ah;
7755 
7756 			if (!pf_pull_hdr(pd->m, start + off, &ah, sizeof(ah),
7757 			    NULL, NULL, pd->af))
7758 				return (PF_DROP);
7759 
7760 			ret = pf_multihome_scan(start + off + sizeof(ah),
7761 			    ntohs(ah.ph.param_length) - sizeof(ah), pd,
7762 			    SCTP_ADD_IP_ADDRESS);
7763 			if (ret != PF_PASS)
7764 				return (ret);
7765 			break;
7766 		}
7767 		case SCTP_DEL_IP_ADDRESS: {
7768 			int ret;
7769 			struct sctp_asconf_paramhdr ah;
7770 
7771 			if (!pf_pull_hdr(pd->m, start + off, &ah, sizeof(ah),
7772 			    NULL, NULL, pd->af))
7773 				return (PF_DROP);
7774 			ret = pf_multihome_scan(start + off + sizeof(ah),
7775 			    ntohs(ah.ph.param_length) - sizeof(ah), pd,
7776 			    SCTP_DEL_IP_ADDRESS);
7777 			if (ret != PF_PASS)
7778 				return (ret);
7779 			break;
7780 		}
7781 		default:
7782 			break;
7783 		}
7784 
7785 		off += roundup(ntohs(h.param_length), 4);
7786 	}
7787 
7788 	return (PF_PASS);
7789 }
7790 
7791 int
7792 pf_multihome_scan_init(int start, int len, struct pf_pdesc *pd)
7793 {
7794 	start += sizeof(struct sctp_init_chunk);
7795 	len -= sizeof(struct sctp_init_chunk);
7796 
7797 	return (pf_multihome_scan(start, len, pd, SCTP_ADD_IP_ADDRESS));
7798 }
7799 
7800 int
7801 pf_multihome_scan_asconf(int start, int len, struct pf_pdesc *pd)
7802 {
7803 	start += sizeof(struct sctp_asconf_chunk);
7804 	len -= sizeof(struct sctp_asconf_chunk);
7805 
7806 	return (pf_multihome_scan(start, len, pd, SCTP_ADD_IP_ADDRESS));
7807 }
7808 
7809 int
7810 pf_icmp_state_lookup(struct pf_state_key_cmp *key, struct pf_pdesc *pd,
7811     struct pf_kstate **state, u_int16_t icmpid, u_int16_t type, int icmp_dir,
7812     int *iidx, int multi, int inner)
7813 {
7814 	int	 action, direction = pd->dir;
7815 
7816 	key->af = pd->af;
7817 	key->proto = pd->proto;
7818 	if (icmp_dir == PF_IN) {
7819 		*iidx = pd->sidx;
7820 		key->port[pd->sidx] = icmpid;
7821 		key->port[pd->didx] = type;
7822 	} else {
7823 		*iidx = pd->didx;
7824 		key->port[pd->sidx] = type;
7825 		key->port[pd->didx] = icmpid;
7826 	}
7827 	if (pf_state_key_addr_setup(pd, key, multi))
7828 		return (PF_DROP);
7829 
7830 	action = pf_find_state(pd, key, state);
7831 	if (action != PF_MATCH)
7832 		return (action);
7833 
7834 	if ((*state)->state_flags & PFSTATE_SLOPPY)
7835 		return (-1);
7836 
7837 	/* Is this ICMP message flowing in right direction? */
7838 	if ((*state)->key[PF_SK_WIRE]->af != (*state)->key[PF_SK_STACK]->af)
7839 		direction = (pd->af == (*state)->key[PF_SK_WIRE]->af) ?
7840 		    PF_IN : PF_OUT;
7841 	else
7842 		direction = (*state)->direction;
7843 	if ((*state)->rule->type &&
7844 	    (((!inner && direction == pd->dir) ||
7845 	    (inner && direction != pd->dir)) ?
7846 	    PF_IN : PF_OUT) != icmp_dir) {
7847 		if (V_pf_status.debug >= PF_DEBUG_MISC) {
7848 			printf("pf: icmp type %d in wrong direction (%d): ",
7849 			    ntohs(type), icmp_dir);
7850 			pf_print_state(*state);
7851 			printf("\n");
7852 		}
7853 		PF_STATE_UNLOCK(*state);
7854 		*state = NULL;
7855 		return (PF_DROP);
7856 	}
7857 	return (-1);
7858 }
7859 
7860 static int
7861 pf_test_state_icmp(struct pf_kstate **state, struct pf_pdesc *pd,
7862     u_short *reason)
7863 {
7864 	struct pf_addr  *saddr = pd->src, *daddr = pd->dst;
7865 	u_int16_t	*icmpsum, virtual_id, virtual_type;
7866 	u_int8_t	 icmptype, icmpcode;
7867 	int		 icmp_dir, iidx, ret;
7868 	struct pf_state_key_cmp key;
7869 #ifdef INET
7870 	u_int16_t	 icmpid;
7871 #endif /* INET*/
7872 
7873 	MPASS(*state == NULL);
7874 
7875 	bzero(&key, sizeof(key));
7876 	switch (pd->proto) {
7877 #ifdef INET
7878 	case IPPROTO_ICMP:
7879 		icmptype = pd->hdr.icmp.icmp_type;
7880 		icmpcode = pd->hdr.icmp.icmp_code;
7881 		icmpid = pd->hdr.icmp.icmp_id;
7882 		icmpsum = &pd->hdr.icmp.icmp_cksum;
7883 		break;
7884 #endif /* INET */
7885 #ifdef INET6
7886 	case IPPROTO_ICMPV6:
7887 		icmptype = pd->hdr.icmp6.icmp6_type;
7888 		icmpcode = pd->hdr.icmp6.icmp6_code;
7889 #ifdef INET
7890 		icmpid = pd->hdr.icmp6.icmp6_id;
7891 #endif /* INET */
7892 		icmpsum = &pd->hdr.icmp6.icmp6_cksum;
7893 		break;
7894 #endif /* INET6 */
7895 	default:
7896 		panic("unhandled proto %d", pd->proto);
7897 	}
7898 
7899 	if (pf_icmp_mapping(pd, icmptype, &icmp_dir, &virtual_id,
7900 	    &virtual_type) == 0) {
7901 		/*
7902 		 * ICMP query/reply message not related to a TCP/UDP/SCTP
7903 		 * packet. Search for an ICMP state.
7904 		 */
7905 		ret = pf_icmp_state_lookup(&key, pd, state, virtual_id,
7906 		    virtual_type, icmp_dir, &iidx, 0, 0);
7907 		/* IPv6? try matching a multicast address */
7908 		if (ret == PF_DROP && pd->af == AF_INET6 && icmp_dir == PF_OUT) {
7909 			MPASS(*state == NULL);
7910 			ret = pf_icmp_state_lookup(&key, pd, state,
7911 			    virtual_id, virtual_type,
7912 			    icmp_dir, &iidx, 1, 0);
7913 		}
7914 		if (ret >= 0) {
7915 			MPASS(*state == NULL);
7916 			return (ret);
7917 		}
7918 
7919 		(*state)->expire = pf_get_uptime();
7920 		(*state)->timeout = PFTM_ICMP_ERROR_REPLY;
7921 
7922 		/* translate source/destination address, if necessary */
7923 		if ((*state)->key[PF_SK_WIRE] != (*state)->key[PF_SK_STACK]) {
7924 			struct pf_state_key	*nk;
7925 			int			 afto, sidx, didx;
7926 
7927 			if (PF_REVERSED_KEY(*state, pd->af))
7928 				nk = (*state)->key[pd->sidx];
7929 			else
7930 				nk = (*state)->key[pd->didx];
7931 
7932 			afto = pd->af != nk->af;
7933 
7934 			if (afto && (*state)->direction == PF_IN) {
7935 				sidx = pd->didx;
7936 				didx = pd->sidx;
7937 				iidx = !iidx;
7938 			} else {
7939 				sidx = pd->sidx;
7940 				didx = pd->didx;
7941 			}
7942 
7943 			switch (pd->af) {
7944 #ifdef INET
7945 			case AF_INET:
7946 #ifdef INET6
7947 				if (afto) {
7948 					if (pf_translate_icmp_af(AF_INET6,
7949 					    &pd->hdr.icmp))
7950 						return (PF_DROP);
7951 					pd->proto = IPPROTO_ICMPV6;
7952 				}
7953 #endif /* INET6 */
7954 				if (!afto &&
7955 				    PF_ANEQ(pd->src, &nk->addr[sidx], AF_INET))
7956 					pf_change_a(&saddr->v4.s_addr,
7957 					    pd->ip_sum,
7958 					    nk->addr[sidx].v4.s_addr,
7959 					    0);
7960 
7961 				if (!afto && PF_ANEQ(pd->dst,
7962 				    &nk->addr[didx], AF_INET))
7963 					pf_change_a(&daddr->v4.s_addr,
7964 					    pd->ip_sum,
7965 					    nk->addr[didx].v4.s_addr, 0);
7966 
7967 				if (nk->port[iidx] !=
7968 				    pd->hdr.icmp.icmp_id) {
7969 					pd->hdr.icmp.icmp_cksum =
7970 					    pf_cksum_fixup(
7971 					    pd->hdr.icmp.icmp_cksum, icmpid,
7972 					    nk->port[iidx], 0);
7973 					pd->hdr.icmp.icmp_id =
7974 					    nk->port[iidx];
7975 				}
7976 
7977 				m_copyback(pd->m, pd->off, ICMP_MINLEN,
7978 				    (caddr_t )&pd->hdr.icmp);
7979 				break;
7980 #endif /* INET */
7981 #ifdef INET6
7982 			case AF_INET6:
7983 #ifdef INET
7984 				if (afto) {
7985 					if (pf_translate_icmp_af(AF_INET,
7986 					    &pd->hdr.icmp6))
7987 						return (PF_DROP);
7988 					pd->proto = IPPROTO_ICMP;
7989 				}
7990 #endif /* INET */
7991 				if (!afto &&
7992 				    PF_ANEQ(pd->src, &nk->addr[sidx], AF_INET6))
7993 					pf_change_a6(saddr,
7994 					    &pd->hdr.icmp6.icmp6_cksum,
7995 					    &nk->addr[sidx], 0);
7996 
7997 				if (!afto && PF_ANEQ(pd->dst,
7998 				    &nk->addr[didx], AF_INET6))
7999 					pf_change_a6(daddr,
8000 					    &pd->hdr.icmp6.icmp6_cksum,
8001 					    &nk->addr[didx], 0);
8002 
8003 				if (nk->port[iidx] != pd->hdr.icmp6.icmp6_id)
8004 					pd->hdr.icmp6.icmp6_id =
8005 					    nk->port[iidx];
8006 
8007 				m_copyback(pd->m, pd->off, sizeof(struct icmp6_hdr),
8008 				    (caddr_t )&pd->hdr.icmp6);
8009 				break;
8010 #endif /* INET6 */
8011 			}
8012 			if (afto) {
8013 				pf_addrcpy(&pd->nsaddr, &nk->addr[sidx],
8014 				    nk->af);
8015 				pf_addrcpy(&pd->ndaddr, &nk->addr[didx],
8016 				    nk->af);
8017 				pd->naf = nk->af;
8018 				return (PF_AFRT);
8019 			}
8020 		}
8021 		return (PF_PASS);
8022 
8023 	} else {
8024 		/*
8025 		 * ICMP error message in response to a TCP/UDP packet.
8026 		 * Extract the inner TCP/UDP header and search for that state.
8027 		 */
8028 
8029 		struct pf_pdesc	pd2;
8030 		bzero(&pd2, sizeof pd2);
8031 #ifdef INET
8032 		struct ip	h2;
8033 #endif /* INET */
8034 #ifdef INET6
8035 		struct ip6_hdr	h2_6;
8036 #endif /* INET6 */
8037 		int		ipoff2 = 0;
8038 
8039 		pd2.af = pd->af;
8040 		pd2.dir = pd->dir;
8041 		/* Payload packet is from the opposite direction. */
8042 		pd2.sidx = (pd->dir == PF_IN) ? 1 : 0;
8043 		pd2.didx = (pd->dir == PF_IN) ? 0 : 1;
8044 		pd2.m = pd->m;
8045 		pd2.pf_mtag = pd->pf_mtag;
8046 		pd2.kif = pd->kif;
8047 		switch (pd->af) {
8048 #ifdef INET
8049 		case AF_INET:
8050 			/* offset of h2 in mbuf chain */
8051 			ipoff2 = pd->off + ICMP_MINLEN;
8052 
8053 			if (!pf_pull_hdr(pd->m, ipoff2, &h2, sizeof(h2),
8054 			    NULL, reason, pd2.af)) {
8055 				DPFPRINTF(PF_DEBUG_MISC,
8056 				    "pf: ICMP error message too short "
8057 				    "(ip)");
8058 				return (PF_DROP);
8059 			}
8060 			/*
8061 			 * ICMP error messages don't refer to non-first
8062 			 * fragments
8063 			 */
8064 			if (h2.ip_off & htons(IP_OFFMASK)) {
8065 				REASON_SET(reason, PFRES_FRAG);
8066 				return (PF_DROP);
8067 			}
8068 
8069 			/* offset of protocol header that follows h2 */
8070 			pd2.off = ipoff2;
8071 			if (pf_walk_header(&pd2, &h2, reason) != PF_PASS)
8072 				return (PF_DROP);
8073 
8074 			pd2.tot_len = ntohs(h2.ip_len);
8075 			pd2.ttl = h2.ip_ttl;
8076 			pd2.src = (struct pf_addr *)&h2.ip_src;
8077 			pd2.dst = (struct pf_addr *)&h2.ip_dst;
8078 			pd2.ip_sum = &h2.ip_sum;
8079 			break;
8080 #endif /* INET */
8081 #ifdef INET6
8082 		case AF_INET6:
8083 			ipoff2 = pd->off + sizeof(struct icmp6_hdr);
8084 
8085 			if (!pf_pull_hdr(pd->m, ipoff2, &h2_6, sizeof(h2_6),
8086 			    NULL, reason, pd2.af)) {
8087 				DPFPRINTF(PF_DEBUG_MISC,
8088 				    "pf: ICMP error message too short "
8089 				    "(ip6)");
8090 				return (PF_DROP);
8091 			}
8092 			pd2.off = ipoff2;
8093 			if (pf_walk_header6(&pd2, &h2_6, reason) != PF_PASS)
8094 				return (PF_DROP);
8095 
8096 			pd2.tot_len = ntohs(h2_6.ip6_plen) +
8097 			    sizeof(struct ip6_hdr);
8098 			pd2.ttl = h2_6.ip6_hlim;
8099 			pd2.src = (struct pf_addr *)&h2_6.ip6_src;
8100 			pd2.dst = (struct pf_addr *)&h2_6.ip6_dst;
8101 			pd2.ip_sum = NULL;
8102 			break;
8103 #endif /* INET6 */
8104 		default:
8105 			unhandled_af(pd->af);
8106 		}
8107 
8108 		if (PF_ANEQ(pd->dst, pd2.src, pd->af)) {
8109 			if (V_pf_status.debug >= PF_DEBUG_MISC) {
8110 				printf("pf: BAD ICMP %d:%d outer dst: ",
8111 				    icmptype, icmpcode);
8112 				pf_print_host(pd->src, 0, pd->af);
8113 				printf(" -> ");
8114 				pf_print_host(pd->dst, 0, pd->af);
8115 				printf(" inner src: ");
8116 				pf_print_host(pd2.src, 0, pd2.af);
8117 				printf(" -> ");
8118 				pf_print_host(pd2.dst, 0, pd2.af);
8119 				printf("\n");
8120 			}
8121 			REASON_SET(reason, PFRES_BADSTATE);
8122 			return (PF_DROP);
8123 		}
8124 
8125 		switch (pd2.proto) {
8126 		case IPPROTO_TCP: {
8127 			struct tcphdr		*th = &pd2.hdr.tcp;
8128 			u_int32_t		 seq;
8129 			struct pf_state_peer	*src, *dst;
8130 			u_int8_t		 dws;
8131 			int			 copyback = 0;
8132 			int			 action;
8133 
8134 			/*
8135 			 * Only the first 8 bytes of the TCP header can be
8136 			 * expected. Don't access any TCP header fields after
8137 			 * th_seq, an ackskew test is not possible.
8138 			 */
8139 			if (!pf_pull_hdr(pd->m, pd2.off, th, 8, NULL, reason,
8140 			    pd2.af)) {
8141 				DPFPRINTF(PF_DEBUG_MISC,
8142 				    "pf: ICMP error message too short "
8143 				    "(tcp)");
8144 				return (PF_DROP);
8145 			}
8146 			pd2.pcksum = &pd2.hdr.tcp.th_sum;
8147 
8148 			key.af = pd2.af;
8149 			key.proto = IPPROTO_TCP;
8150 			pf_addrcpy(&key.addr[pd2.sidx], pd2.src, key.af);
8151 			pf_addrcpy(&key.addr[pd2.didx], pd2.dst, key.af);
8152 			key.port[pd2.sidx] = th->th_sport;
8153 			key.port[pd2.didx] = th->th_dport;
8154 
8155 			action = pf_find_state(&pd2, &key, state);
8156 			if (action != PF_MATCH)
8157 				return (action);
8158 
8159 			if (pd->dir == (*state)->direction) {
8160 				if (PF_REVERSED_KEY(*state, pd->af)) {
8161 					src = &(*state)->src;
8162 					dst = &(*state)->dst;
8163 				} else {
8164 					src = &(*state)->dst;
8165 					dst = &(*state)->src;
8166 				}
8167 			} else {
8168 				if (PF_REVERSED_KEY(*state, pd->af)) {
8169 					src = &(*state)->dst;
8170 					dst = &(*state)->src;
8171 				} else {
8172 					src = &(*state)->src;
8173 					dst = &(*state)->dst;
8174 				}
8175 			}
8176 
8177 			if (src->wscale && dst->wscale)
8178 				dws = dst->wscale & PF_WSCALE_MASK;
8179 			else
8180 				dws = 0;
8181 
8182 			/* Demodulate sequence number */
8183 			seq = ntohl(th->th_seq) - src->seqdiff;
8184 			if (src->seqdiff) {
8185 				pf_change_a(&th->th_seq, icmpsum,
8186 				    htonl(seq), 0);
8187 				copyback = 1;
8188 			}
8189 
8190 			if (!((*state)->state_flags & PFSTATE_SLOPPY) &&
8191 			    (!SEQ_GEQ(src->seqhi, seq) ||
8192 			    !SEQ_GEQ(seq, src->seqlo - (dst->max_win << dws)))) {
8193 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
8194 					printf("pf: BAD ICMP %d:%d ",
8195 					    icmptype, icmpcode);
8196 					pf_print_host(pd->src, 0, pd->af);
8197 					printf(" -> ");
8198 					pf_print_host(pd->dst, 0, pd->af);
8199 					printf(" state: ");
8200 					pf_print_state(*state);
8201 					printf(" seq=%u\n", seq);
8202 				}
8203 				REASON_SET(reason, PFRES_BADSTATE);
8204 				return (PF_DROP);
8205 			} else {
8206 				if (V_pf_status.debug >= PF_DEBUG_MISC) {
8207 					printf("pf: OK ICMP %d:%d ",
8208 					    icmptype, icmpcode);
8209 					pf_print_host(pd->src, 0, pd->af);
8210 					printf(" -> ");
8211 					pf_print_host(pd->dst, 0, pd->af);
8212 					printf(" state: ");
8213 					pf_print_state(*state);
8214 					printf(" seq=%u\n", seq);
8215 				}
8216 			}
8217 
8218 			/* translate source/destination address, if necessary */
8219 			if ((*state)->key[PF_SK_WIRE] !=
8220 			    (*state)->key[PF_SK_STACK]) {
8221 
8222 				struct pf_state_key	*nk;
8223 
8224 				if (PF_REVERSED_KEY(*state, pd->af))
8225 					nk = (*state)->key[pd->sidx];
8226 				else
8227 					nk = (*state)->key[pd->didx];
8228 
8229 #if defined(INET) && defined(INET6)
8230 				int		 afto, sidx, didx;
8231 
8232 				afto = pd->af != nk->af;
8233 
8234 				if (afto && (*state)->direction == PF_IN) {
8235 					sidx = pd2.didx;
8236 					didx = pd2.sidx;
8237 				} else {
8238 					sidx = pd2.sidx;
8239 					didx = pd2.didx;
8240 				}
8241 
8242 				if (afto) {
8243 					if (pf_translate_icmp_af(nk->af,
8244 					    &pd->hdr.icmp))
8245 						return (PF_DROP);
8246 					m_copyback(pd->m, pd->off,
8247 					    sizeof(struct icmp6_hdr),
8248 					    (c_caddr_t)&pd->hdr.icmp6);
8249 					if (pf_change_icmp_af(pd->m, ipoff2, pd,
8250 					    &pd2, &nk->addr[sidx],
8251 					    &nk->addr[didx], pd->af,
8252 					    nk->af))
8253 						return (PF_DROP);
8254 					pf_addrcpy(&pd->nsaddr,
8255 					    &nk->addr[pd2.sidx], nk->af);
8256 					pf_addrcpy(&pd->ndaddr,
8257 					    &nk->addr[pd2.didx], nk->af);
8258 					if (nk->af == AF_INET) {
8259 						pd->proto = IPPROTO_ICMP;
8260 					} else {
8261 						pd->proto = IPPROTO_ICMPV6;
8262 						/*
8263 						 * IPv4 becomes IPv6 so we must
8264 						 * copy IPv4 src addr to least
8265 						 * 32bits in IPv6 address to
8266 						 * keep traceroute/icmp
8267 						 * working.
8268 						 */
8269 						pd->nsaddr.addr32[3] =
8270 						    pd->src->addr32[0];
8271 					}
8272 					pd->naf = pd2.naf = nk->af;
8273 					pf_change_ap(&pd2, pd2.src, &th->th_sport,
8274 					    &nk->addr[pd2.sidx], nk->port[sidx]);
8275 					pf_change_ap(&pd2, pd2.dst, &th->th_dport,
8276 					    &nk->addr[pd2.didx], nk->port[didx]);
8277 					m_copyback(pd2.m, pd2.off, 8, (c_caddr_t)th);
8278 					return (PF_AFRT);
8279 				}
8280 #endif /* INET && INET6 */
8281 
8282 				if (PF_ANEQ(pd2.src,
8283 				    &nk->addr[pd2.sidx], pd2.af) ||
8284 				    nk->port[pd2.sidx] != th->th_sport)
8285 					pf_change_icmp(pd2.src, &th->th_sport,
8286 					    daddr, &nk->addr[pd2.sidx],
8287 					    nk->port[pd2.sidx], NULL,
8288 					    pd2.ip_sum, icmpsum,
8289 					    pd->ip_sum, 0, pd2.af);
8290 
8291 				if (PF_ANEQ(pd2.dst,
8292 				    &nk->addr[pd2.didx], pd2.af) ||
8293 				    nk->port[pd2.didx] != th->th_dport)
8294 					pf_change_icmp(pd2.dst, &th->th_dport,
8295 					    saddr, &nk->addr[pd2.didx],
8296 					    nk->port[pd2.didx], NULL,
8297 					    pd2.ip_sum, icmpsum,
8298 					    pd->ip_sum, 0, pd2.af);
8299 				copyback = 1;
8300 			}
8301 
8302 			if (copyback) {
8303 				switch (pd2.af) {
8304 #ifdef INET
8305 				case AF_INET:
8306 					m_copyback(pd->m, pd->off, ICMP_MINLEN,
8307 					    (caddr_t )&pd->hdr.icmp);
8308 					m_copyback(pd->m, ipoff2, sizeof(h2),
8309 					    (caddr_t )&h2);
8310 					break;
8311 #endif /* INET */
8312 #ifdef INET6
8313 				case AF_INET6:
8314 					m_copyback(pd->m, pd->off,
8315 					    sizeof(struct icmp6_hdr),
8316 					    (caddr_t )&pd->hdr.icmp6);
8317 					m_copyback(pd->m, ipoff2, sizeof(h2_6),
8318 					    (caddr_t )&h2_6);
8319 					break;
8320 #endif /* INET6 */
8321 				default:
8322 					unhandled_af(pd->af);
8323 				}
8324 				m_copyback(pd->m, pd2.off, 8, (caddr_t)th);
8325 			}
8326 
8327 			return (PF_PASS);
8328 			break;
8329 		}
8330 		case IPPROTO_UDP: {
8331 			struct udphdr		*uh = &pd2.hdr.udp;
8332 			int			 action;
8333 
8334 			if (!pf_pull_hdr(pd->m, pd2.off, uh, sizeof(*uh),
8335 			    NULL, reason, pd2.af)) {
8336 				DPFPRINTF(PF_DEBUG_MISC,
8337 				    "pf: ICMP error message too short "
8338 				    "(udp)");
8339 				return (PF_DROP);
8340 			}
8341 			pd2.pcksum = &pd2.hdr.udp.uh_sum;
8342 
8343 			key.af = pd2.af;
8344 			key.proto = IPPROTO_UDP;
8345 			pf_addrcpy(&key.addr[pd2.sidx], pd2.src, key.af);
8346 			pf_addrcpy(&key.addr[pd2.didx], pd2.dst, key.af);
8347 			key.port[pd2.sidx] = uh->uh_sport;
8348 			key.port[pd2.didx] = uh->uh_dport;
8349 
8350 			action = pf_find_state(&pd2, &key, state);
8351 			if (action != PF_MATCH)
8352 				return (action);
8353 
8354 			/* translate source/destination address, if necessary */
8355 			if ((*state)->key[PF_SK_WIRE] !=
8356 			    (*state)->key[PF_SK_STACK]) {
8357 				struct pf_state_key	*nk;
8358 
8359 				if (PF_REVERSED_KEY(*state, pd->af))
8360 					nk = (*state)->key[pd->sidx];
8361 				else
8362 					nk = (*state)->key[pd->didx];
8363 
8364 #if defined(INET) && defined(INET6)
8365 				int	 afto, sidx, didx;
8366 
8367 				afto = pd->af != nk->af;
8368 
8369 				if (afto && (*state)->direction == PF_IN) {
8370 					sidx = pd2.didx;
8371 					didx = pd2.sidx;
8372 				} else {
8373 					sidx = pd2.sidx;
8374 					didx = pd2.didx;
8375 				}
8376 
8377 				if (afto) {
8378 					if (pf_translate_icmp_af(nk->af,
8379 					    &pd->hdr.icmp))
8380 						return (PF_DROP);
8381 					m_copyback(pd->m, pd->off,
8382 					    sizeof(struct icmp6_hdr),
8383 					    (c_caddr_t)&pd->hdr.icmp6);
8384 					if (pf_change_icmp_af(pd->m, ipoff2, pd,
8385 					    &pd2, &nk->addr[sidx],
8386 					    &nk->addr[didx], pd->af,
8387 					    nk->af))
8388 						return (PF_DROP);
8389 					pf_addrcpy(&pd->nsaddr,
8390 					    &nk->addr[pd2.sidx], nk->af);
8391 					pf_addrcpy(&pd->ndaddr,
8392 					    &nk->addr[pd2.didx], nk->af);
8393 					if (nk->af == AF_INET) {
8394 						pd->proto = IPPROTO_ICMP;
8395 					} else {
8396 						pd->proto = IPPROTO_ICMPV6;
8397 						/*
8398 						 * IPv4 becomes IPv6 so we must
8399 						 * copy IPv4 src addr to least
8400 						 * 32bits in IPv6 address to
8401 						 * keep traceroute/icmp
8402 						 * working.
8403 						 */
8404 						pd->nsaddr.addr32[3] =
8405 						    pd->src->addr32[0];
8406 					}
8407 					pd->naf = pd2.naf = nk->af;
8408 					pf_change_ap(&pd2, pd2.src, &uh->uh_sport,
8409 					    &nk->addr[pd2.sidx], nk->port[sidx]);
8410 					pf_change_ap(&pd2, pd2.dst, &uh->uh_dport,
8411 					    &nk->addr[pd2.didx], nk->port[didx]);
8412 					m_copyback(pd2.m, pd2.off, sizeof(*uh),
8413 					    (c_caddr_t)uh);
8414 					return (PF_AFRT);
8415 				}
8416 #endif /* INET && INET6 */
8417 
8418 				if (PF_ANEQ(pd2.src,
8419 				    &nk->addr[pd2.sidx], pd2.af) ||
8420 				    nk->port[pd2.sidx] != uh->uh_sport)
8421 					pf_change_icmp(pd2.src, &uh->uh_sport,
8422 					    daddr, &nk->addr[pd2.sidx],
8423 					    nk->port[pd2.sidx], &uh->uh_sum,
8424 					    pd2.ip_sum, icmpsum,
8425 					    pd->ip_sum, 1, pd2.af);
8426 
8427 				if (PF_ANEQ(pd2.dst,
8428 				    &nk->addr[pd2.didx], pd2.af) ||
8429 				    nk->port[pd2.didx] != uh->uh_dport)
8430 					pf_change_icmp(pd2.dst, &uh->uh_dport,
8431 					    saddr, &nk->addr[pd2.didx],
8432 					    nk->port[pd2.didx], &uh->uh_sum,
8433 					    pd2.ip_sum, icmpsum,
8434 					    pd->ip_sum, 1, pd2.af);
8435 
8436 				switch (pd2.af) {
8437 #ifdef INET
8438 				case AF_INET:
8439 					m_copyback(pd->m, pd->off, ICMP_MINLEN,
8440 					    (caddr_t )&pd->hdr.icmp);
8441 					m_copyback(pd->m, ipoff2, sizeof(h2), (caddr_t)&h2);
8442 					break;
8443 #endif /* INET */
8444 #ifdef INET6
8445 				case AF_INET6:
8446 					m_copyback(pd->m, pd->off,
8447 					    sizeof(struct icmp6_hdr),
8448 					    (caddr_t )&pd->hdr.icmp6);
8449 					m_copyback(pd->m, ipoff2, sizeof(h2_6),
8450 					    (caddr_t )&h2_6);
8451 					break;
8452 #endif /* INET6 */
8453 				}
8454 				m_copyback(pd->m, pd2.off, sizeof(*uh), (caddr_t)uh);
8455 			}
8456 			return (PF_PASS);
8457 			break;
8458 		}
8459 #ifdef INET
8460 		case IPPROTO_SCTP: {
8461 			struct sctphdr		*sh = &pd2.hdr.sctp;
8462 			struct pf_state_peer	*src;
8463 			int			 copyback = 0;
8464 			int			 action;
8465 
8466 			if (! pf_pull_hdr(pd->m, pd2.off, sh, sizeof(*sh), NULL, reason,
8467 			    pd2.af)) {
8468 				DPFPRINTF(PF_DEBUG_MISC,
8469 				    "pf: ICMP error message too short "
8470 				    "(sctp)");
8471 				return (PF_DROP);
8472 			}
8473 			pd2.pcksum = &pd2.sctp_dummy_sum;
8474 
8475 			key.af = pd2.af;
8476 			key.proto = IPPROTO_SCTP;
8477 			pf_addrcpy(&key.addr[pd2.sidx], pd2.src, key.af);
8478 			pf_addrcpy(&key.addr[pd2.didx], pd2.dst, key.af);
8479 			key.port[pd2.sidx] = sh->src_port;
8480 			key.port[pd2.didx] = sh->dest_port;
8481 
8482 			action = pf_find_state(&pd2, &key, state);
8483 			if (action != PF_MATCH)
8484 				return (action);
8485 
8486 			if (pd->dir == (*state)->direction) {
8487 				if (PF_REVERSED_KEY(*state, pd->af))
8488 					src = &(*state)->src;
8489 				else
8490 					src = &(*state)->dst;
8491 			} else {
8492 				if (PF_REVERSED_KEY(*state, pd->af))
8493 					src = &(*state)->dst;
8494 				else
8495 					src = &(*state)->src;
8496 			}
8497 
8498 			if (src->scrub->pfss_v_tag != sh->v_tag) {
8499 				DPFPRINTF(PF_DEBUG_MISC,
8500 				    "pf: ICMP error message has incorrect "
8501 				    "SCTP v_tag");
8502 				return (PF_DROP);
8503 			}
8504 
8505 			/* translate source/destination address, if necessary */
8506 			if ((*state)->key[PF_SK_WIRE] !=
8507 			    (*state)->key[PF_SK_STACK]) {
8508 
8509 				struct pf_state_key	*nk;
8510 
8511 				if (PF_REVERSED_KEY(*state, pd->af))
8512 					nk = (*state)->key[pd->sidx];
8513 				else
8514 					nk = (*state)->key[pd->didx];
8515 
8516 #if defined(INET) && defined(INET6)
8517 				int	 afto, sidx, didx;
8518 
8519 				afto = pd->af != nk->af;
8520 
8521 				if (afto && (*state)->direction == PF_IN) {
8522 					sidx = pd2.didx;
8523 					didx = pd2.sidx;
8524 				} else {
8525 					sidx = pd2.sidx;
8526 					didx = pd2.didx;
8527 				}
8528 
8529 				if (afto) {
8530 					if (pf_translate_icmp_af(nk->af,
8531 					    &pd->hdr.icmp))
8532 						return (PF_DROP);
8533 					m_copyback(pd->m, pd->off,
8534 					    sizeof(struct icmp6_hdr),
8535 					    (c_caddr_t)&pd->hdr.icmp6);
8536 					if (pf_change_icmp_af(pd->m, ipoff2, pd,
8537 					    &pd2, &nk->addr[sidx],
8538 					    &nk->addr[didx], pd->af,
8539 					    nk->af))
8540 						return (PF_DROP);
8541 					sh->src_port = nk->port[sidx];
8542 					sh->dest_port = nk->port[didx];
8543 					m_copyback(pd2.m, pd2.off, sizeof(*sh), (c_caddr_t)sh);
8544 					pf_addrcpy(&pd->nsaddr,
8545 					    &nk->addr[pd2.sidx], nk->af);
8546 					pf_addrcpy(&pd->ndaddr,
8547 					    &nk->addr[pd2.didx], nk->af);
8548 					if (nk->af == AF_INET) {
8549 						pd->proto = IPPROTO_ICMP;
8550 					} else {
8551 						pd->proto = IPPROTO_ICMPV6;
8552 						/*
8553 						 * IPv4 becomes IPv6 so we must
8554 						 * copy IPv4 src addr to least
8555 						 * 32bits in IPv6 address to
8556 						 * keep traceroute/icmp
8557 						 * working.
8558 						 */
8559 						pd->nsaddr.addr32[3] =
8560 						    pd->src->addr32[0];
8561 					}
8562 					pd->naf = nk->af;
8563 					return (PF_AFRT);
8564 				}
8565 #endif /* INET && INET6 */
8566 
8567 				if (PF_ANEQ(pd2.src,
8568 				    &nk->addr[pd2.sidx], pd2.af) ||
8569 				    nk->port[pd2.sidx] != sh->src_port)
8570 					pf_change_icmp(pd2.src, &sh->src_port,
8571 					    daddr, &nk->addr[pd2.sidx],
8572 					    nk->port[pd2.sidx], NULL,
8573 					    pd2.ip_sum, icmpsum,
8574 					    pd->ip_sum, 0, pd2.af);
8575 
8576 				if (PF_ANEQ(pd2.dst,
8577 				    &nk->addr[pd2.didx], pd2.af) ||
8578 				    nk->port[pd2.didx] != sh->dest_port)
8579 					pf_change_icmp(pd2.dst, &sh->dest_port,
8580 					    saddr, &nk->addr[pd2.didx],
8581 					    nk->port[pd2.didx], NULL,
8582 					    pd2.ip_sum, icmpsum,
8583 					    pd->ip_sum, 0, pd2.af);
8584 				copyback = 1;
8585 			}
8586 
8587 			if (copyback) {
8588 				switch (pd2.af) {
8589 #ifdef INET
8590 				case AF_INET:
8591 					m_copyback(pd->m, pd->off, ICMP_MINLEN,
8592 					    (caddr_t )&pd->hdr.icmp);
8593 					m_copyback(pd->m, ipoff2, sizeof(h2),
8594 					    (caddr_t )&h2);
8595 					break;
8596 #endif /* INET */
8597 #ifdef INET6
8598 				case AF_INET6:
8599 					m_copyback(pd->m, pd->off,
8600 					    sizeof(struct icmp6_hdr),
8601 					    (caddr_t )&pd->hdr.icmp6);
8602 					m_copyback(pd->m, ipoff2, sizeof(h2_6),
8603 					    (caddr_t )&h2_6);
8604 					break;
8605 #endif /* INET6 */
8606 				}
8607 				m_copyback(pd->m, pd2.off, sizeof(*sh), (caddr_t)sh);
8608 			}
8609 
8610 			return (PF_PASS);
8611 			break;
8612 		}
8613 		case IPPROTO_ICMP: {
8614 			struct icmp	*iih = &pd2.hdr.icmp;
8615 
8616 			if (pd2.af != AF_INET) {
8617 				REASON_SET(reason, PFRES_NORM);
8618 				return (PF_DROP);
8619 			}
8620 
8621 			if (!pf_pull_hdr(pd->m, pd2.off, iih, ICMP_MINLEN,
8622 			    NULL, reason, pd2.af)) {
8623 				DPFPRINTF(PF_DEBUG_MISC,
8624 				    "pf: ICMP error message too short i"
8625 				    "(icmp)");
8626 				return (PF_DROP);
8627 			}
8628 			pd2.pcksum = &pd2.hdr.icmp.icmp_cksum;
8629 
8630 			icmpid = iih->icmp_id;
8631 			pf_icmp_mapping(&pd2, iih->icmp_type,
8632 			    &icmp_dir, &virtual_id, &virtual_type);
8633 
8634 			ret = pf_icmp_state_lookup(&key, &pd2, state,
8635 			    virtual_id, virtual_type, icmp_dir, &iidx, 0, 1);
8636 			if (ret >= 0) {
8637 				MPASS(*state == NULL);
8638 				return (ret);
8639 			}
8640 
8641 			/* translate source/destination address, if necessary */
8642 			if ((*state)->key[PF_SK_WIRE] !=
8643 			    (*state)->key[PF_SK_STACK]) {
8644 				struct pf_state_key	*nk;
8645 
8646 				if (PF_REVERSED_KEY(*state, pd->af))
8647 					nk = (*state)->key[pd->sidx];
8648 				else
8649 					nk = (*state)->key[pd->didx];
8650 
8651 #if defined(INET) && defined(INET6)
8652 				int	 afto, sidx, didx;
8653 
8654 				afto = pd->af != nk->af;
8655 
8656 				if (afto && (*state)->direction == PF_IN) {
8657 					sidx = pd2.didx;
8658 					didx = pd2.sidx;
8659 					iidx = !iidx;
8660 				} else {
8661 					sidx = pd2.sidx;
8662 					didx = pd2.didx;
8663 				}
8664 
8665 				if (afto) {
8666 					if (nk->af != AF_INET6)
8667 						return (PF_DROP);
8668 					if (pf_translate_icmp_af(nk->af,
8669 					    &pd->hdr.icmp))
8670 						return (PF_DROP);
8671 					m_copyback(pd->m, pd->off,
8672 					    sizeof(struct icmp6_hdr),
8673 					    (c_caddr_t)&pd->hdr.icmp6);
8674 					if (pf_change_icmp_af(pd->m, ipoff2, pd,
8675 					    &pd2, &nk->addr[sidx],
8676 					    &nk->addr[didx], pd->af,
8677 					    nk->af))
8678 						return (PF_DROP);
8679 					pd->proto = IPPROTO_ICMPV6;
8680 					if (pf_translate_icmp_af(nk->af, iih))
8681 						return (PF_DROP);
8682 					if (virtual_type == htons(ICMP_ECHO) &&
8683 					    nk->port[iidx] != iih->icmp_id)
8684 						iih->icmp_id = nk->port[iidx];
8685 					m_copyback(pd2.m, pd2.off, ICMP_MINLEN,
8686 					    (c_caddr_t)iih);
8687 					pf_addrcpy(&pd->nsaddr,
8688 					    &nk->addr[pd2.sidx], nk->af);
8689 					pf_addrcpy(&pd->ndaddr,
8690 					    &nk->addr[pd2.didx], nk->af);
8691 					/*
8692 					 * IPv4 becomes IPv6 so we must copy
8693 					 * IPv4 src addr to least 32bits in
8694 					 * IPv6 address to keep traceroute
8695 					 * working.
8696 					 */
8697 					pd->nsaddr.addr32[3] =
8698 					    pd->src->addr32[0];
8699 					pd->naf = nk->af;
8700 					return (PF_AFRT);
8701 				}
8702 #endif /* INET && INET6 */
8703 
8704 				if (PF_ANEQ(pd2.src,
8705 				    &nk->addr[pd2.sidx], pd2.af) ||
8706 				    (virtual_type == htons(ICMP_ECHO) &&
8707 				    nk->port[iidx] != iih->icmp_id))
8708 					pf_change_icmp(pd2.src,
8709 					    (virtual_type == htons(ICMP_ECHO)) ?
8710 					    &iih->icmp_id : NULL,
8711 					    daddr, &nk->addr[pd2.sidx],
8712 					    (virtual_type == htons(ICMP_ECHO)) ?
8713 					    nk->port[iidx] : 0, NULL,
8714 					    pd2.ip_sum, icmpsum,
8715 					    pd->ip_sum, 0, AF_INET);
8716 
8717 				if (PF_ANEQ(pd2.dst,
8718 				    &nk->addr[pd2.didx], pd2.af))
8719 					pf_change_icmp(pd2.dst, NULL, NULL,
8720 					    &nk->addr[pd2.didx], 0, NULL,
8721 					    pd2.ip_sum, icmpsum, pd->ip_sum, 0,
8722 					    AF_INET);
8723 
8724 				m_copyback(pd->m, pd->off, ICMP_MINLEN, (caddr_t)&pd->hdr.icmp);
8725 				m_copyback(pd->m, ipoff2, sizeof(h2), (caddr_t)&h2);
8726 				m_copyback(pd->m, pd2.off, ICMP_MINLEN, (caddr_t)iih);
8727 			}
8728 			return (PF_PASS);
8729 			break;
8730 		}
8731 #endif /* INET */
8732 #ifdef INET6
8733 		case IPPROTO_ICMPV6: {
8734 			struct icmp6_hdr	*iih = &pd2.hdr.icmp6;
8735 
8736 			if (pd2.af != AF_INET6) {
8737 				REASON_SET(reason, PFRES_NORM);
8738 				return (PF_DROP);
8739 			}
8740 
8741 			if (!pf_pull_hdr(pd->m, pd2.off, iih,
8742 			    sizeof(struct icmp6_hdr), NULL, reason, pd2.af)) {
8743 				DPFPRINTF(PF_DEBUG_MISC,
8744 				    "pf: ICMP error message too short "
8745 				    "(icmp6)");
8746 				return (PF_DROP);
8747 			}
8748 			pd2.pcksum = &pd2.hdr.icmp6.icmp6_cksum;
8749 
8750 			pf_icmp_mapping(&pd2, iih->icmp6_type,
8751 			    &icmp_dir, &virtual_id, &virtual_type);
8752 
8753 			ret = pf_icmp_state_lookup(&key, &pd2, state,
8754 			    virtual_id, virtual_type, icmp_dir, &iidx, 0, 1);
8755 			/* IPv6? try matching a multicast address */
8756 			if (ret == PF_DROP && pd2.af == AF_INET6 &&
8757 			    icmp_dir == PF_OUT) {
8758 				MPASS(*state == NULL);
8759 				ret = pf_icmp_state_lookup(&key, &pd2,
8760 				    state, virtual_id, virtual_type,
8761 				    icmp_dir, &iidx, 1, 1);
8762 			}
8763 			if (ret >= 0) {
8764 				MPASS(*state == NULL);
8765 				return (ret);
8766 			}
8767 
8768 			/* translate source/destination address, if necessary */
8769 			if ((*state)->key[PF_SK_WIRE] !=
8770 			    (*state)->key[PF_SK_STACK]) {
8771 				struct pf_state_key	*nk;
8772 
8773 				if (PF_REVERSED_KEY(*state, pd->af))
8774 					nk = (*state)->key[pd->sidx];
8775 				else
8776 					nk = (*state)->key[pd->didx];
8777 
8778 #if defined(INET) && defined(INET6)
8779 				int	 afto, sidx, didx;
8780 
8781 				afto = pd->af != nk->af;
8782 
8783 				if (afto && (*state)->direction == PF_IN) {
8784 					sidx = pd2.didx;
8785 					didx = pd2.sidx;
8786 					iidx = !iidx;
8787 				} else {
8788 					sidx = pd2.sidx;
8789 					didx = pd2.didx;
8790 				}
8791 
8792 				if (afto) {
8793 					if (nk->af != AF_INET)
8794 						return (PF_DROP);
8795 					if (pf_translate_icmp_af(nk->af,
8796 					    &pd->hdr.icmp))
8797 						return (PF_DROP);
8798 					m_copyback(pd->m, pd->off,
8799 					    sizeof(struct icmp6_hdr),
8800 					    (c_caddr_t)&pd->hdr.icmp6);
8801 					if (pf_change_icmp_af(pd->m, ipoff2, pd,
8802 					    &pd2, &nk->addr[sidx],
8803 					    &nk->addr[didx], pd->af,
8804 					    nk->af))
8805 						return (PF_DROP);
8806 					pd->proto = IPPROTO_ICMP;
8807 					if (pf_translate_icmp_af(nk->af, iih))
8808 						return (PF_DROP);
8809 					if (virtual_type ==
8810 					    htons(ICMP6_ECHO_REQUEST) &&
8811 					    nk->port[iidx] != iih->icmp6_id)
8812 						iih->icmp6_id = nk->port[iidx];
8813 					m_copyback(pd2.m, pd2.off,
8814 					    sizeof(struct icmp6_hdr), (c_caddr_t)iih);
8815 					pf_addrcpy(&pd->nsaddr,
8816 					    &nk->addr[pd2.sidx], nk->af);
8817 					pf_addrcpy(&pd->ndaddr,
8818 					    &nk->addr[pd2.didx], nk->af);
8819 					pd->naf = nk->af;
8820 					return (PF_AFRT);
8821 				}
8822 #endif /* INET && INET6 */
8823 
8824 				if (PF_ANEQ(pd2.src,
8825 				    &nk->addr[pd2.sidx], pd2.af) ||
8826 				    ((virtual_type == htons(ICMP6_ECHO_REQUEST)) &&
8827 				    nk->port[pd2.sidx] != iih->icmp6_id))
8828 					pf_change_icmp(pd2.src,
8829 					    (virtual_type == htons(ICMP6_ECHO_REQUEST))
8830 					    ? &iih->icmp6_id : NULL,
8831 					    daddr, &nk->addr[pd2.sidx],
8832 					    (virtual_type == htons(ICMP6_ECHO_REQUEST))
8833 					    ? nk->port[iidx] : 0, NULL,
8834 					    pd2.ip_sum, icmpsum,
8835 					    pd->ip_sum, 0, AF_INET6);
8836 
8837 				if (PF_ANEQ(pd2.dst,
8838 				    &nk->addr[pd2.didx], pd2.af))
8839 					pf_change_icmp(pd2.dst, NULL, NULL,
8840 					    &nk->addr[pd2.didx], 0, NULL,
8841 					    pd2.ip_sum, icmpsum,
8842 					    pd->ip_sum, 0, AF_INET6);
8843 
8844 				m_copyback(pd->m, pd->off, sizeof(struct icmp6_hdr),
8845 				    (caddr_t)&pd->hdr.icmp6);
8846 				m_copyback(pd->m, ipoff2, sizeof(h2_6), (caddr_t)&h2_6);
8847 				m_copyback(pd->m, pd2.off, sizeof(struct icmp6_hdr),
8848 				    (caddr_t)iih);
8849 			}
8850 			return (PF_PASS);
8851 			break;
8852 		}
8853 #endif /* INET6 */
8854 		default: {
8855 			int	action;
8856 
8857 			/*
8858 			 * Placeholder value, so future calls to pf_change_ap()
8859 			 * don't try to update a NULL checksum pointer.
8860 			 */
8861 			pd->pcksum = &pd->sctp_dummy_sum;
8862 			key.af = pd2.af;
8863 			key.proto = pd2.proto;
8864 			pf_addrcpy(&key.addr[pd2.sidx], pd2.src, key.af);
8865 			pf_addrcpy(&key.addr[pd2.didx], pd2.dst, key.af);
8866 			key.port[0] = key.port[1] = 0;
8867 
8868 			action = pf_find_state(&pd2, &key, state);
8869 			if (action != PF_MATCH)
8870 				return (action);
8871 
8872 			/* translate source/destination address, if necessary */
8873 			if ((*state)->key[PF_SK_WIRE] !=
8874 			    (*state)->key[PF_SK_STACK]) {
8875 				struct pf_state_key *nk =
8876 				    (*state)->key[pd->didx];
8877 
8878 				if (PF_ANEQ(pd2.src,
8879 				    &nk->addr[pd2.sidx], pd2.af))
8880 					pf_change_icmp(pd2.src, NULL, daddr,
8881 					    &nk->addr[pd2.sidx], 0, NULL,
8882 					    pd2.ip_sum, icmpsum,
8883 					    pd->ip_sum, 0, pd2.af);
8884 
8885 				if (PF_ANEQ(pd2.dst,
8886 				    &nk->addr[pd2.didx], pd2.af))
8887 					pf_change_icmp(pd2.dst, NULL, saddr,
8888 					    &nk->addr[pd2.didx], 0, NULL,
8889 					    pd2.ip_sum, icmpsum,
8890 					    pd->ip_sum, 0, pd2.af);
8891 
8892 				switch (pd2.af) {
8893 #ifdef INET
8894 				case AF_INET:
8895 					m_copyback(pd->m, pd->off, ICMP_MINLEN,
8896 					    (caddr_t)&pd->hdr.icmp);
8897 					m_copyback(pd->m, ipoff2, sizeof(h2), (caddr_t)&h2);
8898 					break;
8899 #endif /* INET */
8900 #ifdef INET6
8901 				case AF_INET6:
8902 					m_copyback(pd->m, pd->off,
8903 					    sizeof(struct icmp6_hdr),
8904 					    (caddr_t )&pd->hdr.icmp6);
8905 					m_copyback(pd->m, ipoff2, sizeof(h2_6),
8906 					    (caddr_t )&h2_6);
8907 					break;
8908 #endif /* INET6 */
8909 				}
8910 			}
8911 			return (PF_PASS);
8912 			break;
8913 		}
8914 		}
8915 	}
8916 }
8917 
8918 /*
8919  * ipoff and off are measured from the start of the mbuf chain.
8920  * h must be at "ipoff" on the mbuf chain.
8921  */
8922 void *
8923 pf_pull_hdr(const struct mbuf *m, int off, void *p, int len,
8924     u_short *actionp, u_short *reasonp, sa_family_t af)
8925 {
8926 	int iplen = 0;
8927 	switch (af) {
8928 #ifdef INET
8929 	case AF_INET: {
8930 		const struct ip	*h = mtod(m, struct ip *);
8931 		u_int16_t	 fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3;
8932 
8933 		if (fragoff) {
8934 			if (fragoff >= len)
8935 				ACTION_SET(actionp, PF_PASS);
8936 			else {
8937 				ACTION_SET(actionp, PF_DROP);
8938 				REASON_SET(reasonp, PFRES_FRAG);
8939 			}
8940 			return (NULL);
8941 		}
8942 		iplen = ntohs(h->ip_len);
8943 		break;
8944 	}
8945 #endif /* INET */
8946 #ifdef INET6
8947 	case AF_INET6: {
8948 		const struct ip6_hdr	*h = mtod(m, struct ip6_hdr *);
8949 
8950 		iplen = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr);
8951 		break;
8952 	}
8953 #endif /* INET6 */
8954 	}
8955 	if (m->m_pkthdr.len < off + len || iplen < off + len) {
8956 		ACTION_SET(actionp, PF_DROP);
8957 		REASON_SET(reasonp, PFRES_SHORT);
8958 		return (NULL);
8959 	}
8960 	m_copydata(m, off, len, p);
8961 	return (p);
8962 }
8963 
8964 int
8965 pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kkif *kif,
8966     int rtableid)
8967 {
8968 	struct ifnet		*ifp;
8969 
8970 	/*
8971 	 * Skip check for addresses with embedded interface scope,
8972 	 * as they would always match anyway.
8973 	 */
8974 	if (af == AF_INET6 && IN6_IS_SCOPE_EMBED(&addr->v6))
8975 		return (1);
8976 
8977 	if (af != AF_INET && af != AF_INET6)
8978 		return (0);
8979 
8980 	if (kif == V_pfi_all)
8981 		return (1);
8982 
8983 	/* Skip checks for ipsec interfaces */
8984 	if (kif != NULL && kif->pfik_ifp->if_type == IFT_ENC)
8985 		return (1);
8986 
8987 	ifp = (kif != NULL) ? kif->pfik_ifp : NULL;
8988 
8989 	switch (af) {
8990 #ifdef INET6
8991 	case AF_INET6:
8992 		return (fib6_check_urpf(rtableid, &addr->v6, 0, NHR_NONE,
8993 		    ifp));
8994 #endif /* INET6 */
8995 #ifdef INET
8996 	case AF_INET:
8997 		return (fib4_check_urpf(rtableid, addr->v4, 0, NHR_NONE,
8998 		    ifp));
8999 #endif /* INET */
9000 	}
9001 
9002 	return (0);
9003 }
9004 
9005 #ifdef INET
9006 static int
9007 pf_route(struct pf_krule *r, struct ifnet *oifp,
9008     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
9009 {
9010 	struct mbuf		*m0, *m1, *md;
9011 	struct route_in6	 ro;
9012 	union sockaddr_union	 rt_gw;
9013 	const union sockaddr_union	*gw = (const union sockaddr_union *)&ro.ro_dst;
9014 	union sockaddr_union	*dst;
9015 	struct ip		*ip;
9016 	struct ifnet		*ifp = NULL;
9017 	int			 error = 0;
9018 	uint16_t		 ip_len, ip_off;
9019 	uint16_t		 tmp;
9020 	int			 r_dir;
9021 	bool			 skip_test = false;
9022 	int			 action = PF_PASS;
9023 
9024 	KASSERT(pd->m && r && oifp, ("%s: invalid parameters", __func__));
9025 
9026 	SDT_PROBE4(pf, ip, route_to, entry, pd->m, pd, s, oifp);
9027 
9028 	if (s) {
9029 		r_dir = s->direction;
9030 	} else {
9031 		r_dir = r->direction;
9032 	}
9033 
9034 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
9035 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
9036 	    __func__));
9037 
9038 	if ((pd->pf_mtag == NULL &&
9039 	    ((pd->pf_mtag = pf_get_mtag(pd->m)) == NULL)) ||
9040 	    pd->pf_mtag->routed++ > 3) {
9041 		m0 = pd->m;
9042 		pd->m = NULL;
9043 		SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9044 		action = PF_DROP;
9045 		goto bad_locked;
9046 	}
9047 
9048 	if (pd->act.rt_kif != NULL)
9049 		ifp = pd->act.rt_kif->pfik_ifp;
9050 
9051 	if (pd->act.rt == PF_DUPTO) {
9052 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
9053 			if (s != NULL) {
9054 				PF_STATE_UNLOCK(s);
9055 			}
9056 			if (ifp == oifp) {
9057 				/* When the 2nd interface is not skipped */
9058 				return (action);
9059 			} else {
9060 				m0 = pd->m;
9061 				pd->m = NULL;
9062 				SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9063 				action = PF_DROP;
9064 				goto bad;
9065 			}
9066 		} else {
9067 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
9068 			if (((m0 = m_dup(pd->m, M_NOWAIT)) == NULL)) {
9069 				if (s)
9070 					PF_STATE_UNLOCK(s);
9071 				return (action);
9072 			}
9073 		}
9074 	} else {
9075 		if ((pd->act.rt == PF_REPLYTO) == (r_dir == pd->dir)) {
9076 			if (pd->af == pd->naf) {
9077 				pf_dummynet(pd, s, r, &pd->m);
9078 				if (s)
9079 					PF_STATE_UNLOCK(s);
9080 				return (action);
9081 			} else {
9082 				if (r_dir == PF_IN) {
9083 					skip_test = true;
9084 				}
9085 			}
9086 		}
9087 
9088 		/*
9089 		 * If we're actually doing route-to and af-to and are in the
9090 		 * reply direction.
9091 		 */
9092 		if (pd->act.rt_kif && pd->act.rt_kif->pfik_ifp &&
9093 		    pd->af != pd->naf) {
9094 			if (pd->act.rt == PF_ROUTETO && r->naf != AF_INET) {
9095 				/* Un-set ifp so we do a plain route lookup. */
9096 				ifp = NULL;
9097 			}
9098 			if (pd->act.rt == PF_REPLYTO && r->naf != AF_INET6) {
9099 				/* Un-set ifp so we do a plain route lookup. */
9100 				ifp = NULL;
9101 			}
9102 		}
9103 		m0 = pd->m;
9104 	}
9105 
9106 	ip = mtod(m0, struct ip *);
9107 
9108 	bzero(&ro, sizeof(ro));
9109 	dst = (union sockaddr_union *)&ro.ro_dst;
9110 	dst->sin.sin_family = AF_INET;
9111 	dst->sin.sin_len = sizeof(struct sockaddr_in);
9112 	dst->sin.sin_addr = ip->ip_dst;
9113 	if (ifp) { /* Only needed in forward direction and route-to */
9114 		bzero(&rt_gw, sizeof(rt_gw));
9115 		ro.ro_flags |= RT_HAS_GW;
9116 		gw = &rt_gw;
9117 		switch (pd->act.rt_af) {
9118 #ifdef INET
9119 		case AF_INET:
9120 			rt_gw.sin.sin_family = AF_INET;
9121 			rt_gw.sin.sin_len = sizeof(struct sockaddr_in);
9122 			rt_gw.sin.sin_addr.s_addr = pd->act.rt_addr.v4.s_addr;
9123 			break;
9124 #endif /* INET */
9125 #ifdef INET6
9126 		case AF_INET6:
9127 			rt_gw.sin6.sin6_family = AF_INET6;
9128 			rt_gw.sin6.sin6_len = sizeof(struct sockaddr_in6);
9129 			pf_addrcpy((struct pf_addr *)&rt_gw.sin6.sin6_addr,
9130 			    &pd->act.rt_addr, AF_INET6);
9131 			break;
9132 #endif /* INET6 */
9133 		default:
9134 			/* Normal af-to without route-to */
9135 			break;
9136 		}
9137 	}
9138 
9139 	if (pd->dir == PF_IN) {
9140 		if (ip->ip_ttl <= IPTTLDEC) {
9141 			if (r->rt != PF_DUPTO)
9142 				pf_send_icmp(m0, ICMP_TIMXCEED,
9143 				    ICMP_TIMXCEED_INTRANS, 0, pd->af, r,
9144 				    pd->act.rtableid);
9145 			action = PF_DROP;
9146 			goto bad_locked;
9147 		}
9148 		ip->ip_ttl -= IPTTLDEC;
9149 	}
9150 
9151 	if (s != NULL) {
9152 		if (ifp == NULL && (pd->af != pd->naf)) {
9153 			/* We're in the AFTO case. Do a route lookup. */
9154 			const struct nhop_object *nh;
9155 			nh = fib4_lookup(M_GETFIB(m0), ip->ip_dst, 0, NHR_NONE, 0);
9156 			if (nh) {
9157 				ifp = nh->nh_ifp;
9158 
9159 				/* Use the gateway if needed. */
9160 				if (nh->nh_flags & NHF_GATEWAY) {
9161 					gw = (const union sockaddr_union *)&nh->gw_sa;
9162 					ro.ro_flags |= RT_HAS_GW;
9163 				} else {
9164 					dst->sin.sin_addr = ip->ip_dst;
9165 				}
9166 
9167 				/*
9168 				 * Bind to the correct interface if we're
9169 				 * if-bound. We don't know which interface
9170 				 * that will be until here, so we've inserted
9171 				 * the state on V_pf_all. Fix that now.
9172 				 */
9173 				if (s->kif == V_pfi_all && ifp != NULL &&
9174 				    r->rule_flag & PFRULE_IFBOUND)
9175 					s->kif = ifp->if_pf_kif;
9176 			}
9177 		}
9178 
9179 		if (r->rule_flag & PFRULE_IFBOUND &&
9180 		    pd->act.rt == PF_REPLYTO &&
9181 		    s->kif == V_pfi_all) {
9182 			s->kif = pd->act.rt_kif;
9183 			s->orig_kif = oifp->if_pf_kif;
9184 		}
9185 
9186 		PF_STATE_UNLOCK(s);
9187 	}
9188 
9189 	/* It must have been either set from rt_af or from fib4_lookup */
9190 	KASSERT(gw->sin.sin_family != 0, ("%s: gw address family undetermined", __func__));
9191 
9192 	if (ifp == NULL) {
9193 		m0 = pd->m;
9194 		pd->m = NULL;
9195 		action = PF_DROP;
9196 		SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9197 		goto bad;
9198 	}
9199 
9200 	if (r->rt == PF_DUPTO)
9201 		skip_test = true;
9202 
9203 	if (pd->dir == PF_IN && !skip_test) {
9204 		if (pf_test(AF_INET, PF_OUT, PFIL_FWD, ifp, &m0, inp,
9205 		    &pd->act) != PF_PASS) {
9206 			action = PF_DROP;
9207 			SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9208 			goto bad;
9209 		} else if (m0 == NULL) {
9210 			action = PF_DROP;
9211 			SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9212 			goto done;
9213 		}
9214 		if (m0->m_len < sizeof(struct ip)) {
9215 			DPFPRINTF(PF_DEBUG_URGENT,
9216 			    "%s: m0->m_len < sizeof(struct ip)", __func__);
9217 			SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9218 			action = PF_DROP;
9219 			goto bad;
9220 		}
9221 		ip = mtod(m0, struct ip *);
9222 	}
9223 
9224 	if (ifp->if_flags & IFF_LOOPBACK)
9225 		m0->m_flags |= M_SKIP_FIREWALL;
9226 
9227 	ip_len = ntohs(ip->ip_len);
9228 	ip_off = ntohs(ip->ip_off);
9229 
9230 	/* Copied from FreeBSD 10.0-CURRENT ip_output. */
9231 	m0->m_pkthdr.csum_flags |= CSUM_IP;
9232 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~ifp->if_hwassist) {
9233 		in_delayed_cksum(m0);
9234 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
9235 	}
9236 	if (m0->m_pkthdr.csum_flags & CSUM_SCTP & ~ifp->if_hwassist) {
9237 		pf_sctp_checksum(m0, (uint32_t)(ip->ip_hl << 2));
9238 		m0->m_pkthdr.csum_flags &= ~CSUM_SCTP;
9239 	}
9240 
9241 	if (pd->dir == PF_IN) {
9242 		/*
9243 		 * Make sure dummynet gets the correct direction, in case it needs to
9244 		 * re-inject later.
9245 		 */
9246 		pd->dir = PF_OUT;
9247 
9248 		/*
9249 		 * The following processing is actually the rest of the inbound processing, even
9250 		 * though we've marked it as outbound (so we don't look through dummynet) and it
9251 		 * happens after the outbound processing (pf_test(PF_OUT) above).
9252 		 * Swap the dummynet pipe numbers, because it's going to come to the wrong
9253 		 * conclusion about what direction it's processing, and we can't fix it or it
9254 		 * will re-inject incorrectly. Swapping the pipe numbers means that its incorrect
9255 		 * decision will pick the right pipe, and everything will mostly work as expected.
9256 		 */
9257 		tmp = pd->act.dnrpipe;
9258 		pd->act.dnrpipe = pd->act.dnpipe;
9259 		pd->act.dnpipe = tmp;
9260 	}
9261 
9262 	/*
9263 	 * If small enough for interface, or the interface will take
9264 	 * care of the fragmentation for us, we can just send directly.
9265 	 */
9266 	if (ip_len <= ifp->if_mtu ||
9267 	    (m0->m_pkthdr.csum_flags & ifp->if_hwassist & CSUM_TSO) != 0) {
9268 		ip->ip_sum = 0;
9269 		if (m0->m_pkthdr.csum_flags & CSUM_IP & ~ifp->if_hwassist) {
9270 			ip->ip_sum = in_cksum(m0, ip->ip_hl << 2);
9271 			m0->m_pkthdr.csum_flags &= ~CSUM_IP;
9272 		}
9273 		m_clrprotoflags(m0);	/* Avoid confusing lower layers. */
9274 
9275 		md = m0;
9276 		error = pf_dummynet_route(pd, s, r, ifp,
9277 		    (const struct sockaddr *)gw, &md);
9278 		if (md != NULL) {
9279 			error = (*ifp->if_output)(ifp, md,
9280 			    (const struct sockaddr *)gw, (struct route *)&ro);
9281 			SDT_PROBE2(pf, ip, route_to, output, ifp, error);
9282 		}
9283 		goto done;
9284 	}
9285 
9286 	/* Balk when DF bit is set or the interface didn't support TSO. */
9287 	if ((ip_off & IP_DF) || (m0->m_pkthdr.csum_flags & CSUM_TSO)) {
9288 		error = EMSGSIZE;
9289 		KMOD_IPSTAT_INC(ips_cantfrag);
9290 		if (pd->act.rt != PF_DUPTO) {
9291 			if (s && s->nat_rule != NULL) {
9292 				MPASS(m0 == pd->m);
9293 				PACKET_UNDO_NAT(pd,
9294 				    (ip->ip_hl << 2) + (ip_off & IP_OFFMASK),
9295 				    s);
9296 			}
9297 
9298 			pf_send_icmp(m0, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG,
9299 			   ifp->if_mtu, pd->af, r, pd->act.rtableid);
9300 		}
9301 		SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9302 		action = PF_DROP;
9303 		goto bad;
9304 	}
9305 
9306 	error = ip_fragment(ip, &m0, ifp->if_mtu, ifp->if_hwassist);
9307 	if (error) {
9308 		SDT_PROBE1(pf, ip, route_to, drop, __LINE__);
9309 		action = PF_DROP;
9310 		goto bad;
9311 	}
9312 
9313 	for (; m0; m0 = m1) {
9314 		m1 = m0->m_nextpkt;
9315 		m0->m_nextpkt = NULL;
9316 		if (error == 0) {
9317 			m_clrprotoflags(m0);
9318 			md = m0;
9319 			pd->pf_mtag = pf_find_mtag(md);
9320 			error = pf_dummynet_route(pd, s, r, ifp,
9321 			    (const struct sockaddr *)gw, &md);
9322 			if (md != NULL) {
9323 				error = (*ifp->if_output)(ifp, md,
9324 				    (const struct sockaddr *)gw,
9325 				    (struct route *)&ro);
9326 				SDT_PROBE2(pf, ip, route_to, output, ifp, error);
9327 			}
9328 		} else
9329 			m_freem(m0);
9330 	}
9331 
9332 	if (error == 0)
9333 		KMOD_IPSTAT_INC(ips_fragmented);
9334 
9335 done:
9336 	if (pd->act.rt != PF_DUPTO)
9337 		pd->m = NULL;
9338 	else
9339 		action = PF_PASS;
9340 	return (action);
9341 
9342 bad_locked:
9343 	if (s)
9344 		PF_STATE_UNLOCK(s);
9345 bad:
9346 	m_freem(m0);
9347 	goto done;
9348 }
9349 #endif /* INET */
9350 
9351 #ifdef INET6
9352 static int
9353 pf_route6(struct pf_krule *r, struct ifnet *oifp,
9354     struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp)
9355 {
9356 	struct mbuf		*m0, *md;
9357 	struct m_tag		*mtag;
9358 	struct sockaddr_in6	dst;
9359 	struct ip6_hdr		*ip6;
9360 	struct ifnet		*ifp = NULL;
9361 	int			 r_dir;
9362 	bool			 skip_test = false;
9363 	int			 action = PF_PASS;
9364 
9365 	KASSERT(pd->m && r && oifp, ("%s: invalid parameters", __func__));
9366 
9367 	SDT_PROBE4(pf, ip6, route_to, entry, pd->m, pd, s, oifp);
9368 
9369 	if (s) {
9370 		r_dir = s->direction;
9371 	} else {
9372 		r_dir = r->direction;
9373 	}
9374 
9375 	KASSERT(pd->dir == PF_IN || pd->dir == PF_OUT ||
9376 	    r_dir == PF_IN || r_dir == PF_OUT, ("%s: invalid direction",
9377 	    __func__));
9378 
9379 	if ((pd->pf_mtag == NULL &&
9380 	    ((pd->pf_mtag = pf_get_mtag(pd->m)) == NULL)) ||
9381 	    pd->pf_mtag->routed++ > 3) {
9382 		m0 = pd->m;
9383 		pd->m = NULL;
9384 		action = PF_DROP;
9385 		SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9386 		goto bad_locked;
9387 	}
9388 
9389 	if (pd->act.rt_kif != NULL)
9390 		ifp = pd->act.rt_kif->pfik_ifp;
9391 
9392 	if (pd->act.rt == PF_DUPTO) {
9393 		if ((pd->pf_mtag->flags & PF_MTAG_FLAG_DUPLICATED)) {
9394 			if (s != NULL) {
9395 				PF_STATE_UNLOCK(s);
9396 			}
9397 			if (ifp == oifp) {
9398 				/* When the 2nd interface is not skipped */
9399 				return (action);
9400 			} else {
9401 				m0 = pd->m;
9402 				pd->m = NULL;
9403 				action = PF_DROP;
9404 				SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9405 				goto bad;
9406 			}
9407 		} else {
9408 			pd->pf_mtag->flags |= PF_MTAG_FLAG_DUPLICATED;
9409 			if (((m0 = m_dup(pd->m, M_NOWAIT)) == NULL)) {
9410 				if (s)
9411 					PF_STATE_UNLOCK(s);
9412 				return (action);
9413 			}
9414 		}
9415 	} else {
9416 		if ((pd->act.rt == PF_REPLYTO) == (r_dir == pd->dir)) {
9417 			if (pd->af == pd->naf) {
9418 				pf_dummynet(pd, s, r, &pd->m);
9419 				if (s)
9420 					PF_STATE_UNLOCK(s);
9421 				return (action);
9422 			} else {
9423 				if (r_dir == PF_IN) {
9424 					skip_test = true;
9425 				}
9426 			}
9427 		}
9428 
9429 		/*
9430 		 * If we're actually doing route-to and af-to and are in the
9431 		 * reply direction.
9432 		 */
9433 		if (pd->act.rt_kif && pd->act.rt_kif->pfik_ifp &&
9434 		    pd->af != pd->naf) {
9435 			if (pd->act.rt == PF_ROUTETO && r->naf != AF_INET6) {
9436 				/* Un-set ifp so we do a plain route lookup. */
9437 				ifp = NULL;
9438 			}
9439 			if (pd->act.rt == PF_REPLYTO && r->naf != AF_INET) {
9440 				/* Un-set ifp so we do a plain route lookup. */
9441 				ifp = NULL;
9442 			}
9443 		}
9444 		m0 = pd->m;
9445 	}
9446 
9447 	ip6 = mtod(m0, struct ip6_hdr *);
9448 
9449 	bzero(&dst, sizeof(dst));
9450 	dst.sin6_family = AF_INET6;
9451 	dst.sin6_len = sizeof(dst);
9452 	pf_addrcpy((struct pf_addr *)&dst.sin6_addr, &pd->act.rt_addr,
9453 	    AF_INET6);
9454 
9455 	if (pd->dir == PF_IN) {
9456 		if (ip6->ip6_hlim <= IPV6_HLIMDEC) {
9457 			if (r->rt != PF_DUPTO)
9458 				pf_send_icmp(m0, ICMP6_TIME_EXCEEDED,
9459 				    ICMP6_TIME_EXCEED_TRANSIT, 0, pd->af, r,
9460 				    pd->act.rtableid);
9461 			action = PF_DROP;
9462 			goto bad_locked;
9463 		}
9464 		ip6->ip6_hlim -= IPV6_HLIMDEC;
9465 	}
9466 
9467 	if (s != NULL) {
9468 		if (ifp == NULL && (pd->af != pd->naf)) {
9469 			const struct nhop_object *nh;
9470 			nh = fib6_lookup(M_GETFIB(m0), &ip6->ip6_dst, 0, NHR_NONE, 0);
9471 			if (nh) {
9472 				ifp = nh->nh_ifp;
9473 
9474 				/* Use the gateway if needed. */
9475 				if (nh->nh_flags & NHF_GATEWAY)
9476 					bcopy(&nh->gw6_sa.sin6_addr, &dst.sin6_addr,
9477 					    sizeof(dst.sin6_addr));
9478 				else
9479 					dst.sin6_addr = ip6->ip6_dst;
9480 
9481 				/*
9482 				 * Bind to the correct interface if we're
9483 				 * if-bound. We don't know which interface
9484 				 * that will be until here, so we've inserted
9485 				 * the state on V_pf_all. Fix that now.
9486 				 */
9487 				if (s->kif == V_pfi_all && ifp != NULL &&
9488 				    r->rule_flag & PFRULE_IFBOUND)
9489 					s->kif = ifp->if_pf_kif;
9490 			}
9491 		}
9492 
9493 		if (r->rule_flag & PFRULE_IFBOUND &&
9494 		    pd->act.rt == PF_REPLYTO &&
9495 		    s->kif == V_pfi_all) {
9496 			s->kif = pd->act.rt_kif;
9497 			s->orig_kif = oifp->if_pf_kif;
9498 		}
9499 
9500 		PF_STATE_UNLOCK(s);
9501 	}
9502 
9503 	if (pd->af != pd->naf) {
9504 		struct udphdr *uh = &pd->hdr.udp;
9505 
9506 		if (pd->proto == IPPROTO_UDP && uh->uh_sum == 0) {
9507 			uh->uh_sum = in6_cksum_pseudo(ip6,
9508 			    ntohs(uh->uh_ulen), IPPROTO_UDP, 0);
9509 			m_copyback(m0, pd->off, sizeof(*uh), pd->hdr.any);
9510 		}
9511 	}
9512 
9513 	if (ifp == NULL) {
9514 		m0 = pd->m;
9515 		pd->m = NULL;
9516 		action = PF_DROP;
9517 		SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9518 		goto bad;
9519 	}
9520 
9521 	if (r->rt == PF_DUPTO)
9522 		skip_test = true;
9523 
9524 	if (pd->dir == PF_IN && !skip_test) {
9525 		if (pf_test(AF_INET6, PF_OUT, PFIL_FWD | PF_PFIL_NOREFRAGMENT,
9526 		    ifp, &m0, inp, &pd->act) != PF_PASS) {
9527 			action = PF_DROP;
9528 			SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9529 			goto bad;
9530 		} else if (m0 == NULL) {
9531 			action = PF_DROP;
9532 			SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9533 			goto done;
9534 		}
9535 		if (m0->m_len < sizeof(struct ip6_hdr)) {
9536 			DPFPRINTF(PF_DEBUG_URGENT,
9537 			    "%s: m0->m_len < sizeof(struct ip6_hdr)",
9538 			    __func__);
9539 			action = PF_DROP;
9540 			SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9541 			goto bad;
9542 		}
9543 		ip6 = mtod(m0, struct ip6_hdr *);
9544 	}
9545 
9546 	if (ifp->if_flags & IFF_LOOPBACK)
9547 		m0->m_flags |= M_SKIP_FIREWALL;
9548 
9549 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6 &
9550 	    ~ifp->if_hwassist) {
9551 		uint32_t plen = m0->m_pkthdr.len - sizeof(*ip6);
9552 		in6_delayed_cksum(m0, plen, sizeof(struct ip6_hdr));
9553 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
9554 	}
9555 
9556 	if (pd->dir == PF_IN) {
9557 		uint16_t	 tmp;
9558 		/*
9559 		 * Make sure dummynet gets the correct direction, in case it needs to
9560 		 * re-inject later.
9561 		 */
9562 		pd->dir = PF_OUT;
9563 
9564 		/*
9565 		 * The following processing is actually the rest of the inbound processing, even
9566 		 * though we've marked it as outbound (so we don't look through dummynet) and it
9567 		 * happens after the outbound processing (pf_test(PF_OUT) above).
9568 		 * Swap the dummynet pipe numbers, because it's going to come to the wrong
9569 		 * conclusion about what direction it's processing, and we can't fix it or it
9570 		 * will re-inject incorrectly. Swapping the pipe numbers means that its incorrect
9571 		 * decision will pick the right pipe, and everything will mostly work as expected.
9572 		 */
9573 		tmp = pd->act.dnrpipe;
9574 		pd->act.dnrpipe = pd->act.dnpipe;
9575 		pd->act.dnpipe = tmp;
9576 	}
9577 
9578 	/*
9579 	 * If the packet is too large for the outgoing interface,
9580 	 * send back an icmp6 error.
9581 	 */
9582 	if (IN6_IS_SCOPE_EMBED(&dst.sin6_addr))
9583 		dst.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
9584 	mtag = m_tag_find(m0, PACKET_TAG_PF_REASSEMBLED, NULL);
9585 	if (mtag != NULL) {
9586 		int ret __sdt_used;
9587 		ret = pf_refragment6(ifp, &m0, mtag, ifp, true);
9588 		SDT_PROBE2(pf, ip6, route_to, output, ifp, ret);
9589 		goto done;
9590 	}
9591 
9592 	if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu) {
9593 		md = m0;
9594 		pf_dummynet_route(pd, s, r, ifp, sintosa(&dst), &md);
9595 		if (md != NULL) {
9596 			int ret __sdt_used;
9597 			ret = nd6_output_ifp(ifp, ifp, md, &dst, NULL);
9598 			SDT_PROBE2(pf, ip6, route_to, output, ifp, ret);
9599 		}
9600 	}
9601 	else {
9602 		in6_ifstat_inc(ifp, ifs6_in_toobig);
9603 		if (pd->act.rt != PF_DUPTO) {
9604 			if (s && s->nat_rule != NULL) {
9605 				MPASS(m0 == pd->m);
9606 				PACKET_UNDO_NAT(pd,
9607 				    ((caddr_t)ip6 - m0->m_data) +
9608 				    sizeof(struct ip6_hdr), s);
9609 			}
9610 
9611 			if (r->rt != PF_DUPTO)
9612 				pf_send_icmp(m0, ICMP6_PACKET_TOO_BIG, 0,
9613 				    ifp->if_mtu, pd->af, r, pd->act.rtableid);
9614 		}
9615 		action = PF_DROP;
9616 		SDT_PROBE1(pf, ip6, route_to, drop, __LINE__);
9617 		goto bad;
9618 	}
9619 
9620 done:
9621 	if (pd->act.rt != PF_DUPTO)
9622 		pd->m = NULL;
9623 	else
9624 		action = PF_PASS;
9625 	return (action);
9626 
9627 bad_locked:
9628 	if (s)
9629 		PF_STATE_UNLOCK(s);
9630 bad:
9631 	m_freem(m0);
9632 	goto done;
9633 }
9634 #endif /* INET6 */
9635 
9636 /*
9637  * FreeBSD supports cksum offloads for the following drivers.
9638  *  em(4), fxp(4), lge(4), nge(4), re(4), ti(4), txp(4), xl(4)
9639  *
9640  * CSUM_DATA_VALID | CSUM_PSEUDO_HDR :
9641  *  network driver performed cksum including pseudo header, need to verify
9642  *   csum_data
9643  * CSUM_DATA_VALID :
9644  *  network driver performed cksum, needs to additional pseudo header
9645  *  cksum computation with partial csum_data(i.e. lack of H/W support for
9646  *  pseudo header, for instance sk(4) and possibly gem(4))
9647  *
9648  * After validating the cksum of packet, set both flag CSUM_DATA_VALID and
9649  * CSUM_PSEUDO_HDR in order to avoid recomputation of the cksum in upper
9650  * TCP/UDP layer.
9651  * Also, set csum_data to 0xffff to force cksum validation.
9652  */
9653 static int
9654 pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t af)
9655 {
9656 	u_int16_t sum = 0;
9657 	int hw_assist = 0;
9658 	struct ip *ip;
9659 
9660 	if (off < sizeof(struct ip) || len < sizeof(struct udphdr))
9661 		return (1);
9662 	if (m->m_pkthdr.len < off + len)
9663 		return (1);
9664 
9665 	switch (p) {
9666 	case IPPROTO_TCP:
9667 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
9668 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
9669 				sum = m->m_pkthdr.csum_data;
9670 			} else {
9671 				ip = mtod(m, struct ip *);
9672 				sum = in_pseudo(ip->ip_src.s_addr,
9673 				ip->ip_dst.s_addr, htonl((u_short)len +
9674 				m->m_pkthdr.csum_data + IPPROTO_TCP));
9675 			}
9676 			sum ^= 0xffff;
9677 			++hw_assist;
9678 		}
9679 		break;
9680 	case IPPROTO_UDP:
9681 		if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
9682 			if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) {
9683 				sum = m->m_pkthdr.csum_data;
9684 			} else {
9685 				ip = mtod(m, struct ip *);
9686 				sum = in_pseudo(ip->ip_src.s_addr,
9687 				ip->ip_dst.s_addr, htonl((u_short)len +
9688 				m->m_pkthdr.csum_data + IPPROTO_UDP));
9689 			}
9690 			sum ^= 0xffff;
9691 			++hw_assist;
9692 		}
9693 		break;
9694 	case IPPROTO_ICMP:
9695 #ifdef INET6
9696 	case IPPROTO_ICMPV6:
9697 #endif /* INET6 */
9698 		break;
9699 	default:
9700 		return (1);
9701 	}
9702 
9703 	if (!hw_assist) {
9704 		switch (af) {
9705 		case AF_INET:
9706 			if (m->m_len < sizeof(struct ip))
9707 				return (1);
9708 			sum = in4_cksum(m, (p == IPPROTO_ICMP ? 0 : p), off, len);
9709 			break;
9710 #ifdef INET6
9711 		case AF_INET6:
9712 			if (m->m_len < sizeof(struct ip6_hdr))
9713 				return (1);
9714 			sum = in6_cksum(m, p, off, len);
9715 			break;
9716 #endif /* INET6 */
9717 		}
9718 	}
9719 	if (sum) {
9720 		switch (p) {
9721 		case IPPROTO_TCP:
9722 		    {
9723 			KMOD_TCPSTAT_INC(tcps_rcvbadsum);
9724 			break;
9725 		    }
9726 		case IPPROTO_UDP:
9727 		    {
9728 			KMOD_UDPSTAT_INC(udps_badsum);
9729 			break;
9730 		    }
9731 #ifdef INET
9732 		case IPPROTO_ICMP:
9733 		    {
9734 			KMOD_ICMPSTAT_INC(icps_checksum);
9735 			break;
9736 		    }
9737 #endif
9738 #ifdef INET6
9739 		case IPPROTO_ICMPV6:
9740 		    {
9741 			KMOD_ICMP6STAT_INC(icp6s_checksum);
9742 			break;
9743 		    }
9744 #endif /* INET6 */
9745 		}
9746 		return (1);
9747 	} else {
9748 		if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
9749 			m->m_pkthdr.csum_flags |=
9750 			    (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
9751 			m->m_pkthdr.csum_data = 0xffff;
9752 		}
9753 	}
9754 	return (0);
9755 }
9756 
9757 static bool
9758 pf_pdesc_to_dnflow(const struct pf_pdesc *pd, const struct pf_krule *r,
9759     const struct pf_kstate *s, struct ip_fw_args *dnflow)
9760 {
9761 	int dndir = r->direction;
9762 
9763 	if (s && dndir == PF_INOUT) {
9764 		dndir = s->direction;
9765 	} else if (dndir == PF_INOUT) {
9766 		/* Assume primary direction. Happens when we've set dnpipe in
9767 		 * the ethernet level code. */
9768 		dndir = pd->dir;
9769 	}
9770 
9771 	if (pd->pf_mtag->flags & PF_MTAG_FLAG_DUMMYNETED)
9772 		return (false);
9773 
9774 	memset(dnflow, 0, sizeof(*dnflow));
9775 
9776 	if (pd->dport != NULL)
9777 		dnflow->f_id.dst_port = ntohs(*pd->dport);
9778 	if (pd->sport != NULL)
9779 		dnflow->f_id.src_port = ntohs(*pd->sport);
9780 
9781 	if (pd->dir == PF_IN)
9782 		dnflow->flags |= IPFW_ARGS_IN;
9783 	else
9784 		dnflow->flags |= IPFW_ARGS_OUT;
9785 
9786 	if (pd->dir != dndir && pd->act.dnrpipe) {
9787 		dnflow->rule.info = pd->act.dnrpipe;
9788 	}
9789 	else if (pd->dir == dndir && pd->act.dnpipe) {
9790 		dnflow->rule.info = pd->act.dnpipe;
9791 	}
9792 	else {
9793 		return (false);
9794 	}
9795 
9796 	dnflow->rule.info |= IPFW_IS_DUMMYNET;
9797 	if (r->free_flags & PFRULE_DN_IS_PIPE || pd->act.flags & PFSTATE_DN_IS_PIPE)
9798 		dnflow->rule.info |= IPFW_IS_PIPE;
9799 
9800 	dnflow->f_id.proto = pd->proto;
9801 	dnflow->f_id.extra = dnflow->rule.info;
9802 	switch (pd->naf) {
9803 	case AF_INET:
9804 		dnflow->f_id.addr_type = 4;
9805 		dnflow->f_id.src_ip = ntohl(pd->src->v4.s_addr);
9806 		dnflow->f_id.dst_ip = ntohl(pd->dst->v4.s_addr);
9807 		break;
9808 	case AF_INET6:
9809 		dnflow->flags |= IPFW_ARGS_IP6;
9810 		dnflow->f_id.addr_type = 6;
9811 		dnflow->f_id.src_ip6 = pd->src->v6;
9812 		dnflow->f_id.dst_ip6 = pd->dst->v6;
9813 		break;
9814 	}
9815 
9816 	return (true);
9817 }
9818 
9819 int
9820 pf_test_eth(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
9821     struct inpcb *inp)
9822 {
9823 	struct pfi_kkif		*kif;
9824 	struct mbuf		*m = *m0;
9825 
9826 	M_ASSERTPKTHDR(m);
9827 	MPASS(ifp->if_vnet == curvnet);
9828 	NET_EPOCH_ASSERT();
9829 
9830 	if (!V_pf_status.running)
9831 		return (PF_PASS);
9832 
9833 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
9834 
9835 	if (kif == NULL) {
9836 		DPFPRINTF(PF_DEBUG_URGENT,
9837 		    "%s: kif == NULL, if_xname %s", __func__, ifp->if_xname);
9838 		return (PF_DROP);
9839 	}
9840 	if (kif->pfik_flags & PFI_IFLAG_SKIP)
9841 		return (PF_PASS);
9842 
9843 	if (m->m_flags & M_SKIP_FIREWALL)
9844 		return (PF_PASS);
9845 
9846 	if (__predict_false(! M_WRITABLE(*m0))) {
9847 		m = *m0 = m_unshare(*m0, M_NOWAIT);
9848 		if (*m0 == NULL)
9849 			return (PF_DROP);
9850 	}
9851 
9852 	/* Stateless! */
9853 	return (pf_test_eth_rule(dir, kif, m0));
9854 }
9855 
9856 static __inline void
9857 pf_dummynet_flag_remove(struct mbuf *m, struct pf_mtag *pf_mtag)
9858 {
9859 	struct m_tag *mtag;
9860 
9861 	pf_mtag->flags &= ~PF_MTAG_FLAG_DUMMYNET;
9862 
9863 	/* dummynet adds this tag, but pf does not need it,
9864 	 * and keeping it creates unexpected behavior,
9865 	 * e.g. in case of divert(4) usage right after dummynet. */
9866 	mtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL);
9867 	if (mtag != NULL)
9868 		m_tag_delete(m, mtag);
9869 }
9870 
9871 static int
9872 pf_dummynet(struct pf_pdesc *pd, struct pf_kstate *s,
9873     struct pf_krule *r, struct mbuf **m0)
9874 {
9875 	return (pf_dummynet_route(pd, s, r, NULL, NULL, m0));
9876 }
9877 
9878 static int
9879 pf_dummynet_route(struct pf_pdesc *pd, struct pf_kstate *s,
9880     struct pf_krule *r, struct ifnet *ifp, const struct sockaddr *sa,
9881     struct mbuf **m0)
9882 {
9883 	struct ip_fw_args dnflow;
9884 
9885 	NET_EPOCH_ASSERT();
9886 
9887 	if (pd->act.dnpipe == 0 && pd->act.dnrpipe == 0)
9888 		return (0);
9889 
9890 	if (ip_dn_io_ptr == NULL) {
9891 		m_freem(*m0);
9892 		*m0 = NULL;
9893 		return (ENOMEM);
9894 	}
9895 
9896 	if (pd->pf_mtag == NULL &&
9897 	    ((pd->pf_mtag = pf_get_mtag(*m0)) == NULL)) {
9898 		m_freem(*m0);
9899 		*m0 = NULL;
9900 		return (ENOMEM);
9901 	}
9902 
9903 	if (ifp != NULL) {
9904 		pd->pf_mtag->flags |= PF_MTAG_FLAG_ROUTE_TO;
9905 
9906 		pd->pf_mtag->if_index = ifp->if_index;
9907 		pd->pf_mtag->if_idxgen = ifp->if_idxgen;
9908 
9909 		MPASS(sa != NULL);
9910 
9911 		switch (sa->sa_family) {
9912 		case AF_INET:
9913 			memcpy(&pd->pf_mtag->dst, sa,
9914 			    sizeof(struct sockaddr_in));
9915 			break;
9916 		case AF_INET6:
9917 			memcpy(&pd->pf_mtag->dst, sa,
9918 			    sizeof(struct sockaddr_in6));
9919 			break;
9920 		}
9921 	}
9922 
9923 	if (s != NULL && s->nat_rule != NULL &&
9924 	    s->nat_rule->action == PF_RDR &&
9925 	    (
9926 #ifdef INET
9927 	    (pd->af == AF_INET && IN_LOOPBACK(ntohl(pd->dst->v4.s_addr))) ||
9928 #endif /* INET */
9929 	    (pd->af == AF_INET6 && IN6_IS_ADDR_LOOPBACK(&pd->dst->v6)))) {
9930 		/*
9931 		 * If we're redirecting to loopback mark this packet
9932 		 * as being local. Otherwise it might get dropped
9933 		 * if dummynet re-injects.
9934 		 */
9935 		(*m0)->m_pkthdr.rcvif = V_loif;
9936 	}
9937 
9938 	if (pf_pdesc_to_dnflow(pd, r, s, &dnflow)) {
9939 		pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNET;
9940 		pd->pf_mtag->flags |= PF_MTAG_FLAG_DUMMYNETED;
9941 		ip_dn_io_ptr(m0, &dnflow);
9942 		if (*m0 != NULL) {
9943 			pd->pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
9944 			pf_dummynet_flag_remove(*m0, pd->pf_mtag);
9945 		}
9946 	}
9947 
9948 	return (0);
9949 }
9950 
9951 static int
9952 pf_walk_option(struct pf_pdesc *pd, struct ip *h, int off, int end,
9953     u_short *reason)
9954 {
9955 	uint8_t type, length, opts[15 * 4 - sizeof(struct ip)];
9956 
9957 	/* IP header in payload of ICMP packet may be too short */
9958 	if (pd->m->m_pkthdr.len < end) {
9959 		DPFPRINTF(PF_DEBUG_MISC, "IP option too short");
9960 		REASON_SET(reason, PFRES_SHORT);
9961 		return (PF_DROP);
9962 	}
9963 
9964 	MPASS(end - off <= sizeof(opts));
9965 	m_copydata(pd->m, off, end - off, opts);
9966 	end -= off;
9967 	off = 0;
9968 
9969 	while (off < end) {
9970 		type = opts[off];
9971 		if (type == IPOPT_EOL)
9972 			break;
9973 		if (type == IPOPT_NOP) {
9974 			off++;
9975 			continue;
9976 		}
9977 		if (off + 2 > end) {
9978 			DPFPRINTF(PF_DEBUG_MISC, "IP length opt");
9979 			REASON_SET(reason, PFRES_IPOPTIONS);
9980 			return (PF_DROP);
9981 		}
9982 		length = opts[off + 1];
9983 		if (length < 2) {
9984 			DPFPRINTF(PF_DEBUG_MISC, "IP short opt");
9985 			REASON_SET(reason, PFRES_IPOPTIONS);
9986 			return (PF_DROP);
9987 		}
9988 		if (off + length > end) {
9989 			DPFPRINTF(PF_DEBUG_MISC, "IP long opt");
9990 			REASON_SET(reason, PFRES_IPOPTIONS);
9991 			return (PF_DROP);
9992 		}
9993 		switch (type) {
9994 		case IPOPT_RA:
9995 			pd->badopts |= PF_OPT_ROUTER_ALERT;
9996 			break;
9997 		default:
9998 			pd->badopts |= PF_OPT_OTHER;
9999 			break;
10000 		}
10001 		off += length;
10002 	}
10003 
10004 	return (PF_PASS);
10005 }
10006 
10007 static int
10008 pf_walk_header(struct pf_pdesc *pd, struct ip *h, u_short *reason)
10009 {
10010 	struct ah	 ext;
10011 	u_int32_t	 hlen, end;
10012 	int		 hdr_cnt;
10013 
10014 	hlen = h->ip_hl << 2;
10015 	if (hlen < sizeof(struct ip) || hlen > ntohs(h->ip_len)) {
10016 		REASON_SET(reason, PFRES_SHORT);
10017 		return (PF_DROP);
10018 	}
10019 	if (hlen != sizeof(struct ip)) {
10020 		if (pf_walk_option(pd, h, pd->off + sizeof(struct ip),
10021 		    pd->off + hlen, reason) != PF_PASS)
10022 			return (PF_DROP);
10023 		/* header options which contain only padding is fishy */
10024 		if (pd->badopts == 0)
10025 			pd->badopts |= PF_OPT_OTHER;
10026 	}
10027 	end = pd->off + ntohs(h->ip_len);
10028 	pd->off += hlen;
10029 	pd->proto = h->ip_p;
10030 	/* IGMP packets have router alert options, allow them */
10031 	if (pd->proto == IPPROTO_IGMP) {
10032 		/*
10033 		 * According to RFC 1112 ttl must be set to 1 in all IGMP
10034 		 * packets sent to 224.0.0.1
10035 		 */
10036 		if ((h->ip_ttl != 1) &&
10037 		    (h->ip_dst.s_addr == INADDR_ALLHOSTS_GROUP)) {
10038 			DPFPRINTF(PF_DEBUG_MISC, "Invalid IGMP");
10039 			REASON_SET(reason, PFRES_IPOPTIONS);
10040 			return (PF_DROP);
10041 		}
10042 		pd->badopts &= ~PF_OPT_ROUTER_ALERT;
10043 	}
10044 	/* stop walking over non initial fragments */
10045 	if ((h->ip_off & htons(IP_OFFMASK)) != 0)
10046 		return (PF_PASS);
10047 	for (hdr_cnt = 0; hdr_cnt < PF_HDR_LIMIT; hdr_cnt++) {
10048 		switch (pd->proto) {
10049 		case IPPROTO_AH:
10050 			/* fragments may be short */
10051 			if ((h->ip_off & htons(IP_MF | IP_OFFMASK)) != 0 &&
10052 			    end < pd->off + sizeof(ext))
10053 				return (PF_PASS);
10054 			if (!pf_pull_hdr(pd->m, pd->off, &ext, sizeof(ext),
10055 				NULL, reason, AF_INET)) {
10056 				DPFPRINTF(PF_DEBUG_MISC, "IP short exthdr");
10057 				return (PF_DROP);
10058 			}
10059 			pd->off += (ext.ah_len + 2) * 4;
10060 			pd->proto = ext.ah_nxt;
10061 			break;
10062 		default:
10063 			return (PF_PASS);
10064 		}
10065 	}
10066 	DPFPRINTF(PF_DEBUG_MISC, "IPv4 nested authentication header limit");
10067 	REASON_SET(reason, PFRES_IPOPTIONS);
10068 	return (PF_DROP);
10069 }
10070 
10071 #ifdef INET6
10072 static int
10073 pf_walk_option6(struct pf_pdesc *pd, struct ip6_hdr *h, int off, int end,
10074     u_short *reason)
10075 {
10076 	struct ip6_opt		 opt;
10077 	struct ip6_opt_jumbo	 jumbo;
10078 
10079 	while (off < end) {
10080 		if (!pf_pull_hdr(pd->m, off, &opt.ip6o_type,
10081 		    sizeof(opt.ip6o_type), NULL, reason, AF_INET6)) {
10082 			DPFPRINTF(PF_DEBUG_MISC, "IPv6 short opt type");
10083 			return (PF_DROP);
10084 		}
10085 		if (opt.ip6o_type == IP6OPT_PAD1) {
10086 			off++;
10087 			continue;
10088 		}
10089 		if (!pf_pull_hdr(pd->m, off, &opt, sizeof(opt), NULL,
10090 		    reason, AF_INET6)) {
10091 			DPFPRINTF(PF_DEBUG_MISC, "IPv6 short opt");
10092 			return (PF_DROP);
10093 		}
10094 		if (off + sizeof(opt) + opt.ip6o_len > end) {
10095 			DPFPRINTF(PF_DEBUG_MISC, "IPv6 long opt");
10096 			REASON_SET(reason, PFRES_IPOPTIONS);
10097 			return (PF_DROP);
10098 		}
10099 		switch (opt.ip6o_type) {
10100 		case IP6OPT_PADN:
10101 			break;
10102 		case IP6OPT_JUMBO:
10103 			pd->badopts |= PF_OPT_JUMBO;
10104 			if (pd->jumbolen != 0) {
10105 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 multiple jumbo");
10106 				REASON_SET(reason, PFRES_IPOPTIONS);
10107 				return (PF_DROP);
10108 			}
10109 			if (ntohs(h->ip6_plen) != 0) {
10110 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 bad jumbo plen");
10111 				REASON_SET(reason, PFRES_IPOPTIONS);
10112 				return (PF_DROP);
10113 			}
10114 			if (!pf_pull_hdr(pd->m, off, &jumbo, sizeof(jumbo), NULL,
10115 				reason, AF_INET6)) {
10116 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 short jumbo");
10117 				return (PF_DROP);
10118 			}
10119 			memcpy(&pd->jumbolen, jumbo.ip6oj_jumbo_len,
10120 			    sizeof(pd->jumbolen));
10121 			pd->jumbolen = ntohl(pd->jumbolen);
10122 			if (pd->jumbolen < IPV6_MAXPACKET) {
10123 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 short jumbolen");
10124 				REASON_SET(reason, PFRES_IPOPTIONS);
10125 				return (PF_DROP);
10126 			}
10127 			break;
10128 		case IP6OPT_ROUTER_ALERT:
10129 			pd->badopts |= PF_OPT_ROUTER_ALERT;
10130 			break;
10131 		default:
10132 			pd->badopts |= PF_OPT_OTHER;
10133 			break;
10134 		}
10135 		off += sizeof(opt) + opt.ip6o_len;
10136 	}
10137 
10138 	return (PF_PASS);
10139 }
10140 
10141 int
10142 pf_walk_header6(struct pf_pdesc *pd, struct ip6_hdr *h, u_short *reason)
10143 {
10144 	struct ip6_frag		 frag;
10145 	struct ip6_ext		 ext;
10146 	struct icmp6_hdr	 icmp6;
10147 	struct ip6_rthdr	 rthdr;
10148 	uint32_t		 end;
10149 	int			 hdr_cnt, fraghdr_cnt = 0, rthdr_cnt = 0;
10150 
10151 	pd->off += sizeof(struct ip6_hdr);
10152 	end = pd->off + ntohs(h->ip6_plen);
10153 	pd->fragoff = pd->extoff = pd->jumbolen = 0;
10154 	pd->proto = h->ip6_nxt;
10155 	for (hdr_cnt = 0; hdr_cnt < PF_HDR_LIMIT; hdr_cnt++) {
10156 		switch (pd->proto) {
10157 		case IPPROTO_ROUTING:
10158 		case IPPROTO_DSTOPTS:
10159 			pd->badopts |= PF_OPT_OTHER;
10160 			break;
10161 		case IPPROTO_HOPOPTS:
10162 			if (!pf_pull_hdr(pd->m, pd->off, &ext, sizeof(ext),
10163 			    NULL, reason, AF_INET6)) {
10164 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 short exthdr");
10165 				return (PF_DROP);
10166 			}
10167 			if (pf_walk_option6(pd, h, pd->off + sizeof(ext),
10168 				pd->off + (ext.ip6e_len + 1) * 8,
10169 				reason) != PF_PASS)
10170 				return (PF_DROP);
10171 			/* option header which contains only padding is fishy */
10172 			if (pd->badopts == 0)
10173 				pd->badopts |= PF_OPT_OTHER;
10174 			break;
10175 		}
10176 		switch (pd->proto) {
10177 		case IPPROTO_FRAGMENT:
10178 			if (fraghdr_cnt++) {
10179 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 multiple fragment");
10180 				REASON_SET(reason, PFRES_FRAG);
10181 				return (PF_DROP);
10182 			}
10183 			/* jumbo payload packets cannot be fragmented */
10184 			if (pd->jumbolen != 0) {
10185 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 fragmented jumbo");
10186 				REASON_SET(reason, PFRES_FRAG);
10187 				return (PF_DROP);
10188 			}
10189 			if (!pf_pull_hdr(pd->m, pd->off, &frag, sizeof(frag),
10190 			    NULL, reason, AF_INET6)) {
10191 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 short fragment");
10192 				return (PF_DROP);
10193 			}
10194 			/* stop walking over non initial fragments */
10195 			if (ntohs((frag.ip6f_offlg & IP6F_OFF_MASK)) != 0) {
10196 				pd->fragoff = pd->off;
10197 				return (PF_PASS);
10198 			}
10199 			/* RFC6946:  reassemble only non atomic fragments */
10200 			if (frag.ip6f_offlg & IP6F_MORE_FRAG)
10201 				pd->fragoff = pd->off;
10202 			pd->off += sizeof(frag);
10203 			pd->proto = frag.ip6f_nxt;
10204 			break;
10205 		case IPPROTO_ROUTING:
10206 			if (rthdr_cnt++) {
10207 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 multiple rthdr");
10208 				REASON_SET(reason, PFRES_IPOPTIONS);
10209 				return (PF_DROP);
10210 			}
10211 			/* fragments may be short */
10212 			if (pd->fragoff != 0 && end < pd->off + sizeof(rthdr)) {
10213 				pd->off = pd->fragoff;
10214 				pd->proto = IPPROTO_FRAGMENT;
10215 				return (PF_PASS);
10216 			}
10217 			if (!pf_pull_hdr(pd->m, pd->off, &rthdr, sizeof(rthdr),
10218 			    NULL, reason, AF_INET6)) {
10219 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 short rthdr");
10220 				return (PF_DROP);
10221 			}
10222 			if (rthdr.ip6r_type == IPV6_RTHDR_TYPE_0) {
10223 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 rthdr0");
10224 				REASON_SET(reason, PFRES_IPOPTIONS);
10225 				return (PF_DROP);
10226 			}
10227 			/* FALLTHROUGH */
10228 		case IPPROTO_HOPOPTS:
10229 			/* RFC2460 4.1:  Hop-by-Hop only after IPv6 header */
10230 			if (pd->proto == IPPROTO_HOPOPTS && hdr_cnt > 0) {
10231 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 hopopts not first");
10232 				REASON_SET(reason, PFRES_IPOPTIONS);
10233 				return (PF_DROP);
10234 			}
10235 			/* FALLTHROUGH */
10236 		case IPPROTO_AH:
10237 		case IPPROTO_DSTOPTS:
10238 			if (!pf_pull_hdr(pd->m, pd->off, &ext, sizeof(ext),
10239 			    NULL, reason, AF_INET6)) {
10240 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 short exthdr");
10241 				return (PF_DROP);
10242 			}
10243 			/* fragments may be short */
10244 			if (pd->fragoff != 0 && end < pd->off + sizeof(ext)) {
10245 				pd->off = pd->fragoff;
10246 				pd->proto = IPPROTO_FRAGMENT;
10247 				return (PF_PASS);
10248 			}
10249 			/* reassembly needs the ext header before the frag */
10250 			if (pd->fragoff == 0)
10251 				pd->extoff = pd->off;
10252 			if (pd->proto == IPPROTO_HOPOPTS && pd->fragoff == 0 &&
10253 			    ntohs(h->ip6_plen) == 0 && pd->jumbolen != 0) {
10254 				DPFPRINTF(PF_DEBUG_MISC, "IPv6 missing jumbo");
10255 				REASON_SET(reason, PFRES_IPOPTIONS);
10256 				return (PF_DROP);
10257 			}
10258 			if (pd->proto == IPPROTO_AH)
10259 				pd->off += (ext.ip6e_len + 2) * 4;
10260 			else
10261 				pd->off += (ext.ip6e_len + 1) * 8;
10262 			pd->proto = ext.ip6e_nxt;
10263 			break;
10264 		case IPPROTO_ICMPV6:
10265 			/* fragments may be short, ignore inner header then */
10266 			if (pd->fragoff != 0 && end < pd->off + sizeof(icmp6)) {
10267 				pd->off = pd->fragoff;
10268 				pd->proto = IPPROTO_FRAGMENT;
10269 				return (PF_PASS);
10270 			}
10271 			if (!pf_pull_hdr(pd->m, pd->off, &icmp6, sizeof(icmp6),
10272 				NULL, reason, AF_INET6)) {
10273 				DPFPRINTF(PF_DEBUG_MISC,
10274 				    "IPv6 short icmp6hdr");
10275 				return (PF_DROP);
10276 			}
10277 			/* ICMP multicast packets have router alert options */
10278 			switch (icmp6.icmp6_type) {
10279 			case MLD_LISTENER_QUERY:
10280 			case MLD_LISTENER_REPORT:
10281 			case MLD_LISTENER_DONE:
10282 			case MLDV2_LISTENER_REPORT:
10283 				/*
10284 				 * According to RFC 2710 all MLD messages are
10285 				 * sent with hop-limit (ttl) set to 1, and link
10286 				 * local source address.  If either one is
10287 				 * missing then MLD message is invalid and
10288 				 * should be discarded.
10289 				 */
10290 				if ((h->ip6_hlim != 1) ||
10291 				    !IN6_IS_ADDR_LINKLOCAL(&h->ip6_src)) {
10292 					DPFPRINTF(PF_DEBUG_MISC, "Invalid MLD");
10293 					REASON_SET(reason, PFRES_IPOPTIONS);
10294 					return (PF_DROP);
10295 				}
10296 				pd->badopts &= ~PF_OPT_ROUTER_ALERT;
10297 				break;
10298 			}
10299 			return (PF_PASS);
10300 		case IPPROTO_TCP:
10301 		case IPPROTO_UDP:
10302 		case IPPROTO_SCTP:
10303 			/* fragments may be short, ignore inner header then */
10304 			if (pd->fragoff != 0 && end < pd->off +
10305 			    (pd->proto == IPPROTO_TCP ? sizeof(struct tcphdr) :
10306 			    pd->proto == IPPROTO_UDP ? sizeof(struct udphdr) :
10307 			    pd->proto == IPPROTO_SCTP ? sizeof(struct sctphdr) :
10308 			    sizeof(struct icmp6_hdr))) {
10309 				pd->off = pd->fragoff;
10310 				pd->proto = IPPROTO_FRAGMENT;
10311 			}
10312 			/* FALLTHROUGH */
10313 		default:
10314 			return (PF_PASS);
10315 		}
10316 	}
10317 	DPFPRINTF(PF_DEBUG_MISC, "IPv6 nested extension header limit");
10318 	REASON_SET(reason, PFRES_IPOPTIONS);
10319 	return (PF_DROP);
10320 }
10321 #endif /* INET6 */
10322 
10323 static void
10324 pf_init_pdesc(struct pf_pdesc *pd, struct mbuf *m)
10325 {
10326 	memset(pd, 0, sizeof(*pd));
10327 	pd->pf_mtag = pf_find_mtag(m);
10328 	pd->m = m;
10329 }
10330 
10331 static int
10332 pf_setup_pdesc(sa_family_t af, int dir, struct pf_pdesc *pd, struct mbuf **m0,
10333     u_short *action, u_short *reason, struct pfi_kkif *kif,
10334     struct pf_rule_actions *default_actions)
10335 {
10336 	pd->dir = dir;
10337 	pd->kif = kif;
10338 	pd->m = *m0;
10339 	pd->sidx = (dir == PF_IN) ? 0 : 1;
10340 	pd->didx = (dir == PF_IN) ? 1 : 0;
10341 	pd->af = pd->naf = af;
10342 
10343 	PF_RULES_ASSERT();
10344 
10345 	TAILQ_INIT(&pd->sctp_multihome_jobs);
10346 	if (default_actions != NULL)
10347 		memcpy(&pd->act, default_actions, sizeof(pd->act));
10348 
10349 	if (pd->pf_mtag && pd->pf_mtag->dnpipe) {
10350 		pd->act.dnpipe = pd->pf_mtag->dnpipe;
10351 		pd->act.flags = pd->pf_mtag->dnflags;
10352 	}
10353 
10354 	switch (af) {
10355 #ifdef INET
10356 	case AF_INET: {
10357 		struct ip *h;
10358 
10359 		if (__predict_false((*m0)->m_len < sizeof(struct ip)) &&
10360 		    (pd->m = *m0 = m_pullup(*m0, sizeof(struct ip))) == NULL) {
10361 			DPFPRINTF(PF_DEBUG_URGENT,
10362 			    "%s: m_len < sizeof(struct ip), pullup failed",
10363 			    __func__);
10364 			*action = PF_DROP;
10365 			REASON_SET(reason, PFRES_SHORT);
10366 			return (-1);
10367 		}
10368 
10369 		h = mtod(pd->m, struct ip *);
10370 		if (pd->m->m_pkthdr.len < ntohs(h->ip_len)) {
10371 			*action = PF_DROP;
10372 			REASON_SET(reason, PFRES_SHORT);
10373 			return (-1);
10374 		}
10375 
10376 		if (pf_normalize_ip(reason, pd) != PF_PASS) {
10377 			/* We do IP header normalization and packet reassembly here */
10378 			*m0 = pd->m;
10379 			*action = PF_DROP;
10380 			return (-1);
10381 		}
10382 		*m0 = pd->m;
10383 		h = mtod(pd->m, struct ip *);
10384 
10385 		if (pf_walk_header(pd, h, reason) != PF_PASS) {
10386 			*action = PF_DROP;
10387 			return (-1);
10388 		}
10389 
10390 		pd->src = (struct pf_addr *)&h->ip_src;
10391 		pd->dst = (struct pf_addr *)&h->ip_dst;
10392 		pf_addrcpy(&pd->osrc, pd->src, af);
10393 		pf_addrcpy(&pd->odst, pd->dst, af);
10394 		pd->ip_sum = &h->ip_sum;
10395 		pd->tos = h->ip_tos & ~IPTOS_ECN_MASK;
10396 		pd->ttl = h->ip_ttl;
10397 		pd->tot_len = ntohs(h->ip_len);
10398 		pd->act.rtableid = -1;
10399 		pd->df = h->ip_off & htons(IP_DF);
10400 		pd->virtual_proto = (h->ip_off & htons(IP_MF | IP_OFFMASK)) ?
10401 		    PF_VPROTO_FRAGMENT : pd->proto;
10402 
10403 		break;
10404 	}
10405 #endif /* INET */
10406 #ifdef INET6
10407 	case AF_INET6: {
10408 		struct ip6_hdr *h;
10409 
10410 		if (__predict_false((*m0)->m_len < sizeof(struct ip6_hdr)) &&
10411 		    (pd->m = *m0 = m_pullup(*m0, sizeof(struct ip6_hdr))) == NULL) {
10412 			DPFPRINTF(PF_DEBUG_URGENT,
10413 			    "%s: m_len < sizeof(struct ip6_hdr)"
10414 			     ", pullup failed", __func__);
10415 			*action = PF_DROP;
10416 			REASON_SET(reason, PFRES_SHORT);
10417 			return (-1);
10418 		}
10419 
10420 		h = mtod(pd->m, struct ip6_hdr *);
10421 		if (pd->m->m_pkthdr.len <
10422 		    sizeof(struct ip6_hdr) + ntohs(h->ip6_plen)) {
10423 			*action = PF_DROP;
10424 			REASON_SET(reason, PFRES_SHORT);
10425 			return (-1);
10426 		}
10427 
10428 		/*
10429 		 * we do not support jumbogram.  if we keep going, zero ip6_plen
10430 		 * will do something bad, so drop the packet for now.
10431 		 */
10432 		if (htons(h->ip6_plen) == 0) {
10433 			*action = PF_DROP;
10434 			return (-1);
10435 		}
10436 
10437 		if (pf_walk_header6(pd, h, reason) != PF_PASS) {
10438 			*action = PF_DROP;
10439 			return (-1);
10440 		}
10441 
10442 		h = mtod(pd->m, struct ip6_hdr *);
10443 		pd->src = (struct pf_addr *)&h->ip6_src;
10444 		pd->dst = (struct pf_addr *)&h->ip6_dst;
10445 		pf_addrcpy(&pd->osrc, pd->src, af);
10446 		pf_addrcpy(&pd->odst, pd->dst, af);
10447 		pd->ip_sum = NULL;
10448 		pd->tos = IPV6_DSCP(h);
10449 		pd->ttl = h->ip6_hlim;
10450 		pd->tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr);
10451 		pd->act.rtableid = -1;
10452 
10453 		pd->virtual_proto = (pd->fragoff != 0) ?
10454 		    PF_VPROTO_FRAGMENT : pd->proto;
10455 
10456 		/* We do IP header normalization and packet reassembly here */
10457 		if (pf_normalize_ip6(pd->fragoff, reason, pd) !=
10458 		    PF_PASS) {
10459 			*m0 = pd->m;
10460 			*action = PF_DROP;
10461 			return (-1);
10462 		}
10463 		*m0 = pd->m;
10464 		if (pd->m == NULL) {
10465 			/* packet sits in reassembly queue, no error */
10466 			*action = PF_PASS;
10467 			return (-1);
10468 		}
10469 
10470 		/* Update pointers into the packet. */
10471 		h = mtod(pd->m, struct ip6_hdr *);
10472 		pd->src = (struct pf_addr *)&h->ip6_src;
10473 		pd->dst = (struct pf_addr *)&h->ip6_dst;
10474 
10475 		pd->off = 0;
10476 
10477 		if (pf_walk_header6(pd, h, reason) != PF_PASS) {
10478 			*action = PF_DROP;
10479 			return (-1);
10480 		}
10481 
10482 		if (m_tag_find(pd->m, PACKET_TAG_PF_REASSEMBLED, NULL) != NULL) {
10483 			/*
10484 			 * Reassembly may have changed the next protocol from
10485 			 * fragment to something else, so update.
10486 			 */
10487 			pd->virtual_proto = pd->proto;
10488 			MPASS(pd->fragoff == 0);
10489 		}
10490 
10491 		if (pd->fragoff != 0)
10492 			pd->virtual_proto = PF_VPROTO_FRAGMENT;
10493 
10494 		break;
10495 	}
10496 #endif /* INET6 */
10497 	default:
10498 		panic("pf_setup_pdesc called with illegal af %u", af);
10499 	}
10500 
10501 	switch (pd->virtual_proto) {
10502 	case IPPROTO_TCP: {
10503 		struct tcphdr *th = &pd->hdr.tcp;
10504 
10505 		if (!pf_pull_hdr(pd->m, pd->off, th, sizeof(*th), action,
10506 			reason, af)) {
10507 			*action = PF_DROP;
10508 			REASON_SET(reason, PFRES_SHORT);
10509 			return (-1);
10510 		}
10511 		pd->hdrlen = sizeof(*th);
10512 		pd->p_len = pd->tot_len - pd->off - (th->th_off << 2);
10513 		pd->sport = &th->th_sport;
10514 		pd->dport = &th->th_dport;
10515 		pd->pcksum = &th->th_sum;
10516 		break;
10517 	}
10518 	case IPPROTO_UDP: {
10519 		struct udphdr *uh = &pd->hdr.udp;
10520 
10521 		if (!pf_pull_hdr(pd->m, pd->off, uh, sizeof(*uh), action,
10522 			reason, af)) {
10523 			*action = PF_DROP;
10524 			REASON_SET(reason, PFRES_SHORT);
10525 			return (-1);
10526 		}
10527 		pd->hdrlen = sizeof(*uh);
10528 		if (uh->uh_dport == 0 ||
10529 		    ntohs(uh->uh_ulen) > pd->m->m_pkthdr.len - pd->off ||
10530 		    ntohs(uh->uh_ulen) < sizeof(struct udphdr)) {
10531 			*action = PF_DROP;
10532 			REASON_SET(reason, PFRES_SHORT);
10533 			return (-1);
10534 		}
10535 		pd->sport = &uh->uh_sport;
10536 		pd->dport = &uh->uh_dport;
10537 		pd->pcksum = &uh->uh_sum;
10538 		break;
10539 	}
10540 	case IPPROTO_SCTP: {
10541 		if (!pf_pull_hdr(pd->m, pd->off, &pd->hdr.sctp, sizeof(pd->hdr.sctp),
10542 		    action, reason, af)) {
10543 			*action = PF_DROP;
10544 			REASON_SET(reason, PFRES_SHORT);
10545 			return (-1);
10546 		}
10547 		pd->hdrlen = sizeof(pd->hdr.sctp);
10548 		pd->p_len = pd->tot_len - pd->off;
10549 
10550 		pd->sport = &pd->hdr.sctp.src_port;
10551 		pd->dport = &pd->hdr.sctp.dest_port;
10552 		if (pd->hdr.sctp.src_port == 0 || pd->hdr.sctp.dest_port == 0) {
10553 			*action = PF_DROP;
10554 			REASON_SET(reason, PFRES_SHORT);
10555 			return (-1);
10556 		}
10557 		if (pf_scan_sctp(pd) != PF_PASS) {
10558 			*action = PF_DROP;
10559 			REASON_SET(reason, PFRES_SHORT);
10560 			return (-1);
10561 		}
10562 		/*
10563 		 * Placeholder. The SCTP checksum is 32-bits, but
10564 		 * pf_test_state() expects to update a 16-bit checksum.
10565 		 * Provide a dummy value which we'll subsequently ignore.
10566 		 */
10567 		pd->pcksum = &pd->sctp_dummy_sum;
10568 		break;
10569 	}
10570 	case IPPROTO_ICMP: {
10571 		if (!pf_pull_hdr(pd->m, pd->off, &pd->hdr.icmp, ICMP_MINLEN,
10572 			action, reason, af)) {
10573 			*action = PF_DROP;
10574 			REASON_SET(reason, PFRES_SHORT);
10575 			return (-1);
10576 		}
10577 		pd->pcksum = &pd->hdr.icmp.icmp_cksum;
10578 		pd->hdrlen = ICMP_MINLEN;
10579 		break;
10580 	}
10581 #ifdef INET6
10582 	case IPPROTO_ICMPV6: {
10583 		size_t icmp_hlen = sizeof(struct icmp6_hdr);
10584 
10585 		if (!pf_pull_hdr(pd->m, pd->off, &pd->hdr.icmp6, icmp_hlen,
10586 			action, reason, af)) {
10587 			*action = PF_DROP;
10588 			REASON_SET(reason, PFRES_SHORT);
10589 			return (-1);
10590 		}
10591 		/* ICMP headers we look further into to match state */
10592 		switch (pd->hdr.icmp6.icmp6_type) {
10593 		case MLD_LISTENER_QUERY:
10594 		case MLD_LISTENER_REPORT:
10595 			icmp_hlen = sizeof(struct mld_hdr);
10596 			break;
10597 		case ND_NEIGHBOR_SOLICIT:
10598 		case ND_NEIGHBOR_ADVERT:
10599 			icmp_hlen = sizeof(struct nd_neighbor_solicit);
10600 			/* FALLTHROUGH */
10601 		case ND_ROUTER_SOLICIT:
10602 		case ND_ROUTER_ADVERT:
10603 		case ND_REDIRECT:
10604 			if (pd->ttl != 255) {
10605 				REASON_SET(reason, PFRES_NORM);
10606 				return (PF_DROP);
10607 			}
10608 			break;
10609 		}
10610 		if (icmp_hlen > sizeof(struct icmp6_hdr) &&
10611 		    !pf_pull_hdr(pd->m, pd->off, &pd->hdr.icmp6, icmp_hlen,
10612 			action, reason, af)) {
10613 			*action = PF_DROP;
10614 			REASON_SET(reason, PFRES_SHORT);
10615 			return (-1);
10616 		}
10617 		pd->hdrlen = icmp_hlen;
10618 		pd->pcksum = &pd->hdr.icmp6.icmp6_cksum;
10619 		break;
10620 	}
10621 #endif /* INET6 */
10622 	default:
10623 		/*
10624 		 * Placeholder value, so future calls to pf_change_ap() don't
10625 		 * try to update a NULL checksum pointer.
10626 		*/
10627 		pd->pcksum = &pd->sctp_dummy_sum;
10628 		break;
10629 	}
10630 
10631 	if (pd->sport)
10632 		pd->osport = pd->nsport = *pd->sport;
10633 	if (pd->dport)
10634 		pd->odport = pd->ndport = *pd->dport;
10635 
10636 	MPASS(pd->pcksum != NULL);
10637 
10638 	return (0);
10639 }
10640 
10641 static void
10642 pf_counters_inc(int action, struct pf_pdesc *pd,
10643     struct pf_kstate *s, struct pf_krule *r, struct pf_krule *a)
10644 {
10645 	struct pf_krule		*tr;
10646 	int			 dir = pd->dir;
10647 	int			 dirndx;
10648 
10649 	pf_counter_u64_critical_enter();
10650 	pf_counter_u64_add_protected(
10651 	    &pd->kif->pfik_bytes[pd->af == AF_INET6][dir == PF_OUT][action != PF_PASS],
10652 	    pd->tot_len);
10653 	pf_counter_u64_add_protected(
10654 	    &pd->kif->pfik_packets[pd->af == AF_INET6][dir == PF_OUT][action != PF_PASS],
10655 	    1);
10656 
10657 	if (action == PF_PASS || action == PF_AFRT || r->action == PF_DROP) {
10658 		dirndx = (dir == PF_OUT);
10659 		pf_counter_u64_add_protected(&r->packets[dirndx], 1);
10660 		pf_counter_u64_add_protected(&r->bytes[dirndx], pd->tot_len);
10661 		pf_update_timestamp(r);
10662 
10663 		if (a != NULL) {
10664 			pf_counter_u64_add_protected(&a->packets[dirndx], 1);
10665 			pf_counter_u64_add_protected(&a->bytes[dirndx], pd->tot_len);
10666 		}
10667 		if (s != NULL) {
10668 			struct pf_krule_item	*ri;
10669 
10670 			if (s->nat_rule != NULL) {
10671 				pf_counter_u64_add_protected(&s->nat_rule->packets[dirndx],
10672 				    1);
10673 				pf_counter_u64_add_protected(&s->nat_rule->bytes[dirndx],
10674 				    pd->tot_len);
10675 			}
10676 			/*
10677 			 * Source nodes are accessed unlocked here.
10678 			 * But since we are operating with stateful tracking
10679 			 * and the state is locked, those SNs could not have
10680 			 * been freed.
10681 			 */
10682 			for (pf_sn_types_t sn_type=0; sn_type<PF_SN_MAX; sn_type++) {
10683 				if (s->sns[sn_type] != NULL) {
10684 					counter_u64_add(
10685 					    s->sns[sn_type]->packets[dirndx],
10686 					    1);
10687 					counter_u64_add(
10688 					    s->sns[sn_type]->bytes[dirndx],
10689 					    pd->tot_len);
10690 				}
10691 			}
10692 			dirndx = (dir == s->direction) ? 0 : 1;
10693 			s->packets[dirndx]++;
10694 			s->bytes[dirndx] += pd->tot_len;
10695 
10696 			SLIST_FOREACH(ri, &s->match_rules, entry) {
10697 				pf_counter_u64_add_protected(&ri->r->packets[dirndx], 1);
10698 				pf_counter_u64_add_protected(&ri->r->bytes[dirndx], pd->tot_len);
10699 
10700 				if (ri->r->src.addr.type == PF_ADDR_TABLE)
10701 					pfr_update_stats(ri->r->src.addr.p.tbl,
10702 					    (s == NULL) ? pd->src :
10703 					    &s->key[(s->direction == PF_IN)]->
10704 						addr[(s->direction == PF_OUT)],
10705 					    pd->af, pd->tot_len, dir == PF_OUT,
10706 					    r->action == PF_PASS, ri->r->src.neg);
10707 				if (ri->r->dst.addr.type == PF_ADDR_TABLE)
10708 					pfr_update_stats(ri->r->dst.addr.p.tbl,
10709 					    (s == NULL) ? pd->dst :
10710 					    &s->key[(s->direction == PF_IN)]->
10711 						addr[(s->direction == PF_IN)],
10712 					    pd->af, pd->tot_len, dir == PF_OUT,
10713 					    r->action == PF_PASS, ri->r->dst.neg);
10714 			}
10715 		}
10716 
10717 		tr = r;
10718 		if (s != NULL && s->nat_rule != NULL &&
10719 		    r == &V_pf_default_rule)
10720 			tr = s->nat_rule;
10721 
10722 		if (tr->src.addr.type == PF_ADDR_TABLE)
10723 			pfr_update_stats(tr->src.addr.p.tbl,
10724 			    (s == NULL) ? pd->src :
10725 			    &s->key[(s->direction == PF_IN)]->
10726 				addr[(s->direction == PF_OUT)],
10727 			    pd->af, pd->tot_len, dir == PF_OUT,
10728 			    r->action == PF_PASS, tr->src.neg);
10729 		if (tr->dst.addr.type == PF_ADDR_TABLE)
10730 			pfr_update_stats(tr->dst.addr.p.tbl,
10731 			    (s == NULL) ? pd->dst :
10732 			    &s->key[(s->direction == PF_IN)]->
10733 				addr[(s->direction == PF_IN)],
10734 			    pd->af, pd->tot_len, dir == PF_OUT,
10735 			    r->action == PF_PASS, tr->dst.neg);
10736 	}
10737 	pf_counter_u64_critical_exit();
10738 }
10739 static void
10740 pf_log_matches(struct pf_pdesc *pd, struct pf_krule *rm,
10741     struct pf_krule *am, struct pf_kruleset *ruleset,
10742     struct pf_krule_slist *matchrules)
10743 {
10744 	struct pf_krule_item	*ri;
10745 
10746 	/* if this is the log(matches) rule, packet has been logged already */
10747 	if (rm->log & PF_LOG_MATCHES)
10748 		return;
10749 
10750 	SLIST_FOREACH(ri, matchrules, entry)
10751 		if (ri->r->log & PF_LOG_MATCHES)
10752 			PFLOG_PACKET(rm->action, PFRES_MATCH, rm, am,
10753 			    ruleset, pd, 1, ri->r);
10754 }
10755 
10756 #if defined(INET) || defined(INET6)
10757 int
10758 pf_test(sa_family_t af, int dir, int pflags, struct ifnet *ifp, struct mbuf **m0,
10759     struct inpcb *inp, struct pf_rule_actions *default_actions)
10760 {
10761 	struct pfi_kkif		*kif;
10762 	u_short			 action, reason = 0;
10763 	struct m_tag		*mtag;
10764 	struct pf_krule		*a = NULL, *r = &V_pf_default_rule;
10765 	struct pf_kstate	*s = NULL;
10766 	struct pf_kruleset	*ruleset = NULL;
10767 	struct pf_pdesc		 pd;
10768 	int			 use_2nd_queue = 0;
10769 	uint16_t		 tag;
10770 
10771 	PF_RULES_RLOCK_TRACKER;
10772 	KASSERT(dir == PF_IN || dir == PF_OUT, ("%s: bad direction %d\n", __func__, dir));
10773 	M_ASSERTPKTHDR(*m0);
10774 	NET_EPOCH_ASSERT();
10775 
10776 	if (!V_pf_status.running)
10777 		return (PF_PASS);
10778 
10779 	kif = (struct pfi_kkif *)ifp->if_pf_kif;
10780 
10781 	if (__predict_false(kif == NULL)) {
10782 		DPFPRINTF(PF_DEBUG_URGENT,
10783 		    "%s: kif == NULL, if_xname %s",
10784 		    __func__, ifp->if_xname);
10785 		return (PF_DROP);
10786 	}
10787 	if (kif->pfik_flags & PFI_IFLAG_SKIP) {
10788 		return (PF_PASS);
10789 	}
10790 
10791 	if ((*m0)->m_flags & M_SKIP_FIREWALL) {
10792 		return (PF_PASS);
10793 	}
10794 
10795 	if (__predict_false(! M_WRITABLE(*m0))) {
10796 		*m0 = m_unshare(*m0, M_NOWAIT);
10797 		if (*m0 == NULL) {
10798 			return (PF_DROP);
10799 		}
10800 	}
10801 
10802 	pf_init_pdesc(&pd, *m0);
10803 
10804 	if (pd.pf_mtag != NULL && (pd.pf_mtag->flags & PF_MTAG_FLAG_ROUTE_TO)) {
10805 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_ROUTE_TO;
10806 
10807 		ifp = ifnet_byindexgen(pd.pf_mtag->if_index,
10808 		    pd.pf_mtag->if_idxgen);
10809 		if (ifp == NULL || ifp->if_flags & IFF_DYING) {
10810 			m_freem(*m0);
10811 			*m0 = NULL;
10812 			return (PF_PASS);
10813 		}
10814 		(ifp->if_output)(ifp, *m0, sintosa(&pd.pf_mtag->dst), NULL);
10815 		*m0 = NULL;
10816 		return (PF_PASS);
10817 	}
10818 
10819 	if (ip_dn_io_ptr != NULL && pd.pf_mtag != NULL &&
10820 	    pd.pf_mtag->flags & PF_MTAG_FLAG_DUMMYNET) {
10821 		/* Dummynet re-injects packets after they've
10822 		 * completed their delay. We've already
10823 		 * processed them, so pass unconditionally. */
10824 
10825 		/* But only once. We may see the packet multiple times (e.g.
10826 		 * PFIL_IN/PFIL_OUT). */
10827 		pf_dummynet_flag_remove(pd.m, pd.pf_mtag);
10828 
10829 		return (PF_PASS);
10830 	}
10831 
10832 	PF_RULES_RLOCK();
10833 
10834 	if (pf_setup_pdesc(af, dir, &pd, m0, &action, &reason,
10835 		kif, default_actions) == -1) {
10836 		if (action != PF_PASS)
10837 			pd.act.log |= PF_LOG_FORCE;
10838 		goto done;
10839 	}
10840 
10841 #ifdef INET
10842 	if (af == AF_INET && dir == PF_OUT && pflags & PFIL_FWD &&
10843 	    pd.df && (*m0)->m_pkthdr.len > ifp->if_mtu) {
10844 		PF_RULES_RUNLOCK();
10845 		icmp_error(*m0, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG,
10846 			0, ifp->if_mtu);
10847 		*m0 = NULL;
10848 		return (PF_DROP);
10849 	}
10850 #endif /* INET */
10851 #ifdef INET6
10852 	/*
10853 	 * If we end up changing IP addresses (e.g. binat) the stack may get
10854 	 * confused and fail to send the icmp6 packet too big error. Just send
10855 	 * it here, before we do any NAT.
10856 	 */
10857 	if (af == AF_INET6 && dir == PF_OUT && pflags & PFIL_FWD &&
10858 	    IN6_LINKMTU(ifp) < pf_max_frag_size(*m0)) {
10859 		PF_RULES_RUNLOCK();
10860 		icmp6_error(*m0, ICMP6_PACKET_TOO_BIG, 0, IN6_LINKMTU(ifp));
10861 		*m0 = NULL;
10862 		return (PF_DROP);
10863 	}
10864 #endif /* INET6 */
10865 
10866 	if (__predict_false(ip_divert_ptr != NULL) &&
10867 	    ((mtag = m_tag_locate(pd.m, MTAG_PF_DIVERT, 0, NULL)) != NULL)) {
10868 		struct pf_divert_mtag *dt = (struct pf_divert_mtag *)(mtag+1);
10869 		if ((dt->idir == PF_DIVERT_MTAG_DIR_IN && dir == PF_IN) ||
10870 		    (dt->idir == PF_DIVERT_MTAG_DIR_OUT && dir == PF_OUT)) {
10871 			if (pd.pf_mtag == NULL &&
10872 			    ((pd.pf_mtag = pf_get_mtag(pd.m)) == NULL)) {
10873 				action = PF_DROP;
10874 				goto done;
10875 			}
10876 			pd.pf_mtag->flags |= PF_MTAG_FLAG_PACKET_LOOPED;
10877 		}
10878 		if (pd.pf_mtag && pd.pf_mtag->flags & PF_MTAG_FLAG_FASTFWD_OURS_PRESENT) {
10879 			pd.m->m_flags |= M_FASTFWD_OURS;
10880 			pd.pf_mtag->flags &= ~PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
10881 		}
10882 		m_tag_delete(pd.m, mtag);
10883 
10884 		mtag = m_tag_locate(pd.m, MTAG_IPFW_RULE, 0, NULL);
10885 		if (mtag != NULL)
10886 			m_tag_delete(pd.m, mtag);
10887 	}
10888 
10889 	switch (pd.virtual_proto) {
10890 	case PF_VPROTO_FRAGMENT:
10891 		/*
10892 		 * handle fragments that aren't reassembled by
10893 		 * normalization
10894 		 */
10895 		if (kif == NULL || r == NULL) /* pflog */
10896 			action = PF_DROP;
10897 		else
10898 			action = pf_test_rule(&r, &s, &pd, &a,
10899 			    &ruleset, &reason, inp);
10900 		if (action != PF_PASS)
10901 			REASON_SET(&reason, PFRES_FRAG);
10902 		break;
10903 
10904 	case IPPROTO_TCP: {
10905 		/* Respond to SYN with a syncookie. */
10906 		if ((tcp_get_flags(&pd.hdr.tcp) & (TH_SYN|TH_ACK|TH_RST)) == TH_SYN &&
10907 		    pd.dir == PF_IN && pf_synflood_check(&pd)) {
10908 			pf_syncookie_send(&pd);
10909 			action = PF_DROP;
10910 			break;
10911 		}
10912 
10913 		if ((tcp_get_flags(&pd.hdr.tcp) & TH_ACK) && pd.p_len == 0)
10914 			use_2nd_queue = 1;
10915 		action = pf_normalize_tcp(&pd);
10916 		if (action == PF_DROP)
10917 			break;
10918 		action = pf_test_state(&s, &pd, &reason);
10919 		if (action == PF_PASS || action == PF_AFRT) {
10920 			if (V_pfsync_update_state_ptr != NULL)
10921 				V_pfsync_update_state_ptr(s);
10922 			r = s->rule;
10923 			a = s->anchor;
10924 		} else if (s == NULL) {
10925 			/* Validate remote SYN|ACK, re-create original SYN if
10926 			 * valid. */
10927 			if ((tcp_get_flags(&pd.hdr.tcp) & (TH_SYN|TH_ACK|TH_RST)) ==
10928 			    TH_ACK && pf_syncookie_validate(&pd) &&
10929 			    pd.dir == PF_IN) {
10930 				struct mbuf *msyn;
10931 
10932 				msyn = pf_syncookie_recreate_syn(&pd);
10933 				if (msyn == NULL) {
10934 					action = PF_DROP;
10935 					break;
10936 				}
10937 
10938 				action = pf_test(af, dir, pflags, ifp, &msyn, inp,
10939 				    &pd.act);
10940 				m_freem(msyn);
10941 				if (action != PF_PASS)
10942 					break;
10943 
10944 				action = pf_test_state(&s, &pd, &reason);
10945 				if (action != PF_PASS || s == NULL) {
10946 					action = PF_DROP;
10947 					break;
10948 				}
10949 
10950 				s->src.seqhi = ntohl(pd.hdr.tcp.th_ack) - 1;
10951 				s->src.seqlo = ntohl(pd.hdr.tcp.th_seq) - 1;
10952 				pf_set_protostate(s, PF_PEER_SRC, PF_TCPS_PROXY_DST);
10953 				action = pf_synproxy(&pd, s, &reason);
10954 				break;
10955 			} else {
10956 				action = pf_test_rule(&r, &s, &pd,
10957 				    &a, &ruleset, &reason, inp);
10958 			}
10959 		}
10960 		break;
10961 	}
10962 
10963 	case IPPROTO_SCTP:
10964 		action = pf_normalize_sctp(&pd);
10965 		if (action == PF_DROP)
10966 			break;
10967 		/* fallthrough */
10968 	case IPPROTO_UDP:
10969 	default:
10970 		action = pf_test_state(&s, &pd, &reason);
10971 		if (action == PF_PASS || action == PF_AFRT) {
10972 			if (V_pfsync_update_state_ptr != NULL)
10973 				V_pfsync_update_state_ptr(s);
10974 			r = s->rule;
10975 			a = s->anchor;
10976 		} else if (s == NULL) {
10977 			action = pf_test_rule(&r, &s,
10978 			    &pd, &a, &ruleset, &reason, inp);
10979 		}
10980 		break;
10981 
10982 	case IPPROTO_ICMP:
10983 	case IPPROTO_ICMPV6: {
10984 		if (pd.virtual_proto == IPPROTO_ICMP && af != AF_INET) {
10985 			action = PF_DROP;
10986 			REASON_SET(&reason, PFRES_NORM);
10987 			DPFPRINTF(PF_DEBUG_MISC,
10988 			    "dropping IPv6 packet with ICMPv4 payload");
10989 			break;
10990 		}
10991 		if (pd.virtual_proto == IPPROTO_ICMPV6 && af != AF_INET6) {
10992 			action = PF_DROP;
10993 			REASON_SET(&reason, PFRES_NORM);
10994 			DPFPRINTF(PF_DEBUG_MISC,
10995 			    "pf: dropping IPv4 packet with ICMPv6 payload");
10996 			break;
10997 		}
10998 		action = pf_test_state_icmp(&s, &pd, &reason);
10999 		if (action == PF_PASS || action == PF_AFRT) {
11000 			if (V_pfsync_update_state_ptr != NULL)
11001 				V_pfsync_update_state_ptr(s);
11002 			r = s->rule;
11003 			a = s->anchor;
11004 		} else if (s == NULL)
11005 			action = pf_test_rule(&r, &s, &pd,
11006 			    &a, &ruleset, &reason, inp);
11007 		break;
11008 	}
11009 
11010 	}
11011 
11012 done:
11013 	PF_RULES_RUNLOCK();
11014 
11015 	if (pd.m == NULL)
11016 		goto eat_pkt;
11017 
11018 	if (s)
11019 		memcpy(&pd.act, &s->act, sizeof(s->act));
11020 
11021 	if (action == PF_PASS && pd.badopts != 0 && !pd.act.allow_opts) {
11022 		action = PF_DROP;
11023 		REASON_SET(&reason, PFRES_IPOPTIONS);
11024 		pd.act.log = PF_LOG_FORCE;
11025 		DPFPRINTF(PF_DEBUG_MISC,
11026 		    "pf: dropping packet with dangerous headers");
11027 	}
11028 
11029 	if (pd.act.max_pkt_size && pd.act.max_pkt_size &&
11030 	    pd.tot_len > pd.act.max_pkt_size) {
11031 		action = PF_DROP;
11032 		REASON_SET(&reason, PFRES_NORM);
11033 		pd.act.log = PF_LOG_FORCE;
11034 		DPFPRINTF(PF_DEBUG_MISC,
11035 		    "pf: dropping overly long packet");
11036 	}
11037 
11038 	if (s) {
11039 		uint8_t log = pd.act.log;
11040 		memcpy(&pd.act, &s->act, sizeof(struct pf_rule_actions));
11041 		pd.act.log |= log;
11042 		tag = s->tag;
11043 	} else {
11044 		tag = r->tag;
11045 	}
11046 
11047 	if (tag > 0 && pf_tag_packet(&pd, tag)) {
11048 		action = PF_DROP;
11049 		REASON_SET(&reason, PFRES_MEMORY);
11050 	}
11051 
11052 	pf_scrub(&pd);
11053 	if (pd.proto == IPPROTO_TCP && pd.act.max_mss)
11054 		pf_normalize_mss(&pd);
11055 
11056 	if (pd.act.rtableid >= 0)
11057 		M_SETFIB(pd.m, pd.act.rtableid);
11058 
11059 	if (pd.act.flags & PFSTATE_SETPRIO) {
11060 		if (pd.tos & IPTOS_LOWDELAY)
11061 			use_2nd_queue = 1;
11062 		if (vlan_set_pcp(pd.m, pd.act.set_prio[use_2nd_queue])) {
11063 			action = PF_DROP;
11064 			REASON_SET(&reason, PFRES_MEMORY);
11065 			pd.act.log = PF_LOG_FORCE;
11066 			DPFPRINTF(PF_DEBUG_MISC,
11067 			    "pf: failed to allocate 802.1q mtag");
11068 		}
11069 	}
11070 
11071 #ifdef ALTQ
11072 	if (action == PF_PASS && pd.act.qid) {
11073 		if (pd.pf_mtag == NULL &&
11074 		    ((pd.pf_mtag = pf_get_mtag(pd.m)) == NULL)) {
11075 			action = PF_DROP;
11076 			REASON_SET(&reason, PFRES_MEMORY);
11077 		} else {
11078 			if (s != NULL)
11079 				pd.pf_mtag->qid_hash = pf_state_hash(s);
11080 			if (use_2nd_queue || (pd.tos & IPTOS_LOWDELAY))
11081 				pd.pf_mtag->qid = pd.act.pqid;
11082 			else
11083 				pd.pf_mtag->qid = pd.act.qid;
11084 			/* Add hints for ecn. */
11085 			pd.pf_mtag->hdr = mtod(pd.m, void *);
11086 		}
11087 	}
11088 #endif /* ALTQ */
11089 
11090 	/*
11091 	 * connections redirected to loopback should not match sockets
11092 	 * bound specifically to loopback due to security implications,
11093 	 * see tcp_input() and in_pcblookup_listen().
11094 	 */
11095 	if (dir == PF_IN && action == PF_PASS && (pd.proto == IPPROTO_TCP ||
11096 	    pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule != NULL &&
11097 	    (s->nat_rule->action == PF_RDR ||
11098 	    s->nat_rule->action == PF_BINAT) &&
11099 	    pf_is_loopback(af, pd.dst))
11100 		pd.m->m_flags |= M_SKIP_FIREWALL;
11101 
11102 	if (af == AF_INET && __predict_false(ip_divert_ptr != NULL) &&
11103 	    action == PF_PASS && r->divert.port && !PACKET_LOOPED(&pd)) {
11104 		mtag = m_tag_alloc(MTAG_PF_DIVERT, 0,
11105 		    sizeof(struct pf_divert_mtag), M_NOWAIT | M_ZERO);
11106 		if (mtag != NULL) {
11107 			((struct pf_divert_mtag *)(mtag+1))->port =
11108 			    ntohs(r->divert.port);
11109 			((struct pf_divert_mtag *)(mtag+1))->idir =
11110 			    (dir == PF_IN) ? PF_DIVERT_MTAG_DIR_IN :
11111 			    PF_DIVERT_MTAG_DIR_OUT;
11112 
11113 			if (s)
11114 				PF_STATE_UNLOCK(s);
11115 
11116 			m_tag_prepend(pd.m, mtag);
11117 			if (pd.m->m_flags & M_FASTFWD_OURS) {
11118 				if (pd.pf_mtag == NULL &&
11119 				    ((pd.pf_mtag = pf_get_mtag(pd.m)) == NULL)) {
11120 					action = PF_DROP;
11121 					REASON_SET(&reason, PFRES_MEMORY);
11122 					pd.act.log = PF_LOG_FORCE;
11123 					DPFPRINTF(PF_DEBUG_MISC,
11124 					    "pf: failed to allocate tag");
11125 				} else {
11126 					pd.pf_mtag->flags |=
11127 					    PF_MTAG_FLAG_FASTFWD_OURS_PRESENT;
11128 					pd.m->m_flags &= ~M_FASTFWD_OURS;
11129 				}
11130 			}
11131 			ip_divert_ptr(*m0, dir == PF_IN);
11132 			*m0 = NULL;
11133 
11134 			return (action);
11135 		} else {
11136 			/* XXX: ipfw has the same behaviour! */
11137 			action = PF_DROP;
11138 			REASON_SET(&reason, PFRES_MEMORY);
11139 			pd.act.log = PF_LOG_FORCE;
11140 			DPFPRINTF(PF_DEBUG_MISC,
11141 			    "pf: failed to allocate divert tag");
11142 		}
11143 	}
11144 	/* XXX: Anybody working on it?! */
11145 	if (af == AF_INET6 && r->divert.port)
11146 		printf("pf: divert(9) is not supported for IPv6\n");
11147 
11148 	/* this flag will need revising if the pkt is forwarded */
11149 	if (pd.pf_mtag)
11150 		pd.pf_mtag->flags &= ~PF_MTAG_FLAG_PACKET_LOOPED;
11151 
11152 	if (pd.act.log) {
11153 		struct pf_krule		*lr;
11154 		struct pf_krule_item	*ri;
11155 
11156 		if (s != NULL && s->nat_rule != NULL &&
11157 		    s->nat_rule->log & PF_LOG_ALL)
11158 			lr = s->nat_rule;
11159 		else
11160 			lr = r;
11161 
11162 		if (pd.act.log & PF_LOG_FORCE || lr->log & PF_LOG_ALL)
11163 			PFLOG_PACKET(action, reason, lr, a,
11164 			    ruleset, &pd, (s == NULL), NULL);
11165 		if (s) {
11166 			SLIST_FOREACH(ri, &s->match_rules, entry)
11167 				if (ri->r->log & PF_LOG_ALL)
11168 					PFLOG_PACKET(action,
11169 					    reason, ri->r, a, ruleset, &pd, 0, NULL);
11170 		}
11171 	}
11172 
11173 	pf_counters_inc(action, &pd, s, r, a);
11174 
11175 	switch (action) {
11176 	case PF_SYNPROXY_DROP:
11177 		m_freem(*m0);
11178 	case PF_DEFER:
11179 		*m0 = NULL;
11180 		action = PF_PASS;
11181 		break;
11182 	case PF_DROP:
11183 		m_freem(*m0);
11184 		*m0 = NULL;
11185 		break;
11186 	case PF_AFRT:
11187 		if (pf_translate_af(&pd)) {
11188 			*m0 = pd.m;
11189 			action = PF_DROP;
11190 			break;
11191 		}
11192 #ifdef INET
11193 		if (pd.naf == AF_INET) {
11194 			action = pf_route(r, kif->pfik_ifp, s, &pd,
11195 			    inp);
11196 		}
11197 #endif /* INET */
11198 #ifdef INET6
11199 		if (pd.naf == AF_INET6) {
11200 			action = pf_route6(r, kif->pfik_ifp, s, &pd,
11201 			    inp);
11202 }
11203 #endif /* INET6 */
11204 		*m0 = pd.m;
11205 		goto out;
11206 		break;
11207 	default:
11208 		if (pd.act.rt) {
11209 			switch (af) {
11210 #ifdef INET
11211 			case AF_INET:
11212 				/* pf_route() returns unlocked. */
11213 				action = pf_route(r, kif->pfik_ifp, s, &pd,
11214 				    inp);
11215 				break;
11216 #endif /* INET */
11217 #ifdef INET6
11218 			case AF_INET6:
11219 				/* pf_route6() returns unlocked. */
11220 				action = pf_route6(r, kif->pfik_ifp, s, &pd,
11221 				    inp);
11222 				break;
11223 #endif /* INET6 */
11224 			}
11225 			*m0 = pd.m;
11226 			goto out;
11227 		}
11228 		if (pf_dummynet(&pd, s, r, m0) != 0) {
11229 			action = PF_DROP;
11230 			REASON_SET(&reason, PFRES_MEMORY);
11231 		}
11232 		break;
11233 	}
11234 
11235 eat_pkt:
11236 	SDT_PROBE4(pf, ip, test, done, action, reason, r, s);
11237 
11238 	if (s && action != PF_DROP) {
11239 		if (!s->if_index_in && dir == PF_IN)
11240 			s->if_index_in = ifp->if_index;
11241 		else if (!s->if_index_out && dir == PF_OUT)
11242 			s->if_index_out = ifp->if_index;
11243 	}
11244 
11245 	if (s)
11246 		PF_STATE_UNLOCK(s);
11247 
11248 out:
11249 #ifdef INET6
11250 	/* If reassembled packet passed, create new fragments. */
11251 	if (af == AF_INET6 && action == PF_PASS && *m0 && dir == PF_OUT &&
11252 	    (! (pflags & PF_PFIL_NOREFRAGMENT)) &&
11253 	    (mtag = m_tag_find(pd.m, PACKET_TAG_PF_REASSEMBLED, NULL)) != NULL)
11254 		action = pf_refragment6(ifp, m0, mtag, NULL, pflags & PFIL_FWD);
11255 #endif /* INET6 */
11256 
11257 	pf_sctp_multihome_delayed(&pd, kif, s, action);
11258 
11259 	return (action);
11260 }
11261 #endif /* INET || INET6 */
11262