xref: /freebsd/sys/netipsec/ipsec_output.c (revision c7a063741720ef81d4caa4613242579d12f1d605)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
5  * Copyright (c) 2016 Andrey V. Elsukov <ae@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 /*
33  * IPsec output processing.
34  */
35 #include "opt_inet.h"
36 #include "opt_inet6.h"
37 #include "opt_ipsec.h"
38 #include "opt_sctp.h"
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/mbuf.h>
43 #include <sys/domain.h>
44 #include <sys/protosw.h>
45 #include <sys/socket.h>
46 #include <sys/errno.h>
47 #include <sys/hhook.h>
48 #include <sys/syslog.h>
49 
50 #include <net/if.h>
51 #include <net/if_enc.h>
52 #include <net/if_var.h>
53 #include <net/vnet.h>
54 
55 #include <netinet/in.h>
56 #include <netinet/in_pcb.h>
57 #include <netinet/in_systm.h>
58 #include <netinet/ip.h>
59 #include <netinet/ip_var.h>
60 #include <netinet/in_var.h>
61 #include <netinet/ip_ecn.h>
62 #ifdef INET6
63 #include <netinet6/ip6_ecn.h>
64 #endif
65 #include <netinet/ip_icmp.h>
66 #include <netinet/tcp_var.h>
67 
68 #include <netinet/ip6.h>
69 #ifdef INET6
70 #include <netinet6/ip6_var.h>
71 #include <netinet6/scope6_var.h>
72 #endif
73 #include <netinet/in_pcb.h>
74 #ifdef INET6
75 #include <netinet/icmp6.h>
76 #endif
77 #if defined(SCTP) || defined(SCTP_SUPPORT)
78 #include <netinet/sctp_crc32.h>
79 #endif
80 
81 #include <netinet/udp.h>
82 #include <netipsec/ah.h>
83 #include <netipsec/esp.h>
84 #include <netipsec/ipsec.h>
85 #ifdef INET6
86 #include <netipsec/ipsec6.h>
87 #endif
88 #include <netipsec/ipsec_support.h>
89 #include <netipsec/ah_var.h>
90 #include <netipsec/esp_var.h>
91 #include <netipsec/ipcomp_var.h>
92 
93 #include <netipsec/xform.h>
94 
95 #include <netipsec/key.h>
96 #include <netipsec/keydb.h>
97 #include <netipsec/key_debug.h>
98 
99 #include <machine/in_cksum.h>
100 
101 #define	IPSEC_OSTAT_INC(proto, name)	do {		\
102 	if ((proto) == IPPROTO_ESP)	\
103 		ESPSTAT_INC(esps_##name);	\
104 	else if ((proto) == IPPROTO_AH)\
105 		AHSTAT_INC(ahs_##name);		\
106 	else					\
107 		IPCOMPSTAT_INC(ipcomps_##name);	\
108 } while (0)
109 
110 static int ipsec_encap(struct mbuf **mp, struct secasindex *saidx);
111 static size_t ipsec_get_pmtu(struct secasvar *sav);
112 
113 #ifdef INET
114 static struct secasvar *
115 ipsec4_allocsa(struct mbuf *m, struct secpolicy *sp, u_int *pidx, int *error)
116 {
117 	struct secasindex *saidx, tmpsaidx;
118 	struct ipsecrequest *isr;
119 	struct sockaddr_in *sin;
120 	struct secasvar *sav;
121 	struct ip *ip;
122 
123 	/*
124 	 * Check system global policy controls.
125 	 */
126 next:
127 	isr = sp->req[*pidx];
128 	if ((isr->saidx.proto == IPPROTO_ESP && !V_esp_enable) ||
129 	    (isr->saidx.proto == IPPROTO_AH && !V_ah_enable) ||
130 	    (isr->saidx.proto == IPPROTO_IPCOMP && !V_ipcomp_enable)) {
131 		DPRINTF(("%s: IPsec outbound packet dropped due"
132 			" to policy (check your sysctls)\n", __func__));
133 		IPSEC_OSTAT_INC(isr->saidx.proto, pdrops);
134 		*error = EHOSTUNREACH;
135 		return (NULL);
136 	}
137 	/*
138 	 * Craft SA index to search for proper SA.  Note that
139 	 * we only initialize unspecified SA peers for transport
140 	 * mode; for tunnel mode they must already be filled in.
141 	 */
142 	if (isr->saidx.mode == IPSEC_MODE_TRANSPORT) {
143 		saidx = &tmpsaidx;
144 		*saidx = isr->saidx;
145 		ip = mtod(m, struct ip *);
146 		if (saidx->src.sa.sa_len == 0) {
147 			sin = &saidx->src.sin;
148 			sin->sin_len = sizeof(*sin);
149 			sin->sin_family = AF_INET;
150 			sin->sin_port = IPSEC_PORT_ANY;
151 			sin->sin_addr = ip->ip_src;
152 		}
153 		if (saidx->dst.sa.sa_len == 0) {
154 			sin = &saidx->dst.sin;
155 			sin->sin_len = sizeof(*sin);
156 			sin->sin_family = AF_INET;
157 			sin->sin_port = IPSEC_PORT_ANY;
158 			sin->sin_addr = ip->ip_dst;
159 		}
160 	} else
161 		saidx = &sp->req[*pidx]->saidx;
162 	/*
163 	 * Lookup SA and validate it.
164 	 */
165 	sav = key_allocsa_policy(sp, saidx, error);
166 	if (sav == NULL) {
167 		IPSECSTAT_INC(ips_out_nosa);
168 		if (*error != 0)
169 			return (NULL);
170 		if (ipsec_get_reqlevel(sp, *pidx) != IPSEC_LEVEL_REQUIRE) {
171 			/*
172 			 * We have no SA and policy that doesn't require
173 			 * this IPsec transform, thus we can continue w/o
174 			 * IPsec processing, i.e. return EJUSTRETURN.
175 			 * But first check if there is some bundled transform.
176 			 */
177 			if (sp->tcount > ++(*pidx))
178 				goto next;
179 			*error = EJUSTRETURN;
180 		}
181 		return (NULL);
182 	}
183 	IPSEC_ASSERT(sav->tdb_xform != NULL, ("SA with NULL tdb_xform"));
184 	return (sav);
185 }
186 
187 /*
188  * IPsec output logic for IPv4.
189  */
190 static int
191 ipsec4_perform_request(struct mbuf *m, struct secpolicy *sp,
192     struct inpcb *inp, u_int idx)
193 {
194 	struct ipsec_ctx_data ctx;
195 	union sockaddr_union *dst;
196 	struct secasvar *sav;
197 	struct ip *ip;
198 	int error, i, off;
199 
200 	IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx));
201 
202 	/*
203 	 * We hold the reference to SP. Content of SP couldn't be changed.
204 	 * Craft secasindex and do lookup for suitable SA.
205 	 * Then do encapsulation if needed and call xform's output.
206 	 * We need to store SP in the xform callback parameters.
207 	 * In xform callback we will extract SP and it can be used to
208 	 * determine next transform. At the end of transform we can
209 	 * release reference to SP.
210 	 */
211 	sav = ipsec4_allocsa(m, sp, &idx, &error);
212 	if (sav == NULL) {
213 		if (error == EJUSTRETURN) { /* No IPsec required */
214 			key_freesp(&sp);
215 			return (error);
216 		}
217 		goto bad;
218 	}
219 	/*
220 	 * XXXAE: most likely ip_sum at this point is wrong.
221 	 */
222 	IPSEC_INIT_CTX(&ctx, &m, inp, sav, AF_INET, IPSEC_ENC_BEFORE);
223 	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
224 		goto bad;
225 
226 	ip = mtod(m, struct ip *);
227 	dst = &sav->sah->saidx.dst;
228 	/* Do the appropriate encapsulation, if necessary */
229 	if (sp->req[idx]->saidx.mode == IPSEC_MODE_TUNNEL || /* Tunnel requ'd */
230 	    dst->sa.sa_family != AF_INET ||	    /* PF mismatch */
231 	    (dst->sa.sa_family == AF_INET &&	    /* Proxy */
232 	     dst->sin.sin_addr.s_addr != INADDR_ANY &&
233 	     dst->sin.sin_addr.s_addr != ip->ip_dst.s_addr)) {
234 		/* Fix IPv4 header checksum and length */
235 		ip->ip_len = htons(m->m_pkthdr.len);
236 		ip->ip_sum = 0;
237 		ip->ip_sum = in_cksum(m, ip->ip_hl << 2);
238 		error = ipsec_encap(&m, &sav->sah->saidx);
239 		if (error != 0) {
240 			DPRINTF(("%s: encapsulation for SPI 0x%08x failed "
241 			    "with error %d\n", __func__, ntohl(sav->spi),
242 			    error));
243 			/* XXXAE: IPSEC_OSTAT_INC(tunnel); */
244 			goto bad;
245 		}
246 		inp = NULL;
247 	}
248 
249 	IPSEC_INIT_CTX(&ctx, &m, inp, sav, dst->sa.sa_family, IPSEC_ENC_AFTER);
250 	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
251 		goto bad;
252 
253 	/*
254 	 * Dispatch to the appropriate IPsec transform logic.  The
255 	 * packet will be returned for transmission after crypto
256 	 * processing, etc. are completed.
257 	 *
258 	 * NB: m & sav are ``passed to caller'' who's responsible for
259 	 *     reclaiming their resources.
260 	 */
261 	switch(dst->sa.sa_family) {
262 	case AF_INET:
263 		ip = mtod(m, struct ip *);
264 		i = ip->ip_hl << 2;
265 		off = offsetof(struct ip, ip_p);
266 		break;
267 #ifdef INET6
268 	case AF_INET6:
269 		i = sizeof(struct ip6_hdr);
270 		off = offsetof(struct ip6_hdr, ip6_nxt);
271 		break;
272 #endif /* INET6 */
273 	default:
274 		DPRINTF(("%s: unsupported protocol family %u\n",
275 		    __func__, dst->sa.sa_family));
276 		error = EPFNOSUPPORT;
277 		IPSEC_OSTAT_INC(sav->sah->saidx.proto, nopf);
278 		goto bad;
279 	}
280 	error = (*sav->tdb_xform->xf_output)(m, sp, sav, idx, i, off);
281 	return (error);
282 bad:
283 	IPSECSTAT_INC(ips_out_inval);
284 	if (m != NULL)
285 		m_freem(m);
286 	if (sav != NULL)
287 		key_freesav(&sav);
288 	key_freesp(&sp);
289 	return (error);
290 }
291 
292 int
293 ipsec4_process_packet(struct mbuf *m, struct secpolicy *sp,
294     struct inpcb *inp)
295 {
296 
297 	return (ipsec4_perform_request(m, sp, inp, 0));
298 }
299 
300 int
301 ipsec4_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
302 {
303 	struct secasvar *sav;
304 	struct ip *ip;
305 	size_t hlen, pmtu;
306 	uint32_t idx;
307 	int error;
308 
309 	/* Don't check PMTU if the frame won't have DF bit set. */
310 	if (!V_ip4_ipsec_dfbit)
311 		return (0);
312 	if (V_ip4_ipsec_dfbit == 1)
313 		goto setdf;
314 
315 	/* V_ip4_ipsec_dfbit > 1 - we will copy it from inner header. */
316 	ip = mtod(m, struct ip *);
317 	if (!(ip->ip_off & htons(IP_DF)))
318 		return (0);
319 
320 setdf:
321 	idx = sp->tcount - 1;
322 	sav = ipsec4_allocsa(m, sp, &idx, &error);
323 	if (sav == NULL) {
324 		key_freesp(&sp);
325 		/*
326 		 * No matching SA was found and SADB_ACQUIRE message was generated.
327 		 * Since we have matched a SP to this packet drop it silently.
328 		 */
329 		if (error == 0)
330 			error = EINPROGRESS;
331 		if (error != EJUSTRETURN)
332 			m_freem(m);
333 
334 		return (error);
335 	}
336 
337 	pmtu = ipsec_get_pmtu(sav);
338 	if (pmtu == 0) {
339 		key_freesav(&sav);
340 		return (0);
341 	}
342 
343 	hlen = ipsec_hdrsiz_internal(sp);
344 	key_freesav(&sav);
345 
346 	if (m_length(m, NULL) + hlen > pmtu) {
347 		/*
348 		 * If we're forwarding generate ICMP message here,
349 		 * so that it contains pmtu subtracted by header size.
350 		 * Set error to EINPROGRESS, in order for the frame
351 		 * to be dropped silently.
352 		 */
353 		if (forwarding) {
354 			if (pmtu > hlen)
355 				icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG,
356 				    0, pmtu - hlen);
357 			else
358 				m_freem(m);
359 
360 			key_freesp(&sp);
361 			return (EINPROGRESS); /* Pretend that we consumed it. */
362 		} else {
363 			m_freem(m);
364 			key_freesp(&sp);
365 			return (EMSGSIZE);
366 		}
367 	}
368 
369 	return (0);
370 }
371 
372 static int
373 ipsec4_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
374 {
375 	struct secpolicy *sp;
376 	int error;
377 
378 	/* Lookup for the corresponding outbound security policy */
379 	sp = ipsec4_checkpolicy(m, inp, &error, !forwarding);
380 	if (sp == NULL) {
381 		if (error == -EINVAL) {
382 			/* Discarded by policy. */
383 			m_freem(m);
384 			return (EACCES);
385 		}
386 		return (0); /* No IPsec required. */
387 	}
388 
389 	/*
390 	 * Usually we have to have tunnel mode IPsec security policy
391 	 * when we are forwarding a packet. Otherwise we could not handle
392 	 * encrypted replies, because they are not destined for us. But
393 	 * some users are doing source address translation for forwarded
394 	 * packets, and thus, even if they are forwarded, the replies will
395 	 * return back to us.
396 	 */
397 	if (!forwarding) {
398 		/*
399 		 * Do delayed checksums now because we send before
400 		 * this is done in the normal processing path.
401 		 */
402 		if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
403 			in_delayed_cksum(m);
404 			m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
405 		}
406 #if defined(SCTP) || defined(SCTP_SUPPORT)
407 		if (m->m_pkthdr.csum_flags & CSUM_SCTP) {
408 			struct ip *ip;
409 
410 			ip = mtod(m, struct ip *);
411 			sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
412 			m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
413 		}
414 #endif
415 	}
416 	/* NB: callee frees mbuf and releases reference to SP */
417 	error = ipsec4_check_pmtu(m, sp, forwarding);
418 	if (error != 0) {
419 		if (error == EJUSTRETURN)
420 			return (0);
421 
422 		return (error);
423 	}
424 
425 	error = ipsec4_process_packet(m, sp, inp);
426 	if (error == EJUSTRETURN) {
427 		/*
428 		 * We had a SP with a level of 'use' and no SA. We
429 		 * will just continue to process the packet without
430 		 * IPsec processing and return without error.
431 		 */
432 		return (0);
433 	}
434 	if (error == 0)
435 		return (EINPROGRESS); /* consumed by IPsec */
436 	return (error);
437 }
438 
439 /*
440  * IPSEC_OUTPUT() method implementation for IPv4.
441  * 0 - no IPsec handling needed
442  * other values - mbuf consumed by IPsec.
443  */
444 int
445 ipsec4_output(struct mbuf *m, struct inpcb *inp)
446 {
447 
448 	/*
449 	 * If the packet is resubmitted to ip_output (e.g. after
450 	 * AH, ESP, etc. processing), there will be a tag to bypass
451 	 * the lookup and related policy checking.
452 	 */
453 	if (m_tag_find(m, PACKET_TAG_IPSEC_OUT_DONE, NULL) != NULL)
454 		return (0);
455 
456 	return (ipsec4_common_output(m, inp, 0));
457 }
458 
459 /*
460  * IPSEC_FORWARD() method implementation for IPv4.
461  * 0 - no IPsec handling needed
462  * other values - mbuf consumed by IPsec.
463  */
464 int
465 ipsec4_forward(struct mbuf *m)
466 {
467 
468 	/*
469 	 * Check if this packet has an active inbound SP and needs to be
470 	 * dropped instead of forwarded.
471 	 */
472 	if (ipsec4_in_reject(m, NULL) != 0) {
473 		m_freem(m);
474 		return (EACCES);
475 	}
476 	return (ipsec4_common_output(m, NULL, 1));
477 }
478 #endif
479 
480 #ifdef INET6
481 static int
482 in6_sa_equal_addrwithscope(const struct sockaddr_in6 *sa,
483     const struct in6_addr *ia)
484 {
485 	struct in6_addr ia2;
486 
487 	if (IN6_IS_SCOPE_LINKLOCAL(&sa->sin6_addr)) {
488 		memcpy(&ia2, &sa->sin6_addr, sizeof(ia2));
489 		ia2.s6_addr16[1] = htons(sa->sin6_scope_id);
490 		return (IN6_ARE_ADDR_EQUAL(ia, &ia2));
491 	}
492 	return (IN6_ARE_ADDR_EQUAL(&sa->sin6_addr, ia));
493 }
494 
495 static struct secasvar *
496 ipsec6_allocsa(struct mbuf *m, struct secpolicy *sp, u_int *pidx, int *error)
497 {
498 	struct secasindex *saidx, tmpsaidx;
499 	struct ipsecrequest *isr;
500 	struct sockaddr_in6 *sin6;
501 	struct secasvar *sav;
502 	struct ip6_hdr *ip6;
503 
504 	/*
505 	 * Check system global policy controls.
506 	 */
507 next:
508 	isr = sp->req[*pidx];
509 	if ((isr->saidx.proto == IPPROTO_ESP && !V_esp_enable) ||
510 	    (isr->saidx.proto == IPPROTO_AH && !V_ah_enable) ||
511 	    (isr->saidx.proto == IPPROTO_IPCOMP && !V_ipcomp_enable)) {
512 		DPRINTF(("%s: IPsec outbound packet dropped due"
513 			" to policy (check your sysctls)\n", __func__));
514 		IPSEC_OSTAT_INC(isr->saidx.proto, pdrops);
515 		*error = EHOSTUNREACH;
516 		return (NULL);
517 	}
518 	/*
519 	 * Craft SA index to search for proper SA.  Note that
520 	 * we only fillin unspecified SA peers for transport
521 	 * mode; for tunnel mode they must already be filled in.
522 	 */
523 	if (isr->saidx.mode == IPSEC_MODE_TRANSPORT) {
524 		saidx = &tmpsaidx;
525 		*saidx = isr->saidx;
526 		ip6 = mtod(m, struct ip6_hdr *);
527 		if (saidx->src.sin6.sin6_len == 0) {
528 			sin6 = (struct sockaddr_in6 *)&saidx->src;
529 			sin6->sin6_len = sizeof(*sin6);
530 			sin6->sin6_family = AF_INET6;
531 			sin6->sin6_port = IPSEC_PORT_ANY;
532 			sin6->sin6_addr = ip6->ip6_src;
533 			if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_src)) {
534 				/* fix scope id for comparing SPD */
535 				sin6->sin6_addr.s6_addr16[1] = 0;
536 				sin6->sin6_scope_id =
537 				    ntohs(ip6->ip6_src.s6_addr16[1]);
538 			}
539 		}
540 		if (saidx->dst.sin6.sin6_len == 0) {
541 			sin6 = (struct sockaddr_in6 *)&saidx->dst;
542 			sin6->sin6_len = sizeof(*sin6);
543 			sin6->sin6_family = AF_INET6;
544 			sin6->sin6_port = IPSEC_PORT_ANY;
545 			sin6->sin6_addr = ip6->ip6_dst;
546 			if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_dst)) {
547 				/* fix scope id for comparing SPD */
548 				sin6->sin6_addr.s6_addr16[1] = 0;
549 				sin6->sin6_scope_id =
550 				    ntohs(ip6->ip6_dst.s6_addr16[1]);
551 			}
552 		}
553 	} else
554 		saidx = &sp->req[*pidx]->saidx;
555 	/*
556 	 * Lookup SA and validate it.
557 	 */
558 	sav = key_allocsa_policy(sp, saidx, error);
559 	if (sav == NULL) {
560 		IPSEC6STAT_INC(ips_out_nosa);
561 		if (*error != 0)
562 			return (NULL);
563 		if (ipsec_get_reqlevel(sp, *pidx) != IPSEC_LEVEL_REQUIRE) {
564 			/*
565 			 * We have no SA and policy that doesn't require
566 			 * this IPsec transform, thus we can continue w/o
567 			 * IPsec processing, i.e. return EJUSTRETURN.
568 			 * But first check if there is some bundled transform.
569 			 */
570 			if (sp->tcount > ++(*pidx))
571 				goto next;
572 			*error = EJUSTRETURN;
573 		}
574 		return (NULL);
575 	}
576 	IPSEC_ASSERT(sav->tdb_xform != NULL, ("SA with NULL tdb_xform"));
577 	return (sav);
578 }
579 
580 /*
581  * IPsec output logic for IPv6.
582  */
583 static int
584 ipsec6_perform_request(struct mbuf *m, struct secpolicy *sp,
585     struct inpcb *inp, u_int idx)
586 {
587 	struct ipsec_ctx_data ctx;
588 	union sockaddr_union *dst;
589 	struct secasvar *sav;
590 	struct ip6_hdr *ip6;
591 	int error, i, off;
592 
593 	IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx));
594 
595 	sav = ipsec6_allocsa(m, sp, &idx, &error);
596 	if (sav == NULL) {
597 		if (error == EJUSTRETURN) { /* No IPsec required */
598 			key_freesp(&sp);
599 			return (error);
600 		}
601 		goto bad;
602 	}
603 
604 	/* Fix IP length in case if it is not set yet. */
605 	ip6 = mtod(m, struct ip6_hdr *);
606 	ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
607 
608 	IPSEC_INIT_CTX(&ctx, &m, inp, sav, AF_INET6, IPSEC_ENC_BEFORE);
609 	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
610 		goto bad;
611 
612 	ip6 = mtod(m, struct ip6_hdr *); /* pfil can change mbuf */
613 	dst = &sav->sah->saidx.dst;
614 
615 	/* Do the appropriate encapsulation, if necessary */
616 	if (sp->req[idx]->saidx.mode == IPSEC_MODE_TUNNEL || /* Tunnel requ'd */
617 	    dst->sa.sa_family != AF_INET6 ||        /* PF mismatch */
618 	    ((dst->sa.sa_family == AF_INET6) &&
619 	     (!IN6_IS_ADDR_UNSPECIFIED(&dst->sin6.sin6_addr)) &&
620 	     (!in6_sa_equal_addrwithscope(&dst->sin6, &ip6->ip6_dst)))) {
621 		if (m->m_pkthdr.len - sizeof(*ip6) > IPV6_MAXPACKET) {
622 			/* No jumbogram support. */
623 			error = ENXIO;   /*XXX*/
624 			goto bad;
625 		}
626 		error = ipsec_encap(&m, &sav->sah->saidx);
627 		if (error != 0) {
628 			DPRINTF(("%s: encapsulation for SPI 0x%08x failed "
629 			    "with error %d\n", __func__, ntohl(sav->spi),
630 			    error));
631 			/* XXXAE: IPSEC_OSTAT_INC(tunnel); */
632 			goto bad;
633 		}
634 		inp = NULL;
635 	}
636 
637 	IPSEC_INIT_CTX(&ctx, &m, inp, sav, dst->sa.sa_family, IPSEC_ENC_AFTER);
638 	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
639 		goto bad;
640 
641 	switch(dst->sa.sa_family) {
642 #ifdef INET
643 	case AF_INET:
644 		{
645 		struct ip *ip;
646 		ip = mtod(m, struct ip *);
647 		i = ip->ip_hl << 2;
648 		off = offsetof(struct ip, ip_p);
649 		}
650 		break;
651 #endif /* AF_INET */
652 	case AF_INET6:
653 		i = sizeof(struct ip6_hdr);
654 		off = offsetof(struct ip6_hdr, ip6_nxt);
655 		break;
656 	default:
657 		DPRINTF(("%s: unsupported protocol family %u\n",
658 				 __func__, dst->sa.sa_family));
659 		error = EPFNOSUPPORT;
660 		IPSEC_OSTAT_INC(sav->sah->saidx.proto, nopf);
661 		goto bad;
662 	}
663 	error = (*sav->tdb_xform->xf_output)(m, sp, sav, idx, i, off);
664 	return (error);
665 bad:
666 	IPSEC6STAT_INC(ips_out_inval);
667 	if (m != NULL)
668 		m_freem(m);
669 	if (sav != NULL)
670 		key_freesav(&sav);
671 	key_freesp(&sp);
672 	return (error);
673 }
674 
675 int
676 ipsec6_process_packet(struct mbuf *m, struct secpolicy *sp,
677     struct inpcb *inp)
678 {
679 
680 	return (ipsec6_perform_request(m, sp, inp, 0));
681 }
682 
683 /*
684  * IPv6 implementation is based on IPv4 implementation.
685  */
686 int
687 ipsec6_check_pmtu(struct mbuf *m, struct secpolicy *sp, int forwarding)
688 {
689 	struct secasvar *sav;
690 	size_t hlen, pmtu;
691 	uint32_t idx;
692 	int error;
693 
694 	/*
695 	 * According to RFC8200 L3 fragmentation is supposed to be done only on
696 	 * locally generated packets. During L3 forwarding packets that are too
697 	 * big are always supposed to be dropped, with an ICMPv6 packet being
698 	 * sent back.
699 	 */
700 	if (!forwarding)
701 		return (0);
702 
703 	idx = sp->tcount - 1;
704 	sav = ipsec6_allocsa(m, sp, &idx, &error);
705 	if (sav == NULL) {
706 		key_freesp(&sp);
707 		/*
708 		 * No matching SA was found and SADB_ACQUIRE message was generated.
709 		 * Since we have matched a SP to this packet drop it silently.
710 		 */
711 		if (error == 0)
712 			error = EINPROGRESS;
713 		if (error != EJUSTRETURN)
714 			m_freem(m);
715 
716 		return (error);
717 	}
718 
719 	pmtu = ipsec_get_pmtu(sav);
720 	if (pmtu == 0) {
721 		key_freesav(&sav);
722 		return (0);
723 	}
724 
725 	hlen = ipsec_hdrsiz_internal(sp);
726 	key_freesav(&sav);
727 
728 	if (m_length(m, NULL) + hlen > pmtu) {
729 		/*
730 		 * If we're forwarding generate ICMPv6 message here,
731 		 * so that it contains pmtu subtracted by header size.
732 		 * Set error to EINPROGRESS, in order for the frame
733 		 * to be dropped silently.
734 		 */
735 		if (forwarding) {
736 			if (pmtu > hlen)
737 				icmp6_error(m, ICMP6_PACKET_TOO_BIG, 0, pmtu - hlen);
738 			else
739 				m_freem(m);
740 
741 			key_freesp(&sp);
742 			return (EINPROGRESS); /* Pretend that we consumed it. */
743 		}
744 	}
745 
746 	return (0);
747 }
748 
749 static int
750 ipsec6_common_output(struct mbuf *m, struct inpcb *inp, int forwarding)
751 {
752 	struct secpolicy *sp;
753 	int error;
754 
755 	/* Lookup for the corresponding outbound security policy */
756 	sp = ipsec6_checkpolicy(m, inp, &error, !forwarding);
757 	if (sp == NULL) {
758 		if (error == -EINVAL) {
759 			/* Discarded by policy. */
760 			m_freem(m);
761 			return (EACCES);
762 		}
763 		return (0); /* No IPsec required. */
764 	}
765 
766 	if (!forwarding) {
767 		/*
768 		 * Do delayed checksums now because we send before
769 		 * this is done in the normal processing path.
770 		 */
771 		if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
772 			in6_delayed_cksum(m, m->m_pkthdr.len -
773 			    sizeof(struct ip6_hdr), sizeof(struct ip6_hdr));
774 			m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
775 		}
776 #if defined(SCTP) || defined(SCTP_SUPPORT)
777 		if (m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6) {
778 			sctp_delayed_cksum(m, sizeof(struct ip6_hdr));
779 			m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
780 		}
781 #endif
782 	}
783 
784 	error = ipsec6_check_pmtu(m, sp, forwarding);
785 	if (error != 0) {
786 		if (error == EJUSTRETURN)
787 			return (0);
788 
789 		return (error);
790 	}
791 
792 	/* NB: callee frees mbuf and releases reference to SP */
793 	error = ipsec6_process_packet(m, sp, inp);
794 	if (error == EJUSTRETURN) {
795 		/*
796 		 * We had a SP with a level of 'use' and no SA. We
797 		 * will just continue to process the packet without
798 		 * IPsec processing and return without error.
799 		 */
800 		return (0);
801 	}
802 	if (error == 0)
803 		return (EINPROGRESS); /* consumed by IPsec */
804 	return (error);
805 }
806 
807 /*
808  * IPSEC_OUTPUT() method implementation for IPv6.
809  * 0 - no IPsec handling needed
810  * other values - mbuf consumed by IPsec.
811  */
812 int
813 ipsec6_output(struct mbuf *m, struct inpcb *inp)
814 {
815 
816 	/*
817 	 * If the packet is resubmitted to ip_output (e.g. after
818 	 * AH, ESP, etc. processing), there will be a tag to bypass
819 	 * the lookup and related policy checking.
820 	 */
821 	if (m_tag_find(m, PACKET_TAG_IPSEC_OUT_DONE, NULL) != NULL)
822 		return (0);
823 
824 	return (ipsec6_common_output(m, inp, 0));
825 }
826 
827 /*
828  * IPSEC_FORWARD() method implementation for IPv6.
829  * 0 - no IPsec handling needed
830  * other values - mbuf consumed by IPsec.
831  */
832 int
833 ipsec6_forward(struct mbuf *m)
834 {
835 
836 	/*
837 	 * Check if this packet has an active inbound SP and needs to be
838 	 * dropped instead of forwarded.
839 	 */
840 	if (ipsec6_in_reject(m, NULL) != 0) {
841 		m_freem(m);
842 		return (EACCES);
843 	}
844 	return (ipsec6_common_output(m, NULL, 1));
845 }
846 #endif /* INET6 */
847 
848 int
849 ipsec_process_done(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
850     u_int idx)
851 {
852 	struct epoch_tracker et;
853 	struct xform_history *xh;
854 	struct secasindex *saidx;
855 	struct m_tag *mtag;
856 	int error;
857 
858 	saidx = &sav->sah->saidx;
859 	switch (saidx->dst.sa.sa_family) {
860 #ifdef INET
861 	case AF_INET:
862 		/* Fix the header length, for AH processing. */
863 		mtod(m, struct ip *)->ip_len = htons(m->m_pkthdr.len);
864 		break;
865 #endif /* INET */
866 #ifdef INET6
867 	case AF_INET6:
868 		/* Fix the header length, for AH processing. */
869 		if (m->m_pkthdr.len < sizeof (struct ip6_hdr)) {
870 			error = ENXIO;
871 			goto bad;
872 		}
873 		if (m->m_pkthdr.len - sizeof (struct ip6_hdr) > IPV6_MAXPACKET) {
874 			/* No jumbogram support. */
875 			error = ENXIO;	/*?*/
876 			goto bad;
877 		}
878 		mtod(m, struct ip6_hdr *)->ip6_plen =
879 			htons(m->m_pkthdr.len - sizeof(struct ip6_hdr));
880 		break;
881 #endif /* INET6 */
882 	default:
883 		DPRINTF(("%s: unknown protocol family %u\n", __func__,
884 		    saidx->dst.sa.sa_family));
885 		error = ENXIO;
886 		goto bad;
887 	}
888 
889 	/*
890 	 * Add a record of what we've done to the packet.
891 	 */
892 	mtag = m_tag_get(PACKET_TAG_IPSEC_OUT_DONE, sizeof(*xh), M_NOWAIT);
893 	if (mtag == NULL) {
894 		DPRINTF(("%s: could not get packet tag\n", __func__));
895 		error = ENOMEM;
896 		goto bad;
897 	}
898 
899 	xh = (struct xform_history *)(mtag + 1);
900 	xh->dst = saidx->dst;
901 	xh->proto = saidx->proto;
902 	xh->mode = saidx->mode;
903 	xh->spi = sav->spi;
904 	m_tag_prepend(m, mtag);
905 
906 	key_sa_recordxfer(sav, m);		/* record data transfer */
907 
908 	/*
909 	 * If there's another (bundled) SA to apply, do so.
910 	 * Note that this puts a burden on the kernel stack size.
911 	 * If this is a problem we'll need to introduce a queue
912 	 * to set the packet on so we can unwind the stack before
913 	 * doing further processing.
914 	 */
915 	if (++idx < sp->tcount) {
916 		switch (saidx->dst.sa.sa_family) {
917 #ifdef INET
918 		case AF_INET:
919 			key_freesav(&sav);
920 			IPSECSTAT_INC(ips_out_bundlesa);
921 			return (ipsec4_perform_request(m, sp, NULL, idx));
922 			/* NOTREACHED */
923 #endif
924 #ifdef INET6
925 		case AF_INET6:
926 			key_freesav(&sav);
927 			IPSEC6STAT_INC(ips_out_bundlesa);
928 			return (ipsec6_perform_request(m, sp, NULL, idx));
929 			/* NOTREACHED */
930 #endif /* INET6 */
931 		default:
932 			DPRINTF(("%s: unknown protocol family %u\n", __func__,
933 			    saidx->dst.sa.sa_family));
934 			error = EPFNOSUPPORT;
935 			goto bad;
936 		}
937 	}
938 
939 	key_freesp(&sp), sp = NULL;	/* Release reference to SP */
940 #ifdef INET
941 	/*
942 	 * Do UDP encapsulation if SA requires it.
943 	 */
944 	if (sav->natt != NULL) {
945 		error = udp_ipsec_output(m, sav);
946 		if (error != 0)
947 			goto bad;
948 	}
949 #endif /* INET */
950 	/*
951 	 * We're done with IPsec processing, transmit the packet using the
952 	 * appropriate network protocol (IP or IPv6).
953 	 */
954 	NET_EPOCH_ENTER(et);
955 	switch (saidx->dst.sa.sa_family) {
956 #ifdef INET
957 	case AF_INET:
958 		key_freesav(&sav);
959 		error = ip_output(m, NULL, NULL, IP_RAWOUTPUT, NULL, NULL);
960 		break;
961 #endif /* INET */
962 #ifdef INET6
963 	case AF_INET6:
964 		key_freesav(&sav);
965 		error = ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
966 		break;
967 #endif /* INET6 */
968 	default:
969 		panic("ipsec_process_done");
970 	}
971 	NET_EPOCH_EXIT(et);
972 	return (error);
973 bad:
974 	m_freem(m);
975 	key_freesav(&sav);
976 	if (sp != NULL)
977 		key_freesp(&sp);
978 	return (error);
979 }
980 
981 /*
982  * ipsec_prepend() is optimized version of M_PREPEND().
983  * ipsec_encap() is called by IPsec output routine for tunnel mode SA.
984  * It is expected that after IP encapsulation some IPsec transform will
985  * be performed. Each IPsec transform inserts its variable length header
986  * just after outer IP header using m_makespace(). If given mbuf has not
987  * enough free space at the beginning, we allocate new mbuf and reserve
988  * some space at the beginning and at the end.
989  * This helps avoid allocating of new mbuf and data copying in m_makespace(),
990  * we place outer header in the middle of mbuf's data with reserved leading
991  * and trailing space:
992  *	[ LEADINGSPACE ][ Outer IP header ][ TRAILINGSPACE ]
993  * LEADINGSPACE will be used to add ethernet header, TRAILINGSPACE will
994  * be used to inject AH/ESP/IPCOMP header.
995  */
996 #define	IPSEC_TRAILINGSPACE	(sizeof(struct udphdr) +/* NAT-T */	\
997     max(sizeof(struct newesp) + EALG_MAX_BLOCK_LEN,	/* ESP + IV */	\
998 	sizeof(struct newah) + HASH_MAX_LEN		/* AH + ICV */))
999 static struct mbuf *
1000 ipsec_prepend(struct mbuf *m, int len, int how)
1001 {
1002 	struct mbuf *n;
1003 
1004 	M_ASSERTPKTHDR(m);
1005 	IPSEC_ASSERT(len < MHLEN, ("wrong length"));
1006 	if (M_LEADINGSPACE(m) >= len) {
1007 		/* No need to allocate new mbuf. */
1008 		m->m_data -= len;
1009 		m->m_len += len;
1010 		m->m_pkthdr.len += len;
1011 		return (m);
1012 	}
1013 	n = m_gethdr(how, m->m_type);
1014 	if (n == NULL) {
1015 		m_freem(m);
1016 		return (NULL);
1017 	}
1018 	m_move_pkthdr(n, m);
1019 	n->m_next = m;
1020 	if (len + IPSEC_TRAILINGSPACE < M_SIZE(n))
1021 		m_align(n, len + IPSEC_TRAILINGSPACE);
1022 	n->m_len = len;
1023 	n->m_pkthdr.len += len;
1024 	return (n);
1025 }
1026 
1027 static size_t
1028 ipsec_get_pmtu(struct secasvar *sav)
1029 {
1030 	union sockaddr_union *dst;
1031 	struct in_conninfo inc;
1032 	size_t pmtu;
1033 
1034 	dst = &sav->sah->saidx.dst;
1035 	memset(&inc, 0, sizeof(inc));
1036 
1037 	switch (dst->sa.sa_family) {
1038 #ifdef INET
1039 	case AF_INET:
1040 		inc.inc_faddr = satosin(&dst->sa)->sin_addr;
1041 		break;
1042 #endif
1043 #ifdef INET6
1044 	case AF_INET6:
1045 		inc.inc6_faddr = satosin6(&dst->sa)->sin6_addr;
1046 		inc.inc_flags |= INC_ISIPV6;
1047 		break;
1048 #endif
1049 	default:
1050 		return (0);
1051 	}
1052 
1053 	pmtu = tcp_hc_getmtu(&inc);
1054 	if (pmtu != 0)
1055 		return (pmtu);
1056 
1057 	/* No entry in hostcache. Assume that PMTU is equal to link's MTU */
1058 	switch (dst->sa.sa_family) {
1059 #ifdef INET
1060 	case AF_INET:
1061 		pmtu = tcp_maxmtu(&inc, NULL);
1062 		break;
1063 #endif
1064 #ifdef INET6
1065 	case AF_INET6:
1066 		pmtu = tcp_maxmtu6(&inc, NULL);
1067 		break;
1068 #endif
1069 	default:
1070 		return (0);
1071 	}
1072 	if (pmtu == 0)
1073 		return (0);
1074 
1075 	tcp_hc_updatemtu(&inc, pmtu);
1076 
1077 	return (pmtu);
1078 }
1079 
1080 static int
1081 ipsec_encap(struct mbuf **mp, struct secasindex *saidx)
1082 {
1083 #ifdef INET6
1084 	struct ip6_hdr *ip6;
1085 #endif
1086 	struct ip *ip;
1087 #ifdef INET
1088 	int setdf;
1089 #endif
1090 	uint8_t itos, proto;
1091 
1092 	ip = mtod(*mp, struct ip *);
1093 	switch (ip->ip_v) {
1094 #ifdef INET
1095 	case IPVERSION:
1096 		proto = IPPROTO_IPIP;
1097 		/*
1098 		 * Collect IP_DF state from the inner header
1099 		 * and honor system-wide control of how to handle it.
1100 		 */
1101 		switch (V_ip4_ipsec_dfbit) {
1102 		case 0:	/* clear in outer header */
1103 		case 1:	/* set in outer header */
1104 			setdf = V_ip4_ipsec_dfbit;
1105 			break;
1106 		default:/* propagate to outer header */
1107 			setdf = (ip->ip_off & htons(IP_DF)) != 0;
1108 		}
1109 		itos = ip->ip_tos;
1110 		break;
1111 #endif
1112 #ifdef INET6
1113 	case (IPV6_VERSION >> 4):
1114 		proto = IPPROTO_IPV6;
1115 		ip6 = mtod(*mp, struct ip6_hdr *);
1116 		itos = (ntohl(ip6->ip6_flow) >> 20) & 0xff;
1117 		/* scoped address handling */
1118 		in6_clearscope(&ip6->ip6_src);
1119 		in6_clearscope(&ip6->ip6_dst);
1120 		break;
1121 #endif
1122 	default:
1123 		return (EAFNOSUPPORT);
1124 	}
1125 	switch (saidx->dst.sa.sa_family) {
1126 #ifdef INET
1127 	case AF_INET:
1128 		if (saidx->src.sa.sa_family != AF_INET ||
1129 		    saidx->src.sin.sin_addr.s_addr == INADDR_ANY ||
1130 		    saidx->dst.sin.sin_addr.s_addr == INADDR_ANY)
1131 			return (EINVAL);
1132 		*mp = ipsec_prepend(*mp, sizeof(struct ip), M_NOWAIT);
1133 		if (*mp == NULL)
1134 			return (ENOBUFS);
1135 		ip = mtod(*mp, struct ip *);
1136 		ip->ip_v = IPVERSION;
1137 		ip->ip_hl = sizeof(struct ip) >> 2;
1138 		ip->ip_p = proto;
1139 		ip->ip_len = htons((*mp)->m_pkthdr.len);
1140 		ip->ip_ttl = V_ip_defttl;
1141 		ip->ip_sum = 0;
1142 		ip->ip_off = setdf ? htons(IP_DF): 0;
1143 		ip->ip_src = saidx->src.sin.sin_addr;
1144 		ip->ip_dst = saidx->dst.sin.sin_addr;
1145 		ip_ecn_ingress(V_ip4_ipsec_ecn, &ip->ip_tos, &itos);
1146 		ip_fillid(ip);
1147 		break;
1148 #endif /* INET */
1149 #ifdef INET6
1150 	case AF_INET6:
1151 		if (saidx->src.sa.sa_family != AF_INET6 ||
1152 		    IN6_IS_ADDR_UNSPECIFIED(&saidx->src.sin6.sin6_addr) ||
1153 		    IN6_IS_ADDR_UNSPECIFIED(&saidx->dst.sin6.sin6_addr))
1154 			return (EINVAL);
1155 		*mp = ipsec_prepend(*mp, sizeof(struct ip6_hdr), M_NOWAIT);
1156 		if (*mp == NULL)
1157 			return (ENOBUFS);
1158 		ip6 = mtod(*mp, struct ip6_hdr *);
1159 		ip6->ip6_flow = 0;
1160 		ip6->ip6_vfc = IPV6_VERSION;
1161 		ip6->ip6_hlim = V_ip6_defhlim;
1162 		ip6->ip6_nxt = proto;
1163 		ip6->ip6_dst = saidx->dst.sin6.sin6_addr;
1164 		/* For link-local address embed scope zone id */
1165 		if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_dst))
1166 			ip6->ip6_dst.s6_addr16[1] =
1167 			    htons(saidx->dst.sin6.sin6_scope_id & 0xffff);
1168 		ip6->ip6_src = saidx->src.sin6.sin6_addr;
1169 		if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_src))
1170 			ip6->ip6_src.s6_addr16[1] =
1171 			    htons(saidx->src.sin6.sin6_scope_id & 0xffff);
1172 		ip6->ip6_plen = htons((*mp)->m_pkthdr.len - sizeof(*ip6));
1173 		ip_ecn_ingress(V_ip6_ipsec_ecn, &proto, &itos);
1174 		ip6->ip6_flow |= htonl((uint32_t)proto << 20);
1175 		break;
1176 #endif /* INET6 */
1177 	default:
1178 		return (EAFNOSUPPORT);
1179 	}
1180 	(*mp)->m_flags &= ~(M_BCAST | M_MCAST);
1181 	return (0);
1182 }
1183