xref: /freebsd/sys/netinet/ip_output.c (revision 2259a03020fe043af123bbc9e749cf7fa1b4346c)
1c398230bSWarner Losh /*-
251369649SPedro F. Giffuni  * SPDX-License-Identifier: BSD-3-Clause
351369649SPedro F. Giffuni  *
4df8bae1dSRodney W. Grimes  * Copyright (c) 1982, 1986, 1988, 1990, 1993
5df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
6df8bae1dSRodney W. Grimes  *
7df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
8df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
9df8bae1dSRodney W. Grimes  * are met:
10df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
11df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
12df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
13df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
14df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
15fbbd9655SWarner Losh  * 3. Neither the name of the University nor the names of its contributors
16df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
17df8bae1dSRodney W. Grimes  *    without specific prior written permission.
18df8bae1dSRodney W. Grimes  *
19df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
30df8bae1dSRodney W. Grimes  *
31df8bae1dSRodney W. Grimes  *	@(#)ip_output.c	8.3 (Berkeley) 1/21/94
32df8bae1dSRodney W. Grimes  */
33df8bae1dSRodney W. Grimes 
344b421e2dSMike Silbersack #include <sys/cdefs.h>
354b421e2dSMike Silbersack __FBSDID("$FreeBSD$");
364b421e2dSMike Silbersack 
375d6d7e75SGleb Smirnoff #include "opt_inet.h"
386a800098SYoshinobu Inoue #include "opt_ipsec.h"
39b2e60773SJohn Baldwin #include "opt_kern_tls.h"
4053dcc544SMike Silbersack #include "opt_mbuf_stress_test.h"
41e440aed9SQing Li #include "opt_mpath.h"
4277a01441SJohn Baldwin #include "opt_ratelimit.h"
4357f60867SMark Johnston #include "opt_route.h"
449c423972SAdrian Chadd #include "opt_rss.h"
4577a01441SJohn Baldwin #include "opt_sctp.h"
46fbd1372aSJoerg Wunsch 
47df8bae1dSRodney W. Grimes #include <sys/param.h>
4826f9a767SRodney W. Grimes #include <sys/systm.h>
49f0f6d643SLuigi Rizzo #include <sys/kernel.h>
50b2e60773SJohn Baldwin #include <sys/ktls.h>
51cc0a3c8cSAndrey V. Elsukov #include <sys/lock.h>
52df8bae1dSRodney W. Grimes #include <sys/malloc.h>
53df8bae1dSRodney W. Grimes #include <sys/mbuf.h>
54acd3428bSRobert Watson #include <sys/priv.h>
55c26fe973SBjoern A. Zeeb #include <sys/proc.h>
56df8bae1dSRodney W. Grimes #include <sys/protosw.h>
57cc0a3c8cSAndrey V. Elsukov #include <sys/rmlock.h>
5857f60867SMark Johnston #include <sys/sdt.h>
59df8bae1dSRodney W. Grimes #include <sys/socket.h>
60df8bae1dSRodney W. Grimes #include <sys/socketvar.h>
619d9edc56SMike Silbersack #include <sys/sysctl.h>
62c26fe973SBjoern A. Zeeb #include <sys/ucred.h>
63df8bae1dSRodney W. Grimes 
64df8bae1dSRodney W. Grimes #include <net/if.h>
6576039bc8SGleb Smirnoff #include <net/if_var.h>
663ef5e21dSQing Li #include <net/if_llatbl.h>
679b932e9eSAndre Oppermann #include <net/netisr.h>
68c21fd232SAndre Oppermann #include <net/pfil.h>
69df8bae1dSRodney W. Grimes #include <net/route.h>
70983066f0SAlexander V. Chernikov #include <net/route/nhop.h>
71b2bdc62aSAdrian Chadd #include <net/rss_config.h>
724b79449eSBjoern A. Zeeb #include <net/vnet.h>
73df8bae1dSRodney W. Grimes 
74df8bae1dSRodney W. Grimes #include <netinet/in.h>
756ca363ebSGleb Smirnoff #include <netinet/in_fib.h>
7657f60867SMark Johnston #include <netinet/in_kdtrace.h>
77df8bae1dSRodney W. Grimes #include <netinet/in_systm.h>
78df8bae1dSRodney W. Grimes #include <netinet/ip.h>
79983066f0SAlexander V. Chernikov #include <netinet/in_fib.h>
80df8bae1dSRodney W. Grimes #include <netinet/in_pcb.h>
819c423972SAdrian Chadd #include <netinet/in_rss.h>
82df8bae1dSRodney W. Grimes #include <netinet/in_var.h>
83df8bae1dSRodney W. Grimes #include <netinet/ip_var.h>
84ef39adf0SAndre Oppermann #include <netinet/ip_options.h>
851fdbfb90STom Jones 
861fdbfb90STom Jones #include <netinet/udp.h>
871fdbfb90STom Jones #include <netinet/udp_var.h>
881fdbfb90STom Jones 
8995033af9SMark Johnston #if defined(SCTP) || defined(SCTP_SUPPORT)
902f4afd21SRandall Stewart #include <netinet/sctp.h>
912f4afd21SRandall Stewart #include <netinet/sctp_crc32.h>
922f4afd21SRandall Stewart #endif
93df8bae1dSRodney W. Grimes 
94fcf59617SAndrey V. Elsukov #include <netipsec/ipsec_support.h>
956a800098SYoshinobu Inoue 
961dfcf0d2SAndre Oppermann #include <machine/in_cksum.h>
971dfcf0d2SAndre Oppermann 
98aed55708SRobert Watson #include <security/mac/mac_framework.h>
99aed55708SRobert Watson 
10053dcc544SMike Silbersack #ifdef MBUF_STRESS_TEST
10105b9d121SBjoern A. Zeeb static int mbuf_frag_size = 0;
1029d9edc56SMike Silbersack SYSCTL_INT(_net_inet_ip, OID_AUTO, mbuf_frag_size, CTLFLAG_RW,
1039d9edc56SMike Silbersack 	&mbuf_frag_size, 0, "Fragment outgoing mbufs to this size");
1049d9edc56SMike Silbersack #endif
1059d9edc56SMike Silbersack 
106331dff07SAlexander V. Chernikov static void	ip_mloopback(struct ifnet *, const struct mbuf *, int);
107afed1b49SDarren Reed 
1088b889dbbSBruce M Simpson extern int in_mcast_loop;
10993e0e116SJulian Elischer extern	struct protosw inetsw[];
11093e0e116SJulian Elischer 
111d9f2a782SErmal Luçi static inline int
11205fc9d78SKristof Provost ip_output_pfil(struct mbuf **mp, struct ifnet *ifp, int flags,
11305fc9d78SKristof Provost     struct inpcb *inp, struct sockaddr_in *dst, int *fibnum, int *error)
114d9f2a782SErmal Luçi {
115d9f2a782SErmal Luçi 	struct m_tag *fwd_tag = NULL;
1168f980c01SMark Johnston 	struct mbuf *m;
117d9f2a782SErmal Luçi 	struct in_addr odst;
118d9f2a782SErmal Luçi 	struct ip *ip;
11905fc9d78SKristof Provost 	int pflags = PFIL_OUT;
12005fc9d78SKristof Provost 
12105fc9d78SKristof Provost 	if (flags & IP_FORWARDING)
12205fc9d78SKristof Provost 		pflags |= PFIL_FWD;
123d9f2a782SErmal Luçi 
1248f980c01SMark Johnston 	m = *mp;
125d9f2a782SErmal Luçi 	ip = mtod(m, struct ip *);
126d9f2a782SErmal Luçi 
127d9f2a782SErmal Luçi 	/* Run through list of hooks for output packets. */
128d9f2a782SErmal Luçi 	odst.s_addr = ip->ip_dst.s_addr;
12905fc9d78SKristof Provost 	switch (pfil_run_hooks(V_inet_pfil_head, mp, ifp, pflags, inp)) {
130b252313fSGleb Smirnoff 	case PFIL_DROPPED:
1318d5c56daSGleb Smirnoff 		*error = EACCES;
132b252313fSGleb Smirnoff 		/* FALLTHROUGH */
133b252313fSGleb Smirnoff 	case PFIL_CONSUMED:
134d9f2a782SErmal Luçi 		return 1; /* Finished */
135b252313fSGleb Smirnoff 	case PFIL_PASS:
136b252313fSGleb Smirnoff 		*error = 0;
137b252313fSGleb Smirnoff 	}
138b252313fSGleb Smirnoff 	m = *mp;
139d9f2a782SErmal Luçi 	ip = mtod(m, struct ip *);
140d9f2a782SErmal Luçi 
141d9f2a782SErmal Luçi 	/* See if destination IP address was changed by packet filter. */
142d9f2a782SErmal Luçi 	if (odst.s_addr != ip->ip_dst.s_addr) {
143d9f2a782SErmal Luçi 		m->m_flags |= M_SKIP_FIREWALL;
144d9f2a782SErmal Luçi 		/* If destination is now ourself drop to ip_input(). */
145d9f2a782SErmal Luçi 		if (in_localip(ip->ip_dst)) {
146d9f2a782SErmal Luçi 			m->m_flags |= M_FASTFWD_OURS;
147d9f2a782SErmal Luçi 			if (m->m_pkthdr.rcvif == NULL)
148d9f2a782SErmal Luçi 				m->m_pkthdr.rcvif = V_loif;
149d9f2a782SErmal Luçi 			if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
150d9f2a782SErmal Luçi 				m->m_pkthdr.csum_flags |=
151d9f2a782SErmal Luçi 					CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
152d9f2a782SErmal Luçi 				m->m_pkthdr.csum_data = 0xffff;
153d9f2a782SErmal Luçi 			}
154d9f2a782SErmal Luçi 			m->m_pkthdr.csum_flags |=
155d9f2a782SErmal Luçi 				CSUM_IP_CHECKED | CSUM_IP_VALID;
15695033af9SMark Johnston #if defined(SCTP) || defined(SCTP_SUPPORT)
157d9f2a782SErmal Luçi 			if (m->m_pkthdr.csum_flags & CSUM_SCTP)
158d9f2a782SErmal Luçi 				m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
159d9f2a782SErmal Luçi #endif
160d9f2a782SErmal Luçi 			*error = netisr_queue(NETISR_IP, m);
161d9f2a782SErmal Luçi 			return 1; /* Finished */
162d9f2a782SErmal Luçi 		}
163d9f2a782SErmal Luçi 
164d9f2a782SErmal Luçi 		bzero(dst, sizeof(*dst));
165d9f2a782SErmal Luçi 		dst->sin_family = AF_INET;
166d9f2a782SErmal Luçi 		dst->sin_len = sizeof(*dst);
167d9f2a782SErmal Luçi 		dst->sin_addr = ip->ip_dst;
168d9f2a782SErmal Luçi 
169d9f2a782SErmal Luçi 		return -1; /* Reloop */
170d9f2a782SErmal Luçi 	}
171d9f2a782SErmal Luçi 	/* See if fib was changed by packet filter. */
172d9f2a782SErmal Luçi 	if ((*fibnum) != M_GETFIB(m)) {
173d9f2a782SErmal Luçi 		m->m_flags |= M_SKIP_FIREWALL;
174d9f2a782SErmal Luçi 		*fibnum = M_GETFIB(m);
175d9f2a782SErmal Luçi 		return -1; /* Reloop for FIB change */
176d9f2a782SErmal Luçi 	}
177d9f2a782SErmal Luçi 
178d9f2a782SErmal Luçi 	/* See if local, if yes, send it to netisr with IP_FASTFWD_OURS. */
179d9f2a782SErmal Luçi 	if (m->m_flags & M_FASTFWD_OURS) {
180d9f2a782SErmal Luçi 		if (m->m_pkthdr.rcvif == NULL)
181d9f2a782SErmal Luçi 			m->m_pkthdr.rcvif = V_loif;
182d9f2a782SErmal Luçi 		if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
183d9f2a782SErmal Luçi 			m->m_pkthdr.csum_flags |=
184d9f2a782SErmal Luçi 				CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
185d9f2a782SErmal Luçi 			m->m_pkthdr.csum_data = 0xffff;
186d9f2a782SErmal Luçi 		}
18795033af9SMark Johnston #if defined(SCTP) || defined(SCTP_SUPPORT)
188d9f2a782SErmal Luçi 		if (m->m_pkthdr.csum_flags & CSUM_SCTP)
189d9f2a782SErmal Luçi 			m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
190d9f2a782SErmal Luçi #endif
191d9f2a782SErmal Luçi 		m->m_pkthdr.csum_flags |=
192d9f2a782SErmal Luçi 			CSUM_IP_CHECKED | CSUM_IP_VALID;
193d9f2a782SErmal Luçi 
194d9f2a782SErmal Luçi 		*error = netisr_queue(NETISR_IP, m);
195d9f2a782SErmal Luçi 		return 1; /* Finished */
196d9f2a782SErmal Luçi 	}
197d9f2a782SErmal Luçi 	/* Or forward to some other address? */
198d9f2a782SErmal Luçi 	if ((m->m_flags & M_IP_NEXTHOP) &&
199d9f2a782SErmal Luçi 	    ((fwd_tag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL)) != NULL)) {
200d9f2a782SErmal Luçi 		bcopy((fwd_tag+1), dst, sizeof(struct sockaddr_in));
201d9f2a782SErmal Luçi 		m->m_flags |= M_SKIP_FIREWALL;
202d9f2a782SErmal Luçi 		m->m_flags &= ~M_IP_NEXTHOP;
203d9f2a782SErmal Luçi 		m_tag_delete(m, fwd_tag);
204d9f2a782SErmal Luçi 
205d9f2a782SErmal Luçi 		return -1; /* Reloop for CHANGE of dst */
206d9f2a782SErmal Luçi 	}
207d9f2a782SErmal Luçi 
208d9f2a782SErmal Luçi 	return 0;
209d9f2a782SErmal Luçi }
210d9f2a782SErmal Luçi 
211fb3bc596SJohn Baldwin static int
212fb3bc596SJohn Baldwin ip_output_send(struct inpcb *inp, struct ifnet *ifp, struct mbuf *m,
21335c7bb34SRandall Stewart     const struct sockaddr_in *gw, struct route *ro, bool stamp_tag)
214fb3bc596SJohn Baldwin {
215b2e60773SJohn Baldwin #ifdef KERN_TLS
216b2e60773SJohn Baldwin 	struct ktls_session *tls = NULL;
217b2e60773SJohn Baldwin #endif
218fb3bc596SJohn Baldwin 	struct m_snd_tag *mst;
219fb3bc596SJohn Baldwin 	int error;
220fb3bc596SJohn Baldwin 
221fb3bc596SJohn Baldwin 	MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
222fb3bc596SJohn Baldwin 	mst = NULL;
223fb3bc596SJohn Baldwin 
224b2e60773SJohn Baldwin #ifdef KERN_TLS
225b2e60773SJohn Baldwin 	/*
226b2e60773SJohn Baldwin 	 * If this is an unencrypted TLS record, save a reference to
227b2e60773SJohn Baldwin 	 * the record.  This local reference is used to call
228b2e60773SJohn Baldwin 	 * ktls_output_eagain after the mbuf has been freed (thus
229b2e60773SJohn Baldwin 	 * dropping the mbuf's reference) in if_output.
230b2e60773SJohn Baldwin 	 */
231b2e60773SJohn Baldwin 	if (m->m_next != NULL && mbuf_has_tls_session(m->m_next)) {
2327b6c99d0SGleb Smirnoff 		tls = ktls_hold(m->m_next->m_epg_tls);
233b2e60773SJohn Baldwin 		mst = tls->snd_tag;
234b2e60773SJohn Baldwin 
235b2e60773SJohn Baldwin 		/*
236b2e60773SJohn Baldwin 		 * If a TLS session doesn't have a valid tag, it must
237b2e60773SJohn Baldwin 		 * have had an earlier ifp mismatch, so drop this
238b2e60773SJohn Baldwin 		 * packet.
239b2e60773SJohn Baldwin 		 */
240b2e60773SJohn Baldwin 		if (mst == NULL) {
241b2e60773SJohn Baldwin 			error = EAGAIN;
242b2e60773SJohn Baldwin 			goto done;
243b2e60773SJohn Baldwin 		}
2446043ac20SAndrew Gallatin 		/*
2456043ac20SAndrew Gallatin 		 * Always stamp tags that include NIC ktls.
2466043ac20SAndrew Gallatin 		 */
2476043ac20SAndrew Gallatin 		stamp_tag = true;
248b2e60773SJohn Baldwin 	}
249b2e60773SJohn Baldwin #endif
250fb3bc596SJohn Baldwin #ifdef RATELIMIT
251b2e60773SJohn Baldwin 	if (inp != NULL && mst == NULL) {
252fb3bc596SJohn Baldwin 		if ((inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) != 0 ||
253fb3bc596SJohn Baldwin 		    (inp->inp_snd_tag != NULL &&
254fb3bc596SJohn Baldwin 		    inp->inp_snd_tag->ifp != ifp))
255fb3bc596SJohn Baldwin 			in_pcboutput_txrtlmt(inp, ifp, m);
256fb3bc596SJohn Baldwin 
257fb3bc596SJohn Baldwin 		if (inp->inp_snd_tag != NULL)
258fb3bc596SJohn Baldwin 			mst = inp->inp_snd_tag;
259fb3bc596SJohn Baldwin 	}
260fb3bc596SJohn Baldwin #endif
26135c7bb34SRandall Stewart 	if (stamp_tag && mst != NULL) {
262fb3bc596SJohn Baldwin 		KASSERT(m->m_pkthdr.rcvif == NULL,
263fb3bc596SJohn Baldwin 		    ("trying to add a send tag to a forwarded packet"));
264fb3bc596SJohn Baldwin 		if (mst->ifp != ifp) {
265fb3bc596SJohn Baldwin 			error = EAGAIN;
266fb3bc596SJohn Baldwin 			goto done;
267fb3bc596SJohn Baldwin 		}
268fb3bc596SJohn Baldwin 
269fb3bc596SJohn Baldwin 		/* stamp send tag on mbuf */
270fb3bc596SJohn Baldwin 		m->m_pkthdr.snd_tag = m_snd_tag_ref(mst);
271fb3bc596SJohn Baldwin 		m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
272fb3bc596SJohn Baldwin 	}
273fb3bc596SJohn Baldwin 
274fb3bc596SJohn Baldwin 	error = (*ifp->if_output)(ifp, m, (const struct sockaddr *)gw, ro);
275fb3bc596SJohn Baldwin 
276fb3bc596SJohn Baldwin done:
277fb3bc596SJohn Baldwin 	/* Check for route change invalidating send tags. */
278b2e60773SJohn Baldwin #ifdef KERN_TLS
279b2e60773SJohn Baldwin 	if (tls != NULL) {
280b2e60773SJohn Baldwin 		if (error == EAGAIN)
281b2e60773SJohn Baldwin 			error = ktls_output_eagain(inp, tls);
282b2e60773SJohn Baldwin 		ktls_free(tls);
283b2e60773SJohn Baldwin 	}
284b2e60773SJohn Baldwin #endif
285fb3bc596SJohn Baldwin #ifdef RATELIMIT
286fb3bc596SJohn Baldwin 	if (error == EAGAIN)
287fb3bc596SJohn Baldwin 		in_pcboutput_eagain(inp);
288fb3bc596SJohn Baldwin #endif
289fb3bc596SJohn Baldwin 	return (error);
290fb3bc596SJohn Baldwin }
291fb3bc596SJohn Baldwin 
292983066f0SAlexander V. Chernikov /* rte<>ro_flags translation */
293983066f0SAlexander V. Chernikov static inline void
294983066f0SAlexander V. Chernikov rt_update_ro_flags(struct route *ro)
295983066f0SAlexander V. Chernikov {
296983066f0SAlexander V. Chernikov 	int nh_flags = ro->ro_nh->nh_flags;
297983066f0SAlexander V. Chernikov 
298983066f0SAlexander V. Chernikov 	ro->ro_flags &= ~ (RT_REJECT|RT_BLACKHOLE|RT_HAS_GW);
299983066f0SAlexander V. Chernikov 
300983066f0SAlexander V. Chernikov 	ro->ro_flags |= (nh_flags & NHF_REJECT) ? RT_REJECT : 0;
301983066f0SAlexander V. Chernikov 	ro->ro_flags |= (nh_flags & NHF_BLACKHOLE) ? RT_BLACKHOLE : 0;
302983066f0SAlexander V. Chernikov 	ro->ro_flags |= (nh_flags & NHF_GATEWAY) ? RT_HAS_GW : 0;
303983066f0SAlexander V. Chernikov }
304983066f0SAlexander V. Chernikov 
305df8bae1dSRodney W. Grimes /*
306df8bae1dSRodney W. Grimes  * IP output.  The packet in mbuf chain m contains a skeletal IP
307df8bae1dSRodney W. Grimes  * header (with len, off, ttl, proto, tos, src, dst).
308df8bae1dSRodney W. Grimes  * The mbuf chain containing the packet will be freed.
309df8bae1dSRodney W. Grimes  * The mbuf opt, if present, will not be freed.
310bf984051SGleb Smirnoff  * If route ro is present and has ro_rt initialized, route lookup would be
311bf984051SGleb Smirnoff  * skipped and ro->ro_rt would be used. If ro is present but ro->ro_rt is NULL,
312bf984051SGleb Smirnoff  * then result of route lookup is stored in ro->ro_rt.
313bf984051SGleb Smirnoff  *
3143de758d3SRobert Watson  * In the IP forwarding case, the packet will arrive with options already
3153de758d3SRobert Watson  * inserted, so must have a NULL opt pointer.
316df8bae1dSRodney W. Grimes  */
317df8bae1dSRodney W. Grimes int
318f2565d68SRobert Watson ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
319f2565d68SRobert Watson     struct ip_moptions *imo, struct inpcb *inp)
320df8bae1dSRodney W. Grimes {
321cc0a3c8cSAndrey V. Elsukov 	struct rm_priotracker in_ifa_tracker;
3221e78ac21SJeffrey Hsu 	struct ip *ip;
323fc74d005SBjoern A. Zeeb 	struct ifnet *ifp = NULL;	/* keep compiler happy */
324ac9d7e26SMax Laier 	struct mbuf *m0;
32523bf9953SPoul-Henning Kamp 	int hlen = sizeof (struct ip);
326374ce248SMitchell Horne 	int mtu = 0;
327ca8b83b0SLuigi Rizzo 	int error = 0;
3286ca363ebSGleb Smirnoff 	struct sockaddr_in *dst, sin;
3294c7a6059SGleb Smirnoff 	const struct sockaddr_in *gw;
330374ce248SMitchell Horne 	struct in_ifaddr *ia = NULL;
3316ca363ebSGleb Smirnoff 	struct in_addr src;
33221d172a3SGleb Smirnoff 	int isbroadcast;
333078468edSGleb Smirnoff 	uint16_t ip_len, ip_off;
3349c57a5b6SHiroki Sato 	uint32_t fibnum;
335fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
3361a499816SEdward Tomasz Napierala 	int no_route_but_check_spd = 0;
3371a499816SEdward Tomasz Napierala #endif
33877a01441SJohn Baldwin 
339ac9d7e26SMax Laier 	M_ASSERTPKTHDR(m);
340b9555453SGleb Smirnoff 	NET_EPOCH_ASSERT();
34136e8826fSMax Laier 
342bc97ba51SJulian Elischer 	if (inp != NULL) {
343baa45840SRobert Watson 		INP_LOCK_ASSERT(inp);
34462e1ba83SRobert Watson 		M_SETFIB(m, inp->inp_inc.inc_fibnum);
345c2529042SHans Petter Selasky 		if ((flags & IP_NODEFAULTFLOWID) == 0) {
34680cb9f21SKip Macy 			m->m_pkthdr.flowid = inp->inp_flowid;
3479c423972SAdrian Chadd 			M_HASHTYPE_SET(m, inp->inp_flowtype);
34880cb9f21SKip Macy 		}
34950575ce1SAndrew Gallatin #ifdef NUMA
35050575ce1SAndrew Gallatin 		m->m_pkthdr.numa_domain = inp->inp_numa_domain;
35150575ce1SAndrew Gallatin #endif
352bc97ba51SJulian Elischer 	}
35362e1ba83SRobert Watson 
354df8bae1dSRodney W. Grimes 	if (opt) {
355ca8b83b0SLuigi Rizzo 		int len = 0;
356df8bae1dSRodney W. Grimes 		m = ip_insertoptions(m, opt, &len);
357cb7641e8SMaxim Konovalov 		if (len != 0)
358ca8b83b0SLuigi Rizzo 			hlen = len; /* ip->ip_hl is updated above */
359df8bae1dSRodney W. Grimes 	}
360df8bae1dSRodney W. Grimes 	ip = mtod(m, struct ip *);
3618f134647SGleb Smirnoff 	ip_len = ntohs(ip->ip_len);
3628f134647SGleb Smirnoff 	ip_off = ntohs(ip->ip_off);
3633efc3014SJulian Elischer 
364df8bae1dSRodney W. Grimes 	if ((flags & (IP_FORWARDING|IP_RAWOUTPUT)) == 0) {
36553be11f6SPoul-Henning Kamp 		ip->ip_v = IPVERSION;
36653be11f6SPoul-Henning Kamp 		ip->ip_hl = hlen >> 2;
3676d947416SGleb Smirnoff 		ip_fillid(ip);
368df8bae1dSRodney W. Grimes 	} else {
369ca8b83b0SLuigi Rizzo 		/* Header already set, fetch hlen from there */
37053be11f6SPoul-Henning Kamp 		hlen = ip->ip_hl << 2;
371df8bae1dSRodney W. Grimes 	}
3723924dfa7SMichael Tuexen 	if ((flags & IP_FORWARDING) == 0)
3733924dfa7SMichael Tuexen 		IPSTAT_INC(ips_localout);
3749c9137eaSGarrett Wollman 
375054692a4SAlexander V. Chernikov 	/*
376054692a4SAlexander V. Chernikov 	 * dst/gw handling:
377054692a4SAlexander V. Chernikov 	 *
3783c065f2fSGleb Smirnoff 	 * gw is readonly but can point either to dst OR rt_gateway,
3793c065f2fSGleb Smirnoff 	 * therefore we need restore gw if we're redoing lookup.
380054692a4SAlexander V. Chernikov 	 */
3819c57a5b6SHiroki Sato 	fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : M_GETFIB(m);
3826ca363ebSGleb Smirnoff 	if (ro != NULL)
3836ca363ebSGleb Smirnoff 		dst = (struct sockaddr_in *)&ro->ro_dst;
3846ca363ebSGleb Smirnoff 	else
3856ca363ebSGleb Smirnoff 		dst = &sin;
386983066f0SAlexander V. Chernikov 	if (ro == NULL || ro->ro_nh == NULL) {
387a4a6e773SHajimu UMEMOTO 		bzero(dst, sizeof(*dst));
388df8bae1dSRodney W. Grimes 		dst->sin_family = AF_INET;
389df8bae1dSRodney W. Grimes 		dst->sin_len = sizeof(*dst);
3909b932e9eSAndre Oppermann 		dst->sin_addr = ip->ip_dst;
391df8bae1dSRodney W. Grimes 	}
3926ca363ebSGleb Smirnoff 	gw = dst;
393d9f2a782SErmal Luçi again:
39484cc0778SGeorge V. Neville-Neil 	/*
39584cc0778SGeorge V. Neville-Neil 	 * Validate route against routing table additions;
39684cc0778SGeorge V. Neville-Neil 	 * a better/more specific route might have been added.
39784cc0778SGeorge V. Neville-Neil 	 */
398983066f0SAlexander V. Chernikov 	if (inp != NULL && ro != NULL && ro->ro_nh != NULL)
399983066f0SAlexander V. Chernikov 		NH_VALIDATE(ro, &inp->inp_rt_cookie, fibnum);
40084cc0778SGeorge V. Neville-Neil 	/*
40184cc0778SGeorge V. Neville-Neil 	 * If there is a cached route,
40284cc0778SGeorge V. Neville-Neil 	 * check that it is to the same destination
40384cc0778SGeorge V. Neville-Neil 	 * and is still up.  If not, free it and try again.
40484cc0778SGeorge V. Neville-Neil 	 * The address family should also be checked in case of sharing the
40584cc0778SGeorge V. Neville-Neil 	 * cache with IPv6.
40684cc0778SGeorge V. Neville-Neil 	 * Also check whether routing cache needs invalidation.
40784cc0778SGeorge V. Neville-Neil 	 */
408983066f0SAlexander V. Chernikov 	if (ro != NULL && ro->ro_nh != NULL &&
409174fb9dbSAlexander V. Chernikov 	    ((!NH_IS_VALID(ro->ro_nh)) || dst->sin_family != AF_INET ||
4106ca363ebSGleb Smirnoff 	    dst->sin_addr.s_addr != ip->ip_dst.s_addr))
411fc21c53fSRyan Stone 		RO_INVALIDATE_CACHE(ro);
412d9f2a782SErmal Luçi 	ia = NULL;
413df8bae1dSRodney W. Grimes 	/*
414a3fd02d8SBruce M Simpson 	 * If routing to interface only, short circuit routing lookup.
415a3fd02d8SBruce M Simpson 	 * The use of an all-ones broadcast address implies this; an
416a3fd02d8SBruce M Simpson 	 * interface is specified by the broadcast address of an interface,
417a3fd02d8SBruce M Simpson 	 * or the destination address of a ptp interface.
418df8bae1dSRodney W. Grimes 	 */
419a3fd02d8SBruce M Simpson 	if (flags & IP_SENDONES) {
4204f8585e0SAlan Somers 		if ((ia = ifatoia(ifa_ifwithbroadaddr(sintosa(dst),
42158a39d8cSAlan Somers 						      M_GETFIB(m)))) == NULL &&
4224f8585e0SAlan Somers 		    (ia = ifatoia(ifa_ifwithdstaddr(sintosa(dst),
42358a39d8cSAlan Somers 						    M_GETFIB(m)))) == NULL) {
42486425c62SRobert Watson 			IPSTAT_INC(ips_noroute);
425a3fd02d8SBruce M Simpson 			error = ENETUNREACH;
426a3fd02d8SBruce M Simpson 			goto bad;
427a3fd02d8SBruce M Simpson 		}
428a3fd02d8SBruce M Simpson 		ip->ip_dst.s_addr = INADDR_BROADCAST;
429a3fd02d8SBruce M Simpson 		dst->sin_addr = ip->ip_dst;
430a3fd02d8SBruce M Simpson 		ifp = ia->ia_ifp;
4316ca363ebSGleb Smirnoff 		mtu = ifp->if_mtu;
432a3fd02d8SBruce M Simpson 		ip->ip_ttl = 1;
433a3fd02d8SBruce M Simpson 		isbroadcast = 1;
4346ca363ebSGleb Smirnoff 		src = IA_SIN(ia)->sin_addr;
435a3fd02d8SBruce M Simpson 	} else if (flags & IP_ROUTETOIF) {
4364f8585e0SAlan Somers 		if ((ia = ifatoia(ifa_ifwithdstaddr(sintosa(dst),
43758a39d8cSAlan Somers 						    M_GETFIB(m)))) == NULL &&
4384f8585e0SAlan Somers 		    (ia = ifatoia(ifa_ifwithnet(sintosa(dst), 0,
43958a39d8cSAlan Somers 						M_GETFIB(m)))) == NULL) {
44086425c62SRobert Watson 			IPSTAT_INC(ips_noroute);
441df8bae1dSRodney W. Grimes 			error = ENETUNREACH;
442df8bae1dSRodney W. Grimes 			goto bad;
443df8bae1dSRodney W. Grimes 		}
444df8bae1dSRodney W. Grimes 		ifp = ia->ia_ifp;
4456ca363ebSGleb Smirnoff 		mtu = ifp->if_mtu;
446df8bae1dSRodney W. Grimes 		ip->ip_ttl = 1;
4476c1bd558SRyan Stone 		isbroadcast = ifp->if_flags & IFF_BROADCAST ?
4486c1bd558SRyan Stone 		    in_ifaddr_broadcast(dst->sin_addr, ia) : 0;
4496ca363ebSGleb Smirnoff 		src = IA_SIN(ia)->sin_addr;
4503afefa39SDaniel C. Sobral 	} else if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) &&
45138c1bc35SRuslan Ermilov 	    imo != NULL && imo->imo_multicast_ifp != NULL) {
4523afefa39SDaniel C. Sobral 		/*
45338c1bc35SRuslan Ermilov 		 * Bypass the normal routing lookup for multicast
45438c1bc35SRuslan Ermilov 		 * packets if the interface is specified.
4553afefa39SDaniel C. Sobral 		 */
45638c1bc35SRuslan Ermilov 		ifp = imo->imo_multicast_ifp;
4576ca363ebSGleb Smirnoff 		mtu = ifp->if_mtu;
458cc0a3c8cSAndrey V. Elsukov 		IFP_TO_IA(ifp, ia, &in_ifa_tracker);
45938c1bc35SRuslan Ermilov 		isbroadcast = 0;	/* fool gcc */
46054bb7ac0SGleb Smirnoff 		/* Interface may have no addresses. */
46154bb7ac0SGleb Smirnoff 		if (ia != NULL)
4626ca363ebSGleb Smirnoff 			src = IA_SIN(ia)->sin_addr;
46354bb7ac0SGleb Smirnoff 		else
46454bb7ac0SGleb Smirnoff 			src.s_addr = INADDR_ANY;
4656ca363ebSGleb Smirnoff 	} else if (ro != NULL) {
466983066f0SAlexander V. Chernikov 		if (ro->ro_nh == NULL) {
4672c17fe93SGarrett Wollman 			/*
4686ca363ebSGleb Smirnoff 			 * We want to do any cloning requested by the link
4696ca363ebSGleb Smirnoff 			 * layer, as this is probably required in all cases
4706ca363ebSGleb Smirnoff 			 * for correct operation (as it is for ARP).
4712c17fe93SGarrett Wollman 			 */
4724043ee3cSAlexander V. Chernikov 			uint32_t flowid;
473e440aed9SQing Li #ifdef RADIX_MPATH
4744043ee3cSAlexander V. Chernikov 			flowid = ntohl(ip->ip_src.s_addr ^ ip->ip_dst.s_addr);
475e440aed9SQing Li #else
4764043ee3cSAlexander V. Chernikov 			flowid = m->m_pkthdr.flowid;
477e440aed9SQing Li #endif
4784043ee3cSAlexander V. Chernikov 			ro->ro_nh = fib4_lookup(fibnum, dst->sin_addr, 0,
4794043ee3cSAlexander V. Chernikov 			    NHR_REF, flowid);
4804043ee3cSAlexander V. Chernikov 
481174fb9dbSAlexander V. Chernikov 			if (ro->ro_nh == NULL || (!NH_IS_VALID(ro->ro_nh))) {
482fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
4831a499816SEdward Tomasz Napierala 				/*
4841a499816SEdward Tomasz Napierala 				 * There is no route for this packet, but it is
4851a499816SEdward Tomasz Napierala 				 * possible that a matching SPD entry exists.
4861a499816SEdward Tomasz Napierala 				 */
4871a499816SEdward Tomasz Napierala 				no_route_but_check_spd = 1;
4881a499816SEdward Tomasz Napierala 				goto sendit;
4891a499816SEdward Tomasz Napierala #endif
49086425c62SRobert Watson 				IPSTAT_INC(ips_noroute);
491df8bae1dSRodney W. Grimes 				error = EHOSTUNREACH;
492df8bae1dSRodney W. Grimes 				goto bad;
493df8bae1dSRodney W. Grimes 			}
4946ca363ebSGleb Smirnoff 		}
495983066f0SAlexander V. Chernikov 		ia = ifatoia(ro->ro_nh->nh_ifa);
496983066f0SAlexander V. Chernikov 		ifp = ro->ro_nh->nh_ifp;
497983066f0SAlexander V. Chernikov 		counter_u64_add(ro->ro_nh->nh_pksent, 1);
49836402a68SAlexander V. Chernikov 		rt_update_ro_flags(ro);
499983066f0SAlexander V. Chernikov 		if (ro->ro_nh->nh_flags & NHF_GATEWAY)
500983066f0SAlexander V. Chernikov 			gw = &ro->ro_nh->gw4_sa;
501983066f0SAlexander V. Chernikov 		if (ro->ro_nh->nh_flags & NHF_HOST)
502983066f0SAlexander V. Chernikov 			isbroadcast = (ro->ro_nh->nh_flags & NHF_BROADCAST);
5036c1bd558SRyan Stone 		else if (ifp->if_flags & IFF_BROADCAST)
50490cc51a1SRyan Stone 			isbroadcast = in_ifaddr_broadcast(gw->sin_addr, ia);
5056c1bd558SRyan Stone 		else
5066c1bd558SRyan Stone 			isbroadcast = 0;
507983066f0SAlexander V. Chernikov 		if (ro->ro_nh->nh_flags & NHF_HOST)
508983066f0SAlexander V. Chernikov 			mtu = ro->ro_nh->nh_mtu;
5097f948f12SAlexander V. Chernikov 		else
5103ae2ad08SJohn-Mark Gurney 			mtu = ifp->if_mtu;
5116ca363ebSGleb Smirnoff 		src = IA_SIN(ia)->sin_addr;
5126ca363ebSGleb Smirnoff 	} else {
5133553b300SAlexander V. Chernikov 		struct nhop_object *nh;
5146ca363ebSGleb Smirnoff 
515*2259a030SAlexander V. Chernikov 		nh = fib4_lookup(M_GETFIB(m), ip->ip_dst, 0, NHR_NONE,
516*2259a030SAlexander V. Chernikov 		    m->m_pkthdr.flowid);
5173553b300SAlexander V. Chernikov 		if (nh == NULL) {
5186ca363ebSGleb Smirnoff #if defined(IPSEC) || defined(IPSEC_SUPPORT)
5196ca363ebSGleb Smirnoff 			/*
5206ca363ebSGleb Smirnoff 			 * There is no route for this packet, but it is
5216ca363ebSGleb Smirnoff 			 * possible that a matching SPD entry exists.
5226ca363ebSGleb Smirnoff 			 */
5236ca363ebSGleb Smirnoff 			no_route_but_check_spd = 1;
5246ca363ebSGleb Smirnoff 			goto sendit;
5256ca363ebSGleb Smirnoff #endif
5266ca363ebSGleb Smirnoff 			IPSTAT_INC(ips_noroute);
5276ca363ebSGleb Smirnoff 			error = EHOSTUNREACH;
5286ca363ebSGleb Smirnoff 			goto bad;
5296ca363ebSGleb Smirnoff 		}
5303553b300SAlexander V. Chernikov 		ifp = nh->nh_ifp;
5313553b300SAlexander V. Chernikov 		mtu = nh->nh_mtu;
5326ca363ebSGleb Smirnoff 		/*
5336ca363ebSGleb Smirnoff 		 * We are rewriting here dst to be gw actually, contradicting
5346ca363ebSGleb Smirnoff 		 * comment at the beginning of the function. However, in this
5356ca363ebSGleb Smirnoff 		 * case we are always dealing with on stack dst.
5366ca363ebSGleb Smirnoff 		 * In case if pfil(9) sends us back to beginning of the
5376ca363ebSGleb Smirnoff 		 * function, the dst would be rewritten by ip_output_pfil().
5386ca363ebSGleb Smirnoff 		 */
5396ca363ebSGleb Smirnoff 		MPASS(dst == &sin);
5403553b300SAlexander V. Chernikov 		if (nh->nh_flags & NHF_GATEWAY)
5413553b300SAlexander V. Chernikov 			dst->sin_addr = nh->gw4_sa.sin_addr;
5423553b300SAlexander V. Chernikov 		ia = ifatoia(nh->nh_ifa);
5433553b300SAlexander V. Chernikov 		src = IA_SIN(ia)->sin_addr;
5443553b300SAlexander V. Chernikov 		isbroadcast = (((nh->nh_flags & (NHF_HOST | NHF_BROADCAST)) ==
5456ca363ebSGleb Smirnoff 		    (NHF_HOST | NHF_BROADCAST)) ||
5466ca363ebSGleb Smirnoff 		    ((ifp->if_flags & IFF_BROADCAST) &&
5476ca363ebSGleb Smirnoff 		    in_ifaddr_broadcast(dst->sin_addr, ia)));
5486ca363ebSGleb Smirnoff 	}
5496ca363ebSGleb Smirnoff 
550c744cde4SBjoern A. Zeeb 	/* Catch a possible divide by zero later. */
551983066f0SAlexander V. Chernikov 	KASSERT(mtu > 0, ("%s: mtu %d <= 0, ro=%p (nh_flags=0x%08x) ifp=%p",
5526ca363ebSGleb Smirnoff 	    __func__, mtu, ro,
553983066f0SAlexander V. Chernikov 	    (ro != NULL && ro->ro_nh != NULL) ? ro->ro_nh->nh_flags : 0, ifp));
554d9f2a782SErmal Luçi 
5559b932e9eSAndre Oppermann 	if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) {
556df8bae1dSRodney W. Grimes 		m->m_flags |= M_MCAST;
557df8bae1dSRodney W. Grimes 		/*
558183e1c86SGleb Smirnoff 		 * IP destination address is multicast.  Make sure "gw"
559183e1c86SGleb Smirnoff 		 * still points to the address in "ro".  (It may have been
560183e1c86SGleb Smirnoff 		 * changed to point to a gateway address, above.)
561183e1c86SGleb Smirnoff 		 */
562183e1c86SGleb Smirnoff 		gw = dst;
563183e1c86SGleb Smirnoff 		/*
564df8bae1dSRodney W. Grimes 		 * See if the caller provided any multicast options
565df8bae1dSRodney W. Grimes 		 */
566df8bae1dSRodney W. Grimes 		if (imo != NULL) {
567df8bae1dSRodney W. Grimes 			ip->ip_ttl = imo->imo_multicast_ttl;
5681c5de19aSGarrett Wollman 			if (imo->imo_multicast_vif != -1)
5691c5de19aSGarrett Wollman 				ip->ip_src.s_addr =
570bbb4330bSLuigi Rizzo 				    ip_mcast_src ?
571bbb4330bSLuigi Rizzo 				    ip_mcast_src(imo->imo_multicast_vif) :
572bbb4330bSLuigi Rizzo 				    INADDR_ANY;
573df8bae1dSRodney W. Grimes 		} else
574df8bae1dSRodney W. Grimes 			ip->ip_ttl = IP_DEFAULT_MULTICAST_TTL;
575df8bae1dSRodney W. Grimes 		/*
576df8bae1dSRodney W. Grimes 		 * Confirm that the outgoing interface supports multicast.
577df8bae1dSRodney W. Grimes 		 */
5781c5de19aSGarrett Wollman 		if ((imo == NULL) || (imo->imo_multicast_vif == -1)) {
579df8bae1dSRodney W. Grimes 			if ((ifp->if_flags & IFF_MULTICAST) == 0) {
58086425c62SRobert Watson 				IPSTAT_INC(ips_noroute);
581df8bae1dSRodney W. Grimes 				error = ENETUNREACH;
582df8bae1dSRodney W. Grimes 				goto bad;
583df8bae1dSRodney W. Grimes 			}
5841c5de19aSGarrett Wollman 		}
585df8bae1dSRodney W. Grimes 		/*
586df8bae1dSRodney W. Grimes 		 * If source address not specified yet, use address
587df8bae1dSRodney W. Grimes 		 * of outgoing interface.
588df8bae1dSRodney W. Grimes 		 */
5896ca363ebSGleb Smirnoff 		if (ip->ip_src.s_addr == INADDR_ANY)
5906ca363ebSGleb Smirnoff 			ip->ip_src = src;
591df8bae1dSRodney W. Grimes 
5928b889dbbSBruce M Simpson 		if ((imo == NULL && in_mcast_loop) ||
5938b889dbbSBruce M Simpson 		    (imo && imo->imo_multicast_loop)) {
594df8bae1dSRodney W. Grimes 			/*
5958b889dbbSBruce M Simpson 			 * Loop back multicast datagram if not expressly
5968b889dbbSBruce M Simpson 			 * forbidden to do so, even if we are not a member
5978b889dbbSBruce M Simpson 			 * of the group; ip_input() will filter it later,
5988b889dbbSBruce M Simpson 			 * thus deferring a hash lookup and mutex acquisition
5998b889dbbSBruce M Simpson 			 * at the expense of a cheap copy using m_copym().
600df8bae1dSRodney W. Grimes 			 */
601331dff07SAlexander V. Chernikov 			ip_mloopback(ifp, m, hlen);
6028b889dbbSBruce M Simpson 		} else {
603df8bae1dSRodney W. Grimes 			/*
604df8bae1dSRodney W. Grimes 			 * If we are acting as a multicast router, perform
605df8bae1dSRodney W. Grimes 			 * multicast forwarding as if the packet had just
606df8bae1dSRodney W. Grimes 			 * arrived on the interface to which we are about
607df8bae1dSRodney W. Grimes 			 * to send.  The multicast forwarding function
608df8bae1dSRodney W. Grimes 			 * recursively calls this function, using the
609df8bae1dSRodney W. Grimes 			 * IP_FORWARDING flag to prevent infinite recursion.
610df8bae1dSRodney W. Grimes 			 *
611df8bae1dSRodney W. Grimes 			 * Multicasts that are looped back by ip_mloopback(),
612df8bae1dSRodney W. Grimes 			 * above, will be forwarded by the ip_input() routine,
613df8bae1dSRodney W. Grimes 			 * if necessary.
614df8bae1dSRodney W. Grimes 			 */
615603724d3SBjoern A. Zeeb 			if (V_ip_mrouter && (flags & IP_FORWARDING) == 0) {
616f0068c4aSGarrett Wollman 				/*
617bbb4330bSLuigi Rizzo 				 * If rsvp daemon is not running, do not
618f0068c4aSGarrett Wollman 				 * set ip_moptions. This ensures that the packet
619f0068c4aSGarrett Wollman 				 * is multicast and not just sent down one link
620f0068c4aSGarrett Wollman 				 * as prescribed by rsvpd.
621f0068c4aSGarrett Wollman 				 */
622603724d3SBjoern A. Zeeb 				if (!V_rsvp_on)
623f0068c4aSGarrett Wollman 					imo = NULL;
624bbb4330bSLuigi Rizzo 				if (ip_mforward &&
625bbb4330bSLuigi Rizzo 				    ip_mforward(ip, ifp, m, imo) != 0) {
626df8bae1dSRodney W. Grimes 					m_freem(m);
627df8bae1dSRodney W. Grimes 					goto done;
628df8bae1dSRodney W. Grimes 				}
629df8bae1dSRodney W. Grimes 			}
630df8bae1dSRodney W. Grimes 		}
6315e9ae478SGarrett Wollman 
632df8bae1dSRodney W. Grimes 		/*
633df8bae1dSRodney W. Grimes 		 * Multicasts with a time-to-live of zero may be looped-
634df8bae1dSRodney W. Grimes 		 * back, above, but must not be transmitted on a network.
635df8bae1dSRodney W. Grimes 		 * Also, multicasts addressed to the loopback interface
636df8bae1dSRodney W. Grimes 		 * are not sent -- the above call to ip_mloopback() will
6378b889dbbSBruce M Simpson 		 * loop back a copy. ip_input() will drop the copy if
6388b889dbbSBruce M Simpson 		 * this host does not belong to the destination group on
6398b889dbbSBruce M Simpson 		 * the loopback interface.
640df8bae1dSRodney W. Grimes 		 */
641f5fea3ddSPaul Traina 		if (ip->ip_ttl == 0 || ifp->if_flags & IFF_LOOPBACK) {
642df8bae1dSRodney W. Grimes 			m_freem(m);
643df8bae1dSRodney W. Grimes 			goto done;
644df8bae1dSRodney W. Grimes 		}
645df8bae1dSRodney W. Grimes 
646df8bae1dSRodney W. Grimes 		goto sendit;
647df8bae1dSRodney W. Grimes 	}
6486a7c943cSAndre Oppermann 
649df8bae1dSRodney W. Grimes 	/*
6502b25acc1SLuigi Rizzo 	 * If the source address is not specified yet, use the address
651cb459254SJohn-Mark Gurney 	 * of the outoing interface.
652df8bae1dSRodney W. Grimes 	 */
6536ca363ebSGleb Smirnoff 	if (ip->ip_src.s_addr == INADDR_ANY)
6546ca363ebSGleb Smirnoff 		ip->ip_src = src;
6556a7c943cSAndre Oppermann 
656ca7a789aSMax Laier 	/*
657df8bae1dSRodney W. Grimes 	 * Look for broadcast address and
6588c3f5566SRuslan Ermilov 	 * verify user is allowed to send
659df8bae1dSRodney W. Grimes 	 * such a packet.
660df8bae1dSRodney W. Grimes 	 */
6619f9b3dc4SGarrett Wollman 	if (isbroadcast) {
662df8bae1dSRodney W. Grimes 		if ((ifp->if_flags & IFF_BROADCAST) == 0) {
663df8bae1dSRodney W. Grimes 			error = EADDRNOTAVAIL;
664df8bae1dSRodney W. Grimes 			goto bad;
665df8bae1dSRodney W. Grimes 		}
666df8bae1dSRodney W. Grimes 		if ((flags & IP_ALLOWBROADCAST) == 0) {
667df8bae1dSRodney W. Grimes 			error = EACCES;
668df8bae1dSRodney W. Grimes 			goto bad;
669df8bae1dSRodney W. Grimes 		}
670df8bae1dSRodney W. Grimes 		/* don't allow broadcast messages to be fragmented */
6718f134647SGleb Smirnoff 		if (ip_len > mtu) {
672df8bae1dSRodney W. Grimes 			error = EMSGSIZE;
673df8bae1dSRodney W. Grimes 			goto bad;
674df8bae1dSRodney W. Grimes 		}
675df8bae1dSRodney W. Grimes 		m->m_flags |= M_BCAST;
6769f9b3dc4SGarrett Wollman 	} else {
677df8bae1dSRodney W. Grimes 		m->m_flags &= ~M_BCAST;
6789f9b3dc4SGarrett Wollman 	}
679df8bae1dSRodney W. Grimes 
680f1743588SDarren Reed sendit:
681fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
682fcf59617SAndrey V. Elsukov 	if (IPSEC_ENABLED(ipv4)) {
683fcf59617SAndrey V. Elsukov 		if ((error = IPSEC_OUTPUT(ipv4, m, inp)) != 0) {
684fcf59617SAndrey V. Elsukov 			if (error == EINPROGRESS)
685fcf59617SAndrey V. Elsukov 				error = 0;
6861dfcf0d2SAndre Oppermann 			goto done;
687fcf59617SAndrey V. Elsukov 		}
68833841545SHajimu UMEMOTO 	}
6891a499816SEdward Tomasz Napierala 	/*
6901a499816SEdward Tomasz Napierala 	 * Check if there was a route for this packet; return error if not.
6911a499816SEdward Tomasz Napierala 	 */
6921a499816SEdward Tomasz Napierala 	if (no_route_but_check_spd) {
6931a499816SEdward Tomasz Napierala 		IPSTAT_INC(ips_noroute);
6941a499816SEdward Tomasz Napierala 		error = EHOSTUNREACH;
6951a499816SEdward Tomasz Napierala 		goto bad;
6961a499816SEdward Tomasz Napierala 	}
6971dfcf0d2SAndre Oppermann 	/* Update variables that are affected by ipsec4_output(). */
69833841545SHajimu UMEMOTO 	ip = mtod(m, struct ip *);
69933841545SHajimu UMEMOTO 	hlen = ip->ip_hl << 2;
700b2630c29SGeorge V. Neville-Neil #endif /* IPSEC */
70133841545SHajimu UMEMOTO 
702c21fd232SAndre Oppermann 	/* Jump over all PFIL processing if hooks are not active. */
703b252313fSGleb Smirnoff 	if (PFIL_HOOKED_OUT(V_inet_pfil_head)) {
70405fc9d78SKristof Provost 		switch (ip_output_pfil(&m, ifp, flags, inp, dst, &fibnum,
70505fc9d78SKristof Provost 		    &error)) {
706d9f2a782SErmal Luçi 		case 1: /* Finished */
707c4ac87eaSDarren Reed 			goto done;
7089b932e9eSAndre Oppermann 
709d9f2a782SErmal Luçi 		case 0: /* Continue normally */
710c4ac87eaSDarren Reed 			ip = mtod(m, struct ip *);
711d9f2a782SErmal Luçi 			break;
712fed1c7e9SSøren Schmidt 
713d9f2a782SErmal Luçi 		case -1: /* Need to try again */
714d9f2a782SErmal Luçi 			/* Reset everything for a new round */
7156ca363ebSGleb Smirnoff 			if (ro != NULL) {
716983066f0SAlexander V. Chernikov 				RO_NHFREE(ro);
7174fb3a820SAlexander V. Chernikov 				ro->ro_prepend = NULL;
7186ca363ebSGleb Smirnoff 			}
719d9f2a782SErmal Luçi 			gw = dst;
720d9f2a782SErmal Luçi 			ip = mtod(m, struct ip *);
7219b932e9eSAndre Oppermann 			goto again;
722d9f2a782SErmal Luçi 		}
723099dd043SAndre Oppermann 	}
7242b25acc1SLuigi Rizzo 
7256c1c6ae5SRodney W. Grimes 	/* IN_LOOPBACK must not appear on the wire - RFC1122. */
7266c1c6ae5SRodney W. Grimes 	if (IN_LOOPBACK(ntohl(ip->ip_dst.s_addr)) ||
7276c1c6ae5SRodney W. Grimes 	    IN_LOOPBACK(ntohl(ip->ip_src.s_addr))) {
72851c8ec4aSRuslan Ermilov 		if ((ifp->if_flags & IFF_LOOPBACK) == 0) {
72986425c62SRobert Watson 			IPSTAT_INC(ips_badaddr);
73051c8ec4aSRuslan Ermilov 			error = EADDRNOTAVAIL;
73151c8ec4aSRuslan Ermilov 			goto bad;
73251c8ec4aSRuslan Ermilov 		}
73351c8ec4aSRuslan Ermilov 	}
73451c8ec4aSRuslan Ermilov 
735206a3274SRuslan Ermilov 	m->m_pkthdr.csum_flags |= CSUM_IP;
736078468edSGleb Smirnoff 	if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA & ~ifp->if_hwassist) {
73782334850SJohn Baldwin 		m = mb_unmapped_to_ext(m);
73882334850SJohn Baldwin 		if (m == NULL) {
73982334850SJohn Baldwin 			IPSTAT_INC(ips_odropped);
74082334850SJohn Baldwin 			error = ENOBUFS;
74182334850SJohn Baldwin 			goto bad;
74282334850SJohn Baldwin 		}
743db4f9cc7SJonathan Lemon 		in_delayed_cksum(m);
744078468edSGleb Smirnoff 		m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
74582334850SJohn Baldwin 	} else if ((ifp->if_capenable & IFCAP_NOMAP) == 0) {
74682334850SJohn Baldwin 		m = mb_unmapped_to_ext(m);
74782334850SJohn Baldwin 		if (m == NULL) {
74882334850SJohn Baldwin 			IPSTAT_INC(ips_odropped);
74982334850SJohn Baldwin 			error = ENOBUFS;
75082334850SJohn Baldwin 			goto bad;
75182334850SJohn Baldwin 		}
752db4f9cc7SJonathan Lemon 	}
75395033af9SMark Johnston #if defined(SCTP) || defined(SCTP_SUPPORT)
754078468edSGleb Smirnoff 	if (m->m_pkthdr.csum_flags & CSUM_SCTP & ~ifp->if_hwassist) {
75582334850SJohn Baldwin 		m = mb_unmapped_to_ext(m);
75682334850SJohn Baldwin 		if (m == NULL) {
75782334850SJohn Baldwin 			IPSTAT_INC(ips_odropped);
75882334850SJohn Baldwin 			error = ENOBUFS;
75982334850SJohn Baldwin 			goto bad;
76082334850SJohn Baldwin 		}
7611966e5b5SRandall Stewart 		sctp_delayed_cksum(m, (uint32_t)(ip->ip_hl << 2));
762078468edSGleb Smirnoff 		m->m_pkthdr.csum_flags &= ~CSUM_SCTP;
7632f4afd21SRandall Stewart 	}
7642f4afd21SRandall Stewart #endif
765db4f9cc7SJonathan Lemon 
766e7319babSPoul-Henning Kamp 	/*
767db4f9cc7SJonathan Lemon 	 * If small enough for interface, or the interface will take
768233dcce1SAndre Oppermann 	 * care of the fragmentation for us, we can just send directly.
769b092fd6cSNavdeep Parhar 	 * Note that if_vxlan could have requested TSO even though the outer
770b092fd6cSNavdeep Parhar 	 * frame is UDP.  It is correct to not fragment such datagrams and
771b092fd6cSNavdeep Parhar 	 * instead just pass them on to the driver.
772df8bae1dSRodney W. Grimes 	 */
77321d172a3SGleb Smirnoff 	if (ip_len <= mtu ||
774b092fd6cSNavdeep Parhar 	    (m->m_pkthdr.csum_flags & ifp->if_hwassist &
775b092fd6cSNavdeep Parhar 	    (CSUM_TSO | CSUM_INNER_TSO)) != 0) {
776df8bae1dSRodney W. Grimes 		ip->ip_sum = 0;
777078468edSGleb Smirnoff 		if (m->m_pkthdr.csum_flags & CSUM_IP & ~ifp->if_hwassist) {
778df8bae1dSRodney W. Grimes 			ip->ip_sum = in_cksum(m, hlen);
779078468edSGleb Smirnoff 			m->m_pkthdr.csum_flags &= ~CSUM_IP;
780078468edSGleb Smirnoff 		}
7815da9f8faSJosef Karthauser 
782233dcce1SAndre Oppermann 		/*
783233dcce1SAndre Oppermann 		 * Record statistics for this interface address.
784233dcce1SAndre Oppermann 		 * With CSUM_TSO the byte/packet count will be slightly
785233dcce1SAndre Oppermann 		 * incorrect because we count the IP+TCP headers only
786233dcce1SAndre Oppermann 		 * once instead of for every generated packet.
787233dcce1SAndre Oppermann 		 */
78838c1bc35SRuslan Ermilov 		if (!(flags & IP_FORWARDING) && ia) {
789b092fd6cSNavdeep Parhar 			if (m->m_pkthdr.csum_flags &
790b092fd6cSNavdeep Parhar 			    (CSUM_TSO | CSUM_INNER_TSO))
7917caf4ab7SGleb Smirnoff 				counter_u64_add(ia->ia_ifa.ifa_opackets,
7927caf4ab7SGleb Smirnoff 				    m->m_pkthdr.len / m->m_pkthdr.tso_segsz);
793233dcce1SAndre Oppermann 			else
7947caf4ab7SGleb Smirnoff 				counter_u64_add(ia->ia_ifa.ifa_opackets, 1);
7957caf4ab7SGleb Smirnoff 
7967caf4ab7SGleb Smirnoff 			counter_u64_add(ia->ia_ifa.ifa_obytes, m->m_pkthdr.len);
7975da9f8faSJosef Karthauser 		}
79853dcc544SMike Silbersack #ifdef MBUF_STRESS_TEST
7993390d476SMike Silbersack 		if (mbuf_frag_size && m->m_pkthdr.len > mbuf_frag_size)
800eb1b1807SGleb Smirnoff 			m = m_fragment(m, M_NOWAIT, mbuf_frag_size);
8019d9edc56SMike Silbersack #endif
802147f74d1SAndre Oppermann 		/*
803147f74d1SAndre Oppermann 		 * Reset layer specific mbuf flags
804147f74d1SAndre Oppermann 		 * to avoid confusing lower layers.
805147f74d1SAndre Oppermann 		 */
80686bd0491SAndre Oppermann 		m_clrprotoflags(m);
80757f60867SMark Johnston 		IP_PROBE(send, NULL, NULL, ip, ifp, ip, NULL);
80835c7bb34SRandall Stewart 		error = ip_output_send(inp, ifp, m, gw, ro,
80935c7bb34SRandall Stewart 		    (flags & IP_NO_SND_TAG_RL) ? false : true);
810df8bae1dSRodney W. Grimes 		goto done;
811df8bae1dSRodney W. Grimes 	}
8121e78ac21SJeffrey Hsu 
813233dcce1SAndre Oppermann 	/* Balk when DF bit is set or the interface didn't support TSO. */
814b092fd6cSNavdeep Parhar 	if ((ip_off & IP_DF) ||
815b092fd6cSNavdeep Parhar 	    (m->m_pkthdr.csum_flags & (CSUM_TSO | CSUM_INNER_TSO))) {
816df8bae1dSRodney W. Grimes 		error = EMSGSIZE;
81786425c62SRobert Watson 		IPSTAT_INC(ips_cantfrag);
818df8bae1dSRodney W. Grimes 		goto bad;
819df8bae1dSRodney W. Grimes 	}
8201e78ac21SJeffrey Hsu 
8211e78ac21SJeffrey Hsu 	/*
8221e78ac21SJeffrey Hsu 	 * Too large for interface; fragment if possible. If successful,
8231e78ac21SJeffrey Hsu 	 * on return, m will point to a list of packets to be sent.
8241e78ac21SJeffrey Hsu 	 */
825078468edSGleb Smirnoff 	error = ip_fragment(ip, &m, mtu, ifp->if_hwassist);
8261e78ac21SJeffrey Hsu 	if (error)
827df8bae1dSRodney W. Grimes 		goto bad;
8281e78ac21SJeffrey Hsu 	for (; m; m = m0) {
829df8bae1dSRodney W. Grimes 		m0 = m->m_nextpkt;
830b375c9ecSLuigi Rizzo 		m->m_nextpkt = 0;
83107203494SDaniel C. Sobral 		if (error == 0) {
832fe937674SJosef Karthauser 			/* Record statistics for this interface address. */
83307203494SDaniel C. Sobral 			if (ia != NULL) {
8347caf4ab7SGleb Smirnoff 				counter_u64_add(ia->ia_ifa.ifa_opackets, 1);
8357caf4ab7SGleb Smirnoff 				counter_u64_add(ia->ia_ifa.ifa_obytes,
8367caf4ab7SGleb Smirnoff 				    m->m_pkthdr.len);
83707203494SDaniel C. Sobral 			}
838147f74d1SAndre Oppermann 			/*
839147f74d1SAndre Oppermann 			 * Reset layer specific mbuf flags
840147f74d1SAndre Oppermann 			 * to avoid confusing upper layers.
841147f74d1SAndre Oppermann 			 */
84286bd0491SAndre Oppermann 			m_clrprotoflags(m);
843fe937674SJosef Karthauser 
8442e77d270SAndrey V. Elsukov 			IP_PROBE(send, NULL, NULL, mtod(m, struct ip *), ifp,
8452e77d270SAndrey V. Elsukov 			    mtod(m, struct ip *), NULL);
84635c7bb34SRandall Stewart 			error = ip_output_send(inp, ifp, m, gw, ro, true);
847fe937674SJosef Karthauser 		} else
848df8bae1dSRodney W. Grimes 			m_freem(m);
849df8bae1dSRodney W. Grimes 	}
850df8bae1dSRodney W. Grimes 
851df8bae1dSRodney W. Grimes 	if (error == 0)
85286425c62SRobert Watson 		IPSTAT_INC(ips_fragmented);
8531e78ac21SJeffrey Hsu 
854df8bae1dSRodney W. Grimes done:
855df8bae1dSRodney W. Grimes 	return (error);
856df8bae1dSRodney W. Grimes  bad:
8573528d68fSBill Paul 	m_freem(m);
858df8bae1dSRodney W. Grimes 	goto done;
859df8bae1dSRodney W. Grimes }
860df8bae1dSRodney W. Grimes 
8611e78ac21SJeffrey Hsu /*
8621e78ac21SJeffrey Hsu  * Create a chain of fragments which fit the given mtu. m_frag points to the
8631e78ac21SJeffrey Hsu  * mbuf to be fragmented; on return it points to the chain with the fragments.
8641e78ac21SJeffrey Hsu  * Return 0 if no error. If error, m_frag may contain a partially built
8651e78ac21SJeffrey Hsu  * chain of fragments that should be freed by the caller.
8661e78ac21SJeffrey Hsu  *
8671e78ac21SJeffrey Hsu  * if_hwassist_flags is the hw offload capabilities (see if_data.ifi_hwassist)
8681e78ac21SJeffrey Hsu  */
8691e78ac21SJeffrey Hsu int
8701e78ac21SJeffrey Hsu ip_fragment(struct ip *ip, struct mbuf **m_frag, int mtu,
871078468edSGleb Smirnoff     u_long if_hwassist_flags)
8721e78ac21SJeffrey Hsu {
8731e78ac21SJeffrey Hsu 	int error = 0;
8741e78ac21SJeffrey Hsu 	int hlen = ip->ip_hl << 2;
8751e78ac21SJeffrey Hsu 	int len = (mtu - hlen) & ~7;	/* size of payload in each fragment */
8761e78ac21SJeffrey Hsu 	int off;
8771e78ac21SJeffrey Hsu 	struct mbuf *m0 = *m_frag;	/* the original packet		*/
8781e78ac21SJeffrey Hsu 	int firstlen;
8791e78ac21SJeffrey Hsu 	struct mbuf **mnext;
8801e78ac21SJeffrey Hsu 	int nfrags;
88121d172a3SGleb Smirnoff 	uint16_t ip_len, ip_off;
8821e78ac21SJeffrey Hsu 
88321d172a3SGleb Smirnoff 	ip_len = ntohs(ip->ip_len);
88421d172a3SGleb Smirnoff 	ip_off = ntohs(ip->ip_off);
88521d172a3SGleb Smirnoff 
88621d172a3SGleb Smirnoff 	if (ip_off & IP_DF) {	/* Fragmentation not allowed */
88786425c62SRobert Watson 		IPSTAT_INC(ips_cantfrag);
8881e78ac21SJeffrey Hsu 		return EMSGSIZE;
8891e78ac21SJeffrey Hsu 	}
8901e78ac21SJeffrey Hsu 
8911e78ac21SJeffrey Hsu 	/*
8921e78ac21SJeffrey Hsu 	 * Must be able to put at least 8 bytes per fragment.
8931e78ac21SJeffrey Hsu 	 */
8941e78ac21SJeffrey Hsu 	if (len < 8)
8951e78ac21SJeffrey Hsu 		return EMSGSIZE;
8961e78ac21SJeffrey Hsu 
8971e78ac21SJeffrey Hsu 	/*
8981e78ac21SJeffrey Hsu 	 * If the interface will not calculate checksums on
8991e78ac21SJeffrey Hsu 	 * fragmented packets, then do it here.
9001e78ac21SJeffrey Hsu 	 */
901da2299c5SAndre Oppermann 	if (m0->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
90282334850SJohn Baldwin 		m0 = mb_unmapped_to_ext(m0);
90382334850SJohn Baldwin 		if (m0 == NULL) {
90482334850SJohn Baldwin 			error = ENOBUFS;
90582334850SJohn Baldwin 			IPSTAT_INC(ips_odropped);
90682334850SJohn Baldwin 			goto done;
90782334850SJohn Baldwin 		}
9081e78ac21SJeffrey Hsu 		in_delayed_cksum(m0);
9091e78ac21SJeffrey Hsu 		m0->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
9101e78ac21SJeffrey Hsu 	}
91195033af9SMark Johnston #if defined(SCTP) || defined(SCTP_SUPPORT)
912da2299c5SAndre Oppermann 	if (m0->m_pkthdr.csum_flags & CSUM_SCTP) {
91382334850SJohn Baldwin 		m0 = mb_unmapped_to_ext(m0);
91482334850SJohn Baldwin 		if (m0 == NULL) {
91582334850SJohn Baldwin 			error = ENOBUFS;
91682334850SJohn Baldwin 			IPSTAT_INC(ips_odropped);
91782334850SJohn Baldwin 			goto done;
91882334850SJohn Baldwin 		}
9191966e5b5SRandall Stewart 		sctp_delayed_cksum(m0, hlen);
9202f4afd21SRandall Stewart 		m0->m_pkthdr.csum_flags &= ~CSUM_SCTP;
9212f4afd21SRandall Stewart 	}
9222f4afd21SRandall Stewart #endif
9231e78ac21SJeffrey Hsu 	if (len > PAGE_SIZE) {
9241e78ac21SJeffrey Hsu 		/*
9251e78ac21SJeffrey Hsu 		 * Fragment large datagrams such that each segment
9261e78ac21SJeffrey Hsu 		 * contains a multiple of PAGE_SIZE amount of data,
9271e78ac21SJeffrey Hsu 		 * plus headers. This enables a receiver to perform
9281e78ac21SJeffrey Hsu 		 * page-flipping zero-copy optimizations.
9291e78ac21SJeffrey Hsu 		 *
9301e78ac21SJeffrey Hsu 		 * XXX When does this help given that sender and receiver
9311e78ac21SJeffrey Hsu 		 * could have different page sizes, and also mtu could
9321e78ac21SJeffrey Hsu 		 * be less than the receiver's page size ?
9331e78ac21SJeffrey Hsu 		 */
9341e78ac21SJeffrey Hsu 		int newlen;
9351e78ac21SJeffrey Hsu 
9369c0f6aa7SHans Petter Selasky 		off = MIN(mtu, m0->m_pkthdr.len);
9371e78ac21SJeffrey Hsu 
9381e78ac21SJeffrey Hsu 		/*
9391e78ac21SJeffrey Hsu 		 * firstlen (off - hlen) must be aligned on an
9401e78ac21SJeffrey Hsu 		 * 8-byte boundary
9411e78ac21SJeffrey Hsu 		 */
9421e78ac21SJeffrey Hsu 		if (off < hlen)
9431e78ac21SJeffrey Hsu 			goto smart_frag_failure;
9441e78ac21SJeffrey Hsu 		off = ((off - hlen) & ~7) + hlen;
9451e78ac21SJeffrey Hsu 		newlen = (~PAGE_MASK) & mtu;
9461e78ac21SJeffrey Hsu 		if ((newlen + sizeof (struct ip)) > mtu) {
9471e78ac21SJeffrey Hsu 			/* we failed, go back the default */
9481e78ac21SJeffrey Hsu smart_frag_failure:
9491e78ac21SJeffrey Hsu 			newlen = len;
9501e78ac21SJeffrey Hsu 			off = hlen + len;
9511e78ac21SJeffrey Hsu 		}
9521e78ac21SJeffrey Hsu 		len = newlen;
9531e78ac21SJeffrey Hsu 
9541e78ac21SJeffrey Hsu 	} else {
9551e78ac21SJeffrey Hsu 		off = hlen + len;
9561e78ac21SJeffrey Hsu 	}
9571e78ac21SJeffrey Hsu 
9581e78ac21SJeffrey Hsu 	firstlen = off - hlen;
9591e78ac21SJeffrey Hsu 	mnext = &m0->m_nextpkt;		/* pointer to next packet */
9601e78ac21SJeffrey Hsu 
9611e78ac21SJeffrey Hsu 	/*
9621e78ac21SJeffrey Hsu 	 * Loop through length of segment after first fragment,
9631e78ac21SJeffrey Hsu 	 * make new header and copy data of each part and link onto chain.
9641e78ac21SJeffrey Hsu 	 * Here, m0 is the original packet, m is the fragment being created.
9651e78ac21SJeffrey Hsu 	 * The fragments are linked off the m_nextpkt of the original
9661e78ac21SJeffrey Hsu 	 * packet, which after processing serves as the first fragment.
9671e78ac21SJeffrey Hsu 	 */
96821d172a3SGleb Smirnoff 	for (nfrags = 1; off < ip_len; off += len, nfrags++) {
9691e78ac21SJeffrey Hsu 		struct ip *mhip;	/* ip header on the fragment */
9701e78ac21SJeffrey Hsu 		struct mbuf *m;
9711e78ac21SJeffrey Hsu 		int mhlen = sizeof (struct ip);
9721e78ac21SJeffrey Hsu 
973dc4ad05eSGleb Smirnoff 		m = m_gethdr(M_NOWAIT, MT_DATA);
9740b17fba7SAndre Oppermann 		if (m == NULL) {
9751e78ac21SJeffrey Hsu 			error = ENOBUFS;
97686425c62SRobert Watson 			IPSTAT_INC(ips_odropped);
9771e78ac21SJeffrey Hsu 			goto done;
9781e78ac21SJeffrey Hsu 		}
979c4c4346fSHans Petter Selasky 		/*
980c4c4346fSHans Petter Selasky 		 * Make sure the complete packet header gets copied
981c4c4346fSHans Petter Selasky 		 * from the originating mbuf to the newly created
982c4c4346fSHans Petter Selasky 		 * mbuf. This also ensures that existing firewall
983c4c4346fSHans Petter Selasky 		 * classification(s), VLAN tags and so on get copied
984c4c4346fSHans Petter Selasky 		 * to the resulting fragmented packet(s):
985c4c4346fSHans Petter Selasky 		 */
986c4c4346fSHans Petter Selasky 		if (m_dup_pkthdr(m, m0, M_NOWAIT) == 0) {
987c4c4346fSHans Petter Selasky 			m_free(m);
988c4c4346fSHans Petter Selasky 			error = ENOBUFS;
989c4c4346fSHans Petter Selasky 			IPSTAT_INC(ips_odropped);
990c4c4346fSHans Petter Selasky 			goto done;
991c4c4346fSHans Petter Selasky 		}
9921e78ac21SJeffrey Hsu 		/*
9931e78ac21SJeffrey Hsu 		 * In the first mbuf, leave room for the link header, then
9941e78ac21SJeffrey Hsu 		 * copy the original IP header including options. The payload
9958b889dbbSBruce M Simpson 		 * goes into an additional mbuf chain returned by m_copym().
9961e78ac21SJeffrey Hsu 		 */
9971e78ac21SJeffrey Hsu 		m->m_data += max_linkhdr;
9981e78ac21SJeffrey Hsu 		mhip = mtod(m, struct ip *);
9991e78ac21SJeffrey Hsu 		*mhip = *ip;
10001e78ac21SJeffrey Hsu 		if (hlen > sizeof (struct ip)) {
10011e78ac21SJeffrey Hsu 			mhlen = ip_optcopy(ip, mhip) + sizeof (struct ip);
10021e78ac21SJeffrey Hsu 			mhip->ip_v = IPVERSION;
10031e78ac21SJeffrey Hsu 			mhip->ip_hl = mhlen >> 2;
10041e78ac21SJeffrey Hsu 		}
10051e78ac21SJeffrey Hsu 		m->m_len = mhlen;
100621d172a3SGleb Smirnoff 		/* XXX do we need to add ip_off below ? */
100721d172a3SGleb Smirnoff 		mhip->ip_off = ((off - hlen) >> 3) + ip_off;
1008fb86dfcdSAndre Oppermann 		if (off + len >= ip_len)
100921d172a3SGleb Smirnoff 			len = ip_len - off;
1010fb86dfcdSAndre Oppermann 		else
10111e78ac21SJeffrey Hsu 			mhip->ip_off |= IP_MF;
10121e78ac21SJeffrey Hsu 		mhip->ip_len = htons((u_short)(len + mhlen));
1013eb1b1807SGleb Smirnoff 		m->m_next = m_copym(m0, off, len, M_NOWAIT);
10140b17fba7SAndre Oppermann 		if (m->m_next == NULL) {	/* copy failed */
10151e78ac21SJeffrey Hsu 			m_free(m);
10161e78ac21SJeffrey Hsu 			error = ENOBUFS;	/* ??? */
101786425c62SRobert Watson 			IPSTAT_INC(ips_odropped);
10181e78ac21SJeffrey Hsu 			goto done;
10191e78ac21SJeffrey Hsu 		}
10201e78ac21SJeffrey Hsu 		m->m_pkthdr.len = mhlen + len;
10211e78ac21SJeffrey Hsu #ifdef MAC
102230d239bcSRobert Watson 		mac_netinet_fragment(m0, m);
10231e78ac21SJeffrey Hsu #endif
10241e78ac21SJeffrey Hsu 		mhip->ip_off = htons(mhip->ip_off);
10251e78ac21SJeffrey Hsu 		mhip->ip_sum = 0;
1026078468edSGleb Smirnoff 		if (m->m_pkthdr.csum_flags & CSUM_IP & ~if_hwassist_flags) {
10271e78ac21SJeffrey Hsu 			mhip->ip_sum = in_cksum(m, mhlen);
1028078468edSGleb Smirnoff 			m->m_pkthdr.csum_flags &= ~CSUM_IP;
1029078468edSGleb Smirnoff 		}
10301e78ac21SJeffrey Hsu 		*mnext = m;
10311e78ac21SJeffrey Hsu 		mnext = &m->m_nextpkt;
10321e78ac21SJeffrey Hsu 	}
103386425c62SRobert Watson 	IPSTAT_ADD(ips_ofragments, nfrags);
10341e78ac21SJeffrey Hsu 
10351e78ac21SJeffrey Hsu 	/*
10361e78ac21SJeffrey Hsu 	 * Update first fragment by trimming what's been copied out
10371e78ac21SJeffrey Hsu 	 * and updating header.
10381e78ac21SJeffrey Hsu 	 */
103921d172a3SGleb Smirnoff 	m_adj(m0, hlen + firstlen - ip_len);
10401e78ac21SJeffrey Hsu 	m0->m_pkthdr.len = hlen + firstlen;
10411e78ac21SJeffrey Hsu 	ip->ip_len = htons((u_short)m0->m_pkthdr.len);
104221d172a3SGleb Smirnoff 	ip->ip_off = htons(ip_off | IP_MF);
10431e78ac21SJeffrey Hsu 	ip->ip_sum = 0;
1044078468edSGleb Smirnoff 	if (m0->m_pkthdr.csum_flags & CSUM_IP & ~if_hwassist_flags) {
10451e78ac21SJeffrey Hsu 		ip->ip_sum = in_cksum(m0, hlen);
1046078468edSGleb Smirnoff 		m0->m_pkthdr.csum_flags &= ~CSUM_IP;
1047078468edSGleb Smirnoff 	}
10481e78ac21SJeffrey Hsu 
10491e78ac21SJeffrey Hsu done:
10501e78ac21SJeffrey Hsu 	*m_frag = m0;
10511e78ac21SJeffrey Hsu 	return error;
10521e78ac21SJeffrey Hsu }
10531e78ac21SJeffrey Hsu 
10541c238475SJonathan Lemon void
1055db4f9cc7SJonathan Lemon in_delayed_cksum(struct mbuf *m)
1056db4f9cc7SJonathan Lemon {
1057db4f9cc7SJonathan Lemon 	struct ip *ip;
10581fdbfb90STom Jones 	struct udphdr *uh;
10591fdbfb90STom Jones 	uint16_t cklen, csum, offset;
1060db4f9cc7SJonathan Lemon 
1061db4f9cc7SJonathan Lemon 	ip = mtod(m, struct ip *);
106253be11f6SPoul-Henning Kamp 	offset = ip->ip_hl << 2 ;
10631fdbfb90STom Jones 
10641fdbfb90STom Jones 	if (m->m_pkthdr.csum_flags & CSUM_UDP) {
10651fdbfb90STom Jones 		/* if udp header is not in the first mbuf copy udplen */
1066b6e87011STom Jones 		if (offset + sizeof(struct udphdr) > m->m_len) {
1067590d0a43SAndrey V. Elsukov 			m_copydata(m, offset + offsetof(struct udphdr,
1068590d0a43SAndrey V. Elsukov 			    uh_ulen), sizeof(cklen), (caddr_t)&cklen);
1069b6e87011STom Jones 			cklen = ntohs(cklen);
1070b6e87011STom Jones 		} else {
1071590d0a43SAndrey V. Elsukov 			uh = (struct udphdr *)mtodo(m, offset);
10721fdbfb90STom Jones 			cklen = ntohs(uh->uh_ulen);
10731fdbfb90STom Jones 		}
10741fdbfb90STom Jones 		csum = in_cksum_skip(m, cklen + offset, offset);
10751fdbfb90STom Jones 		if (csum == 0)
1076206a3274SRuslan Ermilov 			csum = 0xffff;
10771fdbfb90STom Jones 	} else {
10781fdbfb90STom Jones 		cklen = ntohs(ip->ip_len);
10791fdbfb90STom Jones 		csum = in_cksum_skip(m, cklen, offset);
10801fdbfb90STom Jones 	}
1081db4f9cc7SJonathan Lemon 	offset += m->m_pkthdr.csum_data;	/* checksum offset */
1082db4f9cc7SJonathan Lemon 
1083590d0a43SAndrey V. Elsukov 	if (offset + sizeof(csum) > m->m_len)
1084590d0a43SAndrey V. Elsukov 		m_copyback(m, offset, sizeof(csum), (caddr_t)&csum);
1085590d0a43SAndrey V. Elsukov 	else
1086590d0a43SAndrey V. Elsukov 		*(u_short *)mtodo(m, offset) = csum;
1087db4f9cc7SJonathan Lemon }
1088db4f9cc7SJonathan Lemon 
1089df8bae1dSRodney W. Grimes /*
1090df8bae1dSRodney W. Grimes  * IP socket option processing.
1091df8bae1dSRodney W. Grimes  */
1092df8bae1dSRodney W. Grimes int
1093f2565d68SRobert Watson ip_ctloutput(struct socket *so, struct sockopt *sopt)
1094df8bae1dSRodney W. Grimes {
1095cfe8b629SGarrett Wollman 	struct inpcb *inp = sotoinpcb(so);
1096cfe8b629SGarrett Wollman 	int	error, optval;
1097dc847eb6SAdrian Chadd #ifdef	RSS
1098dc847eb6SAdrian Chadd 	uint32_t rss_bucket;
1099dc847eb6SAdrian Chadd 	int retval;
1100dc847eb6SAdrian Chadd #endif
1101df8bae1dSRodney W. Grimes 
1102cfe8b629SGarrett Wollman 	error = optval = 0;
1103cfe8b629SGarrett Wollman 	if (sopt->sopt_level != IPPROTO_IP) {
1104fc06cd42SMikolaj Golub 		error = EINVAL;
1105fc06cd42SMikolaj Golub 
1106fc06cd42SMikolaj Golub 		if (sopt->sopt_level == SOL_SOCKET &&
1107fc06cd42SMikolaj Golub 		    sopt->sopt_dir == SOPT_SET) {
1108fc06cd42SMikolaj Golub 			switch (sopt->sopt_name) {
1109fc06cd42SMikolaj Golub 			case SO_REUSEADDR:
1110fc06cd42SMikolaj Golub 				INP_WLOCK(inp);
1111efdf104bSMikolaj Golub 				if ((so->so_options & SO_REUSEADDR) != 0)
1112efdf104bSMikolaj Golub 					inp->inp_flags2 |= INP_REUSEADDR;
1113fc06cd42SMikolaj Golub 				else
1114efdf104bSMikolaj Golub 					inp->inp_flags2 &= ~INP_REUSEADDR;
1115fc06cd42SMikolaj Golub 				INP_WUNLOCK(inp);
1116fc06cd42SMikolaj Golub 				error = 0;
1117fc06cd42SMikolaj Golub 				break;
1118fc06cd42SMikolaj Golub 			case SO_REUSEPORT:
1119fc06cd42SMikolaj Golub 				INP_WLOCK(inp);
1120fc06cd42SMikolaj Golub 				if ((so->so_options & SO_REUSEPORT) != 0)
1121fc06cd42SMikolaj Golub 					inp->inp_flags2 |= INP_REUSEPORT;
1122fc06cd42SMikolaj Golub 				else
1123fc06cd42SMikolaj Golub 					inp->inp_flags2 &= ~INP_REUSEPORT;
1124fc06cd42SMikolaj Golub 				INP_WUNLOCK(inp);
1125fc06cd42SMikolaj Golub 				error = 0;
1126fc06cd42SMikolaj Golub 				break;
11271a43cff9SSean Bruno 			case SO_REUSEPORT_LB:
11281a43cff9SSean Bruno 				INP_WLOCK(inp);
11291a43cff9SSean Bruno 				if ((so->so_options & SO_REUSEPORT_LB) != 0)
11301a43cff9SSean Bruno 					inp->inp_flags2 |= INP_REUSEPORT_LB;
11311a43cff9SSean Bruno 				else
11321a43cff9SSean Bruno 					inp->inp_flags2 &= ~INP_REUSEPORT_LB;
11331a43cff9SSean Bruno 				INP_WUNLOCK(inp);
11341a43cff9SSean Bruno 				error = 0;
11351a43cff9SSean Bruno 				break;
1136fc06cd42SMikolaj Golub 			case SO_SETFIB:
1137fc06cd42SMikolaj Golub 				INP_WLOCK(inp);
1138fc06cd42SMikolaj Golub 				inp->inp_inc.inc_fibnum = so->so_fibnum;
1139fc06cd42SMikolaj Golub 				INP_WUNLOCK(inp);
1140fc06cd42SMikolaj Golub 				error = 0;
1141fc06cd42SMikolaj Golub 				break;
1142f3e7afe2SHans Petter Selasky 			case SO_MAX_PACING_RATE:
1143f3e7afe2SHans Petter Selasky #ifdef RATELIMIT
1144f3e7afe2SHans Petter Selasky 				INP_WLOCK(inp);
1145f3e7afe2SHans Petter Selasky 				inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
1146f3e7afe2SHans Petter Selasky 				INP_WUNLOCK(inp);
1147f3e7afe2SHans Petter Selasky 				error = 0;
1148f3e7afe2SHans Petter Selasky #else
1149f3e7afe2SHans Petter Selasky 				error = EOPNOTSUPP;
1150f3e7afe2SHans Petter Selasky #endif
1151f3e7afe2SHans Petter Selasky 				break;
1152fc06cd42SMikolaj Golub 			default:
1153fc06cd42SMikolaj Golub 				break;
1154fc06cd42SMikolaj Golub 			}
1155fc06cd42SMikolaj Golub 		}
1156fc06cd42SMikolaj Golub 		return (error);
1157cfe8b629SGarrett Wollman 	}
1158df8bae1dSRodney W. Grimes 
1159cfe8b629SGarrett Wollman 	switch (sopt->sopt_dir) {
1160cfe8b629SGarrett Wollman 	case SOPT_SET:
1161cfe8b629SGarrett Wollman 		switch (sopt->sopt_name) {
1162df8bae1dSRodney W. Grimes 		case IP_OPTIONS:
1163df8bae1dSRodney W. Grimes #ifdef notyet
1164df8bae1dSRodney W. Grimes 		case IP_RETOPTS:
1165df8bae1dSRodney W. Grimes #endif
1166cfe8b629SGarrett Wollman 		{
1167cfe8b629SGarrett Wollman 			struct mbuf *m;
1168cfe8b629SGarrett Wollman 			if (sopt->sopt_valsize > MLEN) {
1169cfe8b629SGarrett Wollman 				error = EMSGSIZE;
1170cfe8b629SGarrett Wollman 				break;
1171cfe8b629SGarrett Wollman 			}
1172dc4ad05eSGleb Smirnoff 			m = m_get(sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
11730b17fba7SAndre Oppermann 			if (m == NULL) {
1174cfe8b629SGarrett Wollman 				error = ENOBUFS;
1175cfe8b629SGarrett Wollman 				break;
1176cfe8b629SGarrett Wollman 			}
1177cfe8b629SGarrett Wollman 			m->m_len = sopt->sopt_valsize;
1178cfe8b629SGarrett Wollman 			error = sooptcopyin(sopt, mtod(m, char *), m->m_len,
1179cfe8b629SGarrett Wollman 					    m->m_len);
1180635354c4SMaxim Konovalov 			if (error) {
1181635354c4SMaxim Konovalov 				m_free(m);
1182635354c4SMaxim Konovalov 				break;
1183635354c4SMaxim Konovalov 			}
11848501a69cSRobert Watson 			INP_WLOCK(inp);
1185993d9505SRobert Watson 			error = ip_pcbopts(inp, sopt->sopt_name, m);
11868501a69cSRobert Watson 			INP_WUNLOCK(inp);
1187993d9505SRobert Watson 			return (error);
1188cfe8b629SGarrett Wollman 		}
1189df8bae1dSRodney W. Grimes 
1190f44270e7SPawel Jakub Dawidek 		case IP_BINDANY:
1191f44270e7SPawel Jakub Dawidek 			if (sopt->sopt_td != NULL) {
1192f44270e7SPawel Jakub Dawidek 				error = priv_check(sopt->sopt_td,
1193f44270e7SPawel Jakub Dawidek 				    PRIV_NETINET_BINDANY);
1194f44270e7SPawel Jakub Dawidek 				if (error)
1195be9347e3SAdrian Chadd 					break;
1196be9347e3SAdrian Chadd 			}
1197cef27294SAdrian Chadd 			/* FALLTHROUGH */
11980a100a6fSAdrian Chadd 		case IP_BINDMULTI:
11990a100a6fSAdrian Chadd #ifdef	RSS
12000a100a6fSAdrian Chadd 		case IP_RSS_LISTEN_BUCKET:
12010a100a6fSAdrian Chadd #endif
1202df8bae1dSRodney W. Grimes 		case IP_TOS:
1203df8bae1dSRodney W. Grimes 		case IP_TTL:
1204936cd18dSAndre Oppermann 		case IP_MINTTL:
1205df8bae1dSRodney W. Grimes 		case IP_RECVOPTS:
1206df8bae1dSRodney W. Grimes 		case IP_RECVRETOPTS:
1207dce33a45SErmal Luçi 		case IP_ORIGDSTADDR:
1208df8bae1dSRodney W. Grimes 		case IP_RECVDSTADDR:
12094957466bSMatthew N. Dodd 		case IP_RECVTTL:
121082c23ebaSBill Fenner 		case IP_RECVIF:
12118afa2304SBruce M Simpson 		case IP_ONESBCAST:
1212b2828ad2SAndre Oppermann 		case IP_DONTFRAG:
12133cca425bSMichael Tuexen 		case IP_RECVTOS:
12149d3ddf43SAdrian Chadd 		case IP_RECVFLOWID:
12159d3ddf43SAdrian Chadd #ifdef	RSS
12169d3ddf43SAdrian Chadd 		case IP_RECVRSSBUCKETID:
12179d3ddf43SAdrian Chadd #endif
1218cfe8b629SGarrett Wollman 			error = sooptcopyin(sopt, &optval, sizeof optval,
1219cfe8b629SGarrett Wollman 					    sizeof optval);
1220cfe8b629SGarrett Wollman 			if (error)
1221cfe8b629SGarrett Wollman 				break;
1222df8bae1dSRodney W. Grimes 
1223cfe8b629SGarrett Wollman 			switch (sopt->sopt_name) {
1224df8bae1dSRodney W. Grimes 			case IP_TOS:
1225ca98b82cSDavid Greenman 				inp->inp_ip_tos = optval;
1226df8bae1dSRodney W. Grimes 				break;
1227df8bae1dSRodney W. Grimes 
1228df8bae1dSRodney W. Grimes 			case IP_TTL:
1229ca98b82cSDavid Greenman 				inp->inp_ip_ttl = optval;
1230df8bae1dSRodney W. Grimes 				break;
1231936cd18dSAndre Oppermann 
1232936cd18dSAndre Oppermann 			case IP_MINTTL:
1233a603c811SRobert Watson 				if (optval >= 0 && optval <= MAXTTL)
1234936cd18dSAndre Oppermann 					inp->inp_ip_minttl = optval;
1235936cd18dSAndre Oppermann 				else
1236936cd18dSAndre Oppermann 					error = EINVAL;
1237936cd18dSAndre Oppermann 				break;
1238936cd18dSAndre Oppermann 
1239a138d217SRobert Watson #define	OPTSET(bit) do {						\
12408501a69cSRobert Watson 	INP_WLOCK(inp);							\
1241df8bae1dSRodney W. Grimes 	if (optval)							\
1242df8bae1dSRodney W. Grimes 		inp->inp_flags |= bit;					\
1243df8bae1dSRodney W. Grimes 	else								\
1244a138d217SRobert Watson 		inp->inp_flags &= ~bit;					\
12458501a69cSRobert Watson 	INP_WUNLOCK(inp);						\
1246a138d217SRobert Watson } while (0)
1247df8bae1dSRodney W. Grimes 
12480a100a6fSAdrian Chadd #define	OPTSET2(bit, val) do {						\
12490a100a6fSAdrian Chadd 	INP_WLOCK(inp);							\
12500a100a6fSAdrian Chadd 	if (val)							\
12510a100a6fSAdrian Chadd 		inp->inp_flags2 |= bit;					\
12520a100a6fSAdrian Chadd 	else								\
12530a100a6fSAdrian Chadd 		inp->inp_flags2 &= ~bit;				\
12540a100a6fSAdrian Chadd 	INP_WUNLOCK(inp);						\
12550a100a6fSAdrian Chadd } while (0)
12560a100a6fSAdrian Chadd 
1257df8bae1dSRodney W. Grimes 			case IP_RECVOPTS:
1258df8bae1dSRodney W. Grimes 				OPTSET(INP_RECVOPTS);
1259df8bae1dSRodney W. Grimes 				break;
1260df8bae1dSRodney W. Grimes 
1261df8bae1dSRodney W. Grimes 			case IP_RECVRETOPTS:
1262df8bae1dSRodney W. Grimes 				OPTSET(INP_RECVRETOPTS);
1263df8bae1dSRodney W. Grimes 				break;
1264df8bae1dSRodney W. Grimes 
1265df8bae1dSRodney W. Grimes 			case IP_RECVDSTADDR:
1266df8bae1dSRodney W. Grimes 				OPTSET(INP_RECVDSTADDR);
1267df8bae1dSRodney W. Grimes 				break;
126882c23ebaSBill Fenner 
1269dce33a45SErmal Luçi 			case IP_ORIGDSTADDR:
1270dce33a45SErmal Luçi 				OPTSET2(INP_ORIGDSTADDR, optval);
1271dce33a45SErmal Luçi 				break;
1272dce33a45SErmal Luçi 
12734957466bSMatthew N. Dodd 			case IP_RECVTTL:
12744957466bSMatthew N. Dodd 				OPTSET(INP_RECVTTL);
12754957466bSMatthew N. Dodd 				break;
12764957466bSMatthew N. Dodd 
127782c23ebaSBill Fenner 			case IP_RECVIF:
127882c23ebaSBill Fenner 				OPTSET(INP_RECVIF);
127982c23ebaSBill Fenner 				break;
12806a800098SYoshinobu Inoue 
12818afa2304SBruce M Simpson 			case IP_ONESBCAST:
12828afa2304SBruce M Simpson 				OPTSET(INP_ONESBCAST);
12838afa2304SBruce M Simpson 				break;
1284b2828ad2SAndre Oppermann 			case IP_DONTFRAG:
1285b2828ad2SAndre Oppermann 				OPTSET(INP_DONTFRAG);
1286b2828ad2SAndre Oppermann 				break;
1287f44270e7SPawel Jakub Dawidek 			case IP_BINDANY:
1288f44270e7SPawel Jakub Dawidek 				OPTSET(INP_BINDANY);
1289be9347e3SAdrian Chadd 				break;
12903cca425bSMichael Tuexen 			case IP_RECVTOS:
12913cca425bSMichael Tuexen 				OPTSET(INP_RECVTOS);
12923cca425bSMichael Tuexen 				break;
12930a100a6fSAdrian Chadd 			case IP_BINDMULTI:
12940a100a6fSAdrian Chadd 				OPTSET2(INP_BINDMULTI, optval);
12950a100a6fSAdrian Chadd 				break;
12969d3ddf43SAdrian Chadd 			case IP_RECVFLOWID:
12979d3ddf43SAdrian Chadd 				OPTSET2(INP_RECVFLOWID, optval);
12989d3ddf43SAdrian Chadd 				break;
12990a100a6fSAdrian Chadd #ifdef	RSS
13000a100a6fSAdrian Chadd 			case IP_RSS_LISTEN_BUCKET:
13010a100a6fSAdrian Chadd 				if ((optval >= 0) &&
13020a100a6fSAdrian Chadd 				    (optval < rss_getnumbuckets())) {
13030a100a6fSAdrian Chadd 					inp->inp_rss_listen_bucket = optval;
13040a100a6fSAdrian Chadd 					OPTSET2(INP_RSS_BUCKET_SET, 1);
13050a100a6fSAdrian Chadd 				} else {
13060a100a6fSAdrian Chadd 					error = EINVAL;
13070a100a6fSAdrian Chadd 				}
13080a100a6fSAdrian Chadd 				break;
13099d3ddf43SAdrian Chadd 			case IP_RECVRSSBUCKETID:
13109d3ddf43SAdrian Chadd 				OPTSET2(INP_RECVRSSBUCKETID, optval);
13119d3ddf43SAdrian Chadd 				break;
13120a100a6fSAdrian Chadd #endif
1313df8bae1dSRodney W. Grimes 			}
1314df8bae1dSRodney W. Grimes 			break;
1315df8bae1dSRodney W. Grimes #undef OPTSET
13160a100a6fSAdrian Chadd #undef OPTSET2
1317df8bae1dSRodney W. Grimes 
131871498f30SBruce M Simpson 		/*
131971498f30SBruce M Simpson 		 * Multicast socket options are processed by the in_mcast
132071498f30SBruce M Simpson 		 * module.
132171498f30SBruce M Simpson 		 */
1322df8bae1dSRodney W. Grimes 		case IP_MULTICAST_IF:
1323f0068c4aSGarrett Wollman 		case IP_MULTICAST_VIF:
1324df8bae1dSRodney W. Grimes 		case IP_MULTICAST_TTL:
1325df8bae1dSRodney W. Grimes 		case IP_MULTICAST_LOOP:
1326df8bae1dSRodney W. Grimes 		case IP_ADD_MEMBERSHIP:
1327df8bae1dSRodney W. Grimes 		case IP_DROP_MEMBERSHIP:
132871498f30SBruce M Simpson 		case IP_ADD_SOURCE_MEMBERSHIP:
132971498f30SBruce M Simpson 		case IP_DROP_SOURCE_MEMBERSHIP:
133071498f30SBruce M Simpson 		case IP_BLOCK_SOURCE:
133171498f30SBruce M Simpson 		case IP_UNBLOCK_SOURCE:
133271498f30SBruce M Simpson 		case IP_MSFILTER:
133371498f30SBruce M Simpson 		case MCAST_JOIN_GROUP:
133471498f30SBruce M Simpson 		case MCAST_LEAVE_GROUP:
133571498f30SBruce M Simpson 		case MCAST_JOIN_SOURCE_GROUP:
133671498f30SBruce M Simpson 		case MCAST_LEAVE_SOURCE_GROUP:
133771498f30SBruce M Simpson 		case MCAST_BLOCK_SOURCE:
133871498f30SBruce M Simpson 		case MCAST_UNBLOCK_SOURCE:
133971498f30SBruce M Simpson 			error = inp_setmoptions(inp, sopt);
1340df8bae1dSRodney W. Grimes 			break;
1341df8bae1dSRodney W. Grimes 
134233b3ac06SPeter Wemm 		case IP_PORTRANGE:
1343cfe8b629SGarrett Wollman 			error = sooptcopyin(sopt, &optval, sizeof optval,
1344cfe8b629SGarrett Wollman 					    sizeof optval);
1345cfe8b629SGarrett Wollman 			if (error)
1346cfe8b629SGarrett Wollman 				break;
134733b3ac06SPeter Wemm 
13488501a69cSRobert Watson 			INP_WLOCK(inp);
134933b3ac06SPeter Wemm 			switch (optval) {
135033b3ac06SPeter Wemm 			case IP_PORTRANGE_DEFAULT:
135133b3ac06SPeter Wemm 				inp->inp_flags &= ~(INP_LOWPORT);
135233b3ac06SPeter Wemm 				inp->inp_flags &= ~(INP_HIGHPORT);
135333b3ac06SPeter Wemm 				break;
135433b3ac06SPeter Wemm 
135533b3ac06SPeter Wemm 			case IP_PORTRANGE_HIGH:
135633b3ac06SPeter Wemm 				inp->inp_flags &= ~(INP_LOWPORT);
135733b3ac06SPeter Wemm 				inp->inp_flags |= INP_HIGHPORT;
135833b3ac06SPeter Wemm 				break;
135933b3ac06SPeter Wemm 
136033b3ac06SPeter Wemm 			case IP_PORTRANGE_LOW:
136133b3ac06SPeter Wemm 				inp->inp_flags &= ~(INP_HIGHPORT);
136233b3ac06SPeter Wemm 				inp->inp_flags |= INP_LOWPORT;
136333b3ac06SPeter Wemm 				break;
136433b3ac06SPeter Wemm 
136533b3ac06SPeter Wemm 			default:
136633b3ac06SPeter Wemm 				error = EINVAL;
136733b3ac06SPeter Wemm 				break;
136833b3ac06SPeter Wemm 			}
13698501a69cSRobert Watson 			INP_WUNLOCK(inp);
1370ce8c72b1SPeter Wemm 			break;
137133b3ac06SPeter Wemm 
1372fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
13736a800098SYoshinobu Inoue 		case IP_IPSEC_POLICY:
1374fcf59617SAndrey V. Elsukov 			if (IPSEC_ENABLED(ipv4)) {
1375fcf59617SAndrey V. Elsukov 				error = IPSEC_PCBCTL(ipv4, inp, sopt);
13766a800098SYoshinobu Inoue 				break;
13776a800098SYoshinobu Inoue 			}
1378fcf59617SAndrey V. Elsukov 			/* FALLTHROUGH */
1379b2630c29SGeorge V. Neville-Neil #endif /* IPSEC */
13806a800098SYoshinobu Inoue 
1381df8bae1dSRodney W. Grimes 		default:
1382df8bae1dSRodney W. Grimes 			error = ENOPROTOOPT;
1383df8bae1dSRodney W. Grimes 			break;
1384df8bae1dSRodney W. Grimes 		}
1385df8bae1dSRodney W. Grimes 		break;
1386df8bae1dSRodney W. Grimes 
1387cfe8b629SGarrett Wollman 	case SOPT_GET:
1388cfe8b629SGarrett Wollman 		switch (sopt->sopt_name) {
1389df8bae1dSRodney W. Grimes 		case IP_OPTIONS:
1390df8bae1dSRodney W. Grimes 		case IP_RETOPTS:
1391e5e3e746SMatt Macy 			INP_RLOCK(inp);
1392e5e3e746SMatt Macy 			if (inp->inp_options) {
1393e5e3e746SMatt Macy 				struct mbuf *options;
1394e5e3e746SMatt Macy 
139510731c54SMichael Tuexen 				options = m_copym(inp->inp_options, 0,
139610731c54SMichael Tuexen 				    M_COPYALL, M_NOWAIT);
1397e5e3e746SMatt Macy 				INP_RUNLOCK(inp);
1398e5e3e746SMatt Macy 				if (options != NULL) {
1399cfe8b629SGarrett Wollman 					error = sooptcopyout(sopt,
1400e5e3e746SMatt Macy 							     mtod(options, char *),
1401e5e3e746SMatt Macy 							     options->m_len);
1402e5e3e746SMatt Macy 					m_freem(options);
1403e5e3e746SMatt Macy 				} else
1404e5e3e746SMatt Macy 					error = ENOMEM;
1405e5e3e746SMatt Macy 			} else {
1406e5e3e746SMatt Macy 				INP_RUNLOCK(inp);
1407cfe8b629SGarrett Wollman 				sopt->sopt_valsize = 0;
1408e5e3e746SMatt Macy 			}
1409df8bae1dSRodney W. Grimes 			break;
1410df8bae1dSRodney W. Grimes 
1411df8bae1dSRodney W. Grimes 		case IP_TOS:
1412df8bae1dSRodney W. Grimes 		case IP_TTL:
1413936cd18dSAndre Oppermann 		case IP_MINTTL:
1414df8bae1dSRodney W. Grimes 		case IP_RECVOPTS:
1415df8bae1dSRodney W. Grimes 		case IP_RECVRETOPTS:
1416dce33a45SErmal Luçi 		case IP_ORIGDSTADDR:
1417df8bae1dSRodney W. Grimes 		case IP_RECVDSTADDR:
14184957466bSMatthew N. Dodd 		case IP_RECVTTL:
141982c23ebaSBill Fenner 		case IP_RECVIF:
1420cfe8b629SGarrett Wollman 		case IP_PORTRANGE:
14218afa2304SBruce M Simpson 		case IP_ONESBCAST:
1422b2828ad2SAndre Oppermann 		case IP_DONTFRAG:
14235f6bf451SAttilio Rao 		case IP_BINDANY:
14243cca425bSMichael Tuexen 		case IP_RECVTOS:
14250a100a6fSAdrian Chadd 		case IP_BINDMULTI:
14269c423972SAdrian Chadd 		case IP_FLOWID:
14279c423972SAdrian Chadd 		case IP_FLOWTYPE:
14289d3ddf43SAdrian Chadd 		case IP_RECVFLOWID:
14290a100a6fSAdrian Chadd #ifdef	RSS
14300a100a6fSAdrian Chadd 		case IP_RSSBUCKETID:
14319d3ddf43SAdrian Chadd 		case IP_RECVRSSBUCKETID:
14320a100a6fSAdrian Chadd #endif
1433cfe8b629SGarrett Wollman 			switch (sopt->sopt_name) {
1434df8bae1dSRodney W. Grimes 			case IP_TOS:
1435ca98b82cSDavid Greenman 				optval = inp->inp_ip_tos;
1436df8bae1dSRodney W. Grimes 				break;
1437df8bae1dSRodney W. Grimes 
1438df8bae1dSRodney W. Grimes 			case IP_TTL:
1439ca98b82cSDavid Greenman 				optval = inp->inp_ip_ttl;
1440df8bae1dSRodney W. Grimes 				break;
1441df8bae1dSRodney W. Grimes 
1442936cd18dSAndre Oppermann 			case IP_MINTTL:
1443936cd18dSAndre Oppermann 				optval = inp->inp_ip_minttl;
1444936cd18dSAndre Oppermann 				break;
1445936cd18dSAndre Oppermann 
1446df8bae1dSRodney W. Grimes #define	OPTBIT(bit)	(inp->inp_flags & bit ? 1 : 0)
14470a100a6fSAdrian Chadd #define	OPTBIT2(bit)	(inp->inp_flags2 & bit ? 1 : 0)
1448df8bae1dSRodney W. Grimes 
1449df8bae1dSRodney W. Grimes 			case IP_RECVOPTS:
1450df8bae1dSRodney W. Grimes 				optval = OPTBIT(INP_RECVOPTS);
1451df8bae1dSRodney W. Grimes 				break;
1452df8bae1dSRodney W. Grimes 
1453df8bae1dSRodney W. Grimes 			case IP_RECVRETOPTS:
1454df8bae1dSRodney W. Grimes 				optval = OPTBIT(INP_RECVRETOPTS);
1455df8bae1dSRodney W. Grimes 				break;
1456df8bae1dSRodney W. Grimes 
1457df8bae1dSRodney W. Grimes 			case IP_RECVDSTADDR:
1458df8bae1dSRodney W. Grimes 				optval = OPTBIT(INP_RECVDSTADDR);
1459df8bae1dSRodney W. Grimes 				break;
146082c23ebaSBill Fenner 
1461dce33a45SErmal Luçi 			case IP_ORIGDSTADDR:
1462dce33a45SErmal Luçi 				optval = OPTBIT2(INP_ORIGDSTADDR);
1463dce33a45SErmal Luçi 				break;
1464dce33a45SErmal Luçi 
14654957466bSMatthew N. Dodd 			case IP_RECVTTL:
14664957466bSMatthew N. Dodd 				optval = OPTBIT(INP_RECVTTL);
14674957466bSMatthew N. Dodd 				break;
14684957466bSMatthew N. Dodd 
146982c23ebaSBill Fenner 			case IP_RECVIF:
147082c23ebaSBill Fenner 				optval = OPTBIT(INP_RECVIF);
147182c23ebaSBill Fenner 				break;
1472cfe8b629SGarrett Wollman 
1473cfe8b629SGarrett Wollman 			case IP_PORTRANGE:
1474cfe8b629SGarrett Wollman 				if (inp->inp_flags & INP_HIGHPORT)
1475cfe8b629SGarrett Wollman 					optval = IP_PORTRANGE_HIGH;
1476cfe8b629SGarrett Wollman 				else if (inp->inp_flags & INP_LOWPORT)
1477cfe8b629SGarrett Wollman 					optval = IP_PORTRANGE_LOW;
1478cfe8b629SGarrett Wollman 				else
1479cfe8b629SGarrett Wollman 					optval = 0;
1480cfe8b629SGarrett Wollman 				break;
14816a800098SYoshinobu Inoue 
14828afa2304SBruce M Simpson 			case IP_ONESBCAST:
14838afa2304SBruce M Simpson 				optval = OPTBIT(INP_ONESBCAST);
14848afa2304SBruce M Simpson 				break;
1485b2828ad2SAndre Oppermann 			case IP_DONTFRAG:
1486b2828ad2SAndre Oppermann 				optval = OPTBIT(INP_DONTFRAG);
1487b2828ad2SAndre Oppermann 				break;
14885f6bf451SAttilio Rao 			case IP_BINDANY:
14895f6bf451SAttilio Rao 				optval = OPTBIT(INP_BINDANY);
14905f6bf451SAttilio Rao 				break;
14913cca425bSMichael Tuexen 			case IP_RECVTOS:
14923cca425bSMichael Tuexen 				optval = OPTBIT(INP_RECVTOS);
14933cca425bSMichael Tuexen 				break;
14949c423972SAdrian Chadd 			case IP_FLOWID:
14959c423972SAdrian Chadd 				optval = inp->inp_flowid;
14969c423972SAdrian Chadd 				break;
14979c423972SAdrian Chadd 			case IP_FLOWTYPE:
14989c423972SAdrian Chadd 				optval = inp->inp_flowtype;
14999c423972SAdrian Chadd 				break;
15009d3ddf43SAdrian Chadd 			case IP_RECVFLOWID:
15019d3ddf43SAdrian Chadd 				optval = OPTBIT2(INP_RECVFLOWID);
15029d3ddf43SAdrian Chadd 				break;
15039c423972SAdrian Chadd #ifdef	RSS
15047847796aSAdrian Chadd 			case IP_RSSBUCKETID:
15057847796aSAdrian Chadd 				retval = rss_hash2bucket(inp->inp_flowid,
15067847796aSAdrian Chadd 				    inp->inp_flowtype,
15077847796aSAdrian Chadd 				    &rss_bucket);
15087847796aSAdrian Chadd 				if (retval == 0)
15097847796aSAdrian Chadd 					optval = rss_bucket;
15107847796aSAdrian Chadd 				else
15117847796aSAdrian Chadd 					error = EINVAL;
15129c423972SAdrian Chadd 				break;
15139d3ddf43SAdrian Chadd 			case IP_RECVRSSBUCKETID:
15149d3ddf43SAdrian Chadd 				optval = OPTBIT2(INP_RECVRSSBUCKETID);
15159d3ddf43SAdrian Chadd 				break;
15169c423972SAdrian Chadd #endif
15170a100a6fSAdrian Chadd 			case IP_BINDMULTI:
15180a100a6fSAdrian Chadd 				optval = OPTBIT2(INP_BINDMULTI);
15190a100a6fSAdrian Chadd 				break;
1520df8bae1dSRodney W. Grimes 			}
1521cfe8b629SGarrett Wollman 			error = sooptcopyout(sopt, &optval, sizeof optval);
1522df8bae1dSRodney W. Grimes 			break;
1523df8bae1dSRodney W. Grimes 
152471498f30SBruce M Simpson 		/*
152571498f30SBruce M Simpson 		 * Multicast socket options are processed by the in_mcast
152671498f30SBruce M Simpson 		 * module.
152771498f30SBruce M Simpson 		 */
1528df8bae1dSRodney W. Grimes 		case IP_MULTICAST_IF:
1529f0068c4aSGarrett Wollman 		case IP_MULTICAST_VIF:
1530df8bae1dSRodney W. Grimes 		case IP_MULTICAST_TTL:
1531df8bae1dSRodney W. Grimes 		case IP_MULTICAST_LOOP:
153271498f30SBruce M Simpson 		case IP_MSFILTER:
153371498f30SBruce M Simpson 			error = inp_getmoptions(inp, sopt);
153433b3ac06SPeter Wemm 			break;
153533b3ac06SPeter Wemm 
1536fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
15376a800098SYoshinobu Inoue 		case IP_IPSEC_POLICY:
1538fcf59617SAndrey V. Elsukov 			if (IPSEC_ENABLED(ipv4)) {
1539fcf59617SAndrey V. Elsukov 				error = IPSEC_PCBCTL(ipv4, inp, sopt);
15406a800098SYoshinobu Inoue 				break;
15416a800098SYoshinobu Inoue 			}
1542fcf59617SAndrey V. Elsukov 			/* FALLTHROUGH */
1543b2630c29SGeorge V. Neville-Neil #endif /* IPSEC */
15446a800098SYoshinobu Inoue 
1545df8bae1dSRodney W. Grimes 		default:
1546df8bae1dSRodney W. Grimes 			error = ENOPROTOOPT;
1547df8bae1dSRodney W. Grimes 			break;
1548df8bae1dSRodney W. Grimes 		}
1549df8bae1dSRodney W. Grimes 		break;
1550df8bae1dSRodney W. Grimes 	}
1551df8bae1dSRodney W. Grimes 	return (error);
1552df8bae1dSRodney W. Grimes }
1553df8bae1dSRodney W. Grimes 
1554df8bae1dSRodney W. Grimes /*
1555df8bae1dSRodney W. Grimes  * Routine called from ip_output() to loop back a copy of an IP multicast
1556df8bae1dSRodney W. Grimes  * packet to the input queue of a specified interface.  Note that this
1557df8bae1dSRodney W. Grimes  * calls the output routine of the loopback "driver", but with an interface
1558f5fea3ddSPaul Traina  * pointer that might NOT be a loopback interface -- evil, but easier than
1559f5fea3ddSPaul Traina  * replicating that code here.
1560df8bae1dSRodney W. Grimes  */
1561df8bae1dSRodney W. Grimes static void
1562331dff07SAlexander V. Chernikov ip_mloopback(struct ifnet *ifp, const struct mbuf *m, int hlen)
1563df8bae1dSRodney W. Grimes {
1564331dff07SAlexander V. Chernikov 	struct ip *ip;
1565df8bae1dSRodney W. Grimes 	struct mbuf *copym;
1566df8bae1dSRodney W. Grimes 
1567e4762f75SGeorge V. Neville-Neil 	/*
1568e4762f75SGeorge V. Neville-Neil 	 * Make a deep copy of the packet because we're going to
1569e4762f75SGeorge V. Neville-Neil 	 * modify the pack in order to generate checksums.
1570e4762f75SGeorge V. Neville-Neil 	 */
1571eb1b1807SGleb Smirnoff 	copym = m_dup(m, M_NOWAIT);
1572f0cace5dSRobert Watson 	if (copym != NULL && (!M_WRITABLE(copym) || copym->m_len < hlen))
157386b1d6d2SBill Fenner 		copym = m_pullup(copym, hlen);
1574df8bae1dSRodney W. Grimes 	if (copym != NULL) {
1575390cdc6aSRuslan Ermilov 		/* If needed, compute the checksum and mark it as valid. */
1576390cdc6aSRuslan Ermilov 		if (copym->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
1577390cdc6aSRuslan Ermilov 			in_delayed_cksum(copym);
1578390cdc6aSRuslan Ermilov 			copym->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
1579390cdc6aSRuslan Ermilov 			copym->m_pkthdr.csum_flags |=
1580390cdc6aSRuslan Ermilov 			    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1581390cdc6aSRuslan Ermilov 			copym->m_pkthdr.csum_data = 0xffff;
1582390cdc6aSRuslan Ermilov 		}
1583df8bae1dSRodney W. Grimes 		/*
1584df8bae1dSRodney W. Grimes 		 * We don't bother to fragment if the IP length is greater
1585df8bae1dSRodney W. Grimes 		 * than the interface's MTU.  Can this possibly matter?
1586df8bae1dSRodney W. Grimes 		 */
15878f134647SGleb Smirnoff 		ip = mtod(copym, struct ip *);
1588df8bae1dSRodney W. Grimes 		ip->ip_sum = 0;
158986b1d6d2SBill Fenner 		ip->ip_sum = in_cksum(copym, hlen);
1590331dff07SAlexander V. Chernikov 		if_simloop(ifp, copym, AF_INET, 0);
1591df8bae1dSRodney W. Grimes 	}
1592df8bae1dSRodney W. Grimes }
1593