xref: /freebsd/sys/netinet/ip_fastfwd.c (revision cfd6422a5217410fbd66f7a7a8a64d9d85e61229)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 2003 Andre Oppermann, Internet Business Solutions AG
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. The name of the author may not be used to endorse or promote
16  *    products derived from this software without specific prior written
17  *    permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  */
31 
32 /*
33  * ip_fastforward gets its speed from processing the forwarded packet to
34  * completion (if_output on the other side) without any queues or netisr's.
35  * The receiving interface DMAs the packet into memory, the upper half of
36  * driver calls ip_fastforward, we do our routing table lookup and directly
37  * send it off to the outgoing interface, which DMAs the packet to the
38  * network card. The only part of the packet we touch with the CPU is the
39  * IP header (unless there are complex firewall rules touching other parts
40  * of the packet, but that is up to you). We are essentially limited by bus
41  * bandwidth and how fast the network card/driver can set up receives and
42  * transmits.
43  *
44  * We handle basic errors, IP header errors, checksum errors,
45  * destination unreachable, fragmentation and fragmentation needed and
46  * report them via ICMP to the sender.
47  *
48  * Else if something is not pure IPv4 unicast forwarding we fall back to
49  * the normal ip_input processing path. We should only be called from
50  * interfaces connected to the outside world.
51  *
52  * Firewalling is fully supported including divert, ipfw fwd and ipfilter
53  * ipnat and address rewrite.
54  *
55  * IPSEC is not supported if this host is a tunnel broker. IPSEC is
56  * supported for connections to/from local host.
57  *
58  * We try to do the least expensive (in CPU ops) checks and operations
59  * first to catch junk with as little overhead as possible.
60  *
61  * We take full advantage of hardware support for IP checksum and
62  * fragmentation offloading.
63  *
64  * We don't do ICMP redirect in the fast forwarding path. I have had my own
65  * cases where two core routers with Zebra routing suite would send millions
66  * ICMP redirects to connected hosts if the destination router was not the
67  * default gateway. In one case it was filling the routing table of a host
68  * with approximately 300.000 cloned redirect entries until it ran out of
69  * kernel memory. However the networking code proved very robust and it didn't
70  * crash or fail in other ways.
71  */
72 
73 /*
74  * Many thanks to Matt Thomas of NetBSD for basic structure of ip_flow.c which
75  * is being followed here.
76  */
77 
78 #include <sys/cdefs.h>
79 __FBSDID("$FreeBSD$");
80 
81 #include "opt_ipstealth.h"
82 
83 #include <sys/param.h>
84 #include <sys/systm.h>
85 #include <sys/kernel.h>
86 #include <sys/malloc.h>
87 #include <sys/mbuf.h>
88 #include <sys/protosw.h>
89 #include <sys/sdt.h>
90 #include <sys/socket.h>
91 #include <sys/sysctl.h>
92 
93 #include <net/if.h>
94 #include <net/if_types.h>
95 #include <net/if_var.h>
96 #include <net/if_dl.h>
97 #include <net/pfil.h>
98 #include <net/route.h>
99 #include <net/route/nhop.h>
100 #include <net/vnet.h>
101 
102 #include <netinet/in.h>
103 #include <netinet/in_fib.h>
104 #include <netinet/in_kdtrace.h>
105 #include <netinet/in_systm.h>
106 #include <netinet/in_var.h>
107 #include <netinet/ip.h>
108 #include <netinet/ip_var.h>
109 #include <netinet/ip_icmp.h>
110 #include <netinet/ip_options.h>
111 
112 #include <machine/in_cksum.h>
113 
114 #define	V_ipsendredirects	VNET(ipsendredirects)
115 
116 static struct mbuf *
117 ip_redir_alloc(struct mbuf *m, struct nhop_object *nh,
118     struct ip *ip, in_addr_t *addr)
119 {
120 	struct mbuf *mcopy = m_gethdr(M_NOWAIT, m->m_type);
121 
122 	if (mcopy == NULL)
123 		return (NULL);
124 
125 	if (m_dup_pkthdr(mcopy, m, M_NOWAIT) == 0) {
126 		/*
127 		 * It's probably ok if the pkthdr dup fails (because
128 		 * the deep copy of the tag chain failed), but for now
129 		 * be conservative and just discard the copy since
130 		 * code below may some day want the tags.
131 		 */
132 		m_free(mcopy);
133 		return (NULL);
134 	}
135 	mcopy->m_len = min(ntohs(ip->ip_len), M_TRAILINGSPACE(mcopy));
136 	mcopy->m_pkthdr.len = mcopy->m_len;
137 	m_copydata(m, 0, mcopy->m_len, mtod(mcopy, caddr_t));
138 
139 	if (nh != NULL &&
140 	    ((nh->nh_flags & (NHF_REDIRECT|NHF_DEFAULT)) == 0)) {
141 		struct in_ifaddr *nh_ia = (struct in_ifaddr *)(nh->nh_ifa);
142 		u_long src = ntohl(ip->ip_src.s_addr);
143 
144 		if (nh_ia != NULL &&
145 		    (src & nh_ia->ia_subnetmask) == nh_ia->ia_subnet) {
146 			if (nh->nh_flags & NHF_GATEWAY)
147 				*addr = nh->gw4_sa.sin_addr.s_addr;
148 			else
149 				*addr = ip->ip_dst.s_addr;
150 		}
151 	}
152 	return (mcopy);
153 }
154 
155 
156 static int
157 ip_findroute(struct nhop_object **pnh, struct in_addr dest, struct mbuf *m)
158 {
159 	struct nhop_object *nh;
160 
161 	nh = fib4_lookup(M_GETFIB(m), dest, 0, NHR_NONE,
162 	    m->m_pkthdr.flowid);
163 	if (nh == NULL) {
164 		IPSTAT_INC(ips_noroute);
165 		IPSTAT_INC(ips_cantforward);
166 		icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0);
167 		return (EHOSTUNREACH);
168 	}
169 	/*
170 	 * Drop blackholed traffic and directed broadcasts.
171 	 */
172 	if ((nh->nh_flags & (NHF_BLACKHOLE | NHF_BROADCAST)) != 0) {
173 		IPSTAT_INC(ips_cantforward);
174 		m_freem(m);
175 		return (EHOSTUNREACH);
176 	}
177 
178 	if (nh->nh_flags & NHF_REJECT) {
179 		IPSTAT_INC(ips_cantforward);
180 		icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0);
181 		return (EHOSTUNREACH);
182 	}
183 
184 	*pnh = nh;
185 
186 	return (0);
187 }
188 
189 /*
190  * Try to forward a packet based on the destination address.
191  * This is a fast path optimized for the plain forwarding case.
192  * If the packet is handled (and consumed) here then we return NULL;
193  * otherwise mbuf is returned and the packet should be delivered
194  * to ip_input for full processing.
195  */
196 struct mbuf *
197 ip_tryforward(struct mbuf *m)
198 {
199 	struct ip *ip;
200 	struct mbuf *m0 = NULL;
201 	struct nhop_object *nh = NULL;
202 	struct sockaddr_in dst;
203 	struct in_addr dest, odest, rtdest;
204 	uint16_t ip_len, ip_off;
205 	int error = 0;
206 	struct m_tag *fwd_tag = NULL;
207 	struct mbuf *mcopy = NULL;
208 	struct in_addr redest;
209 	/*
210 	 * Are we active and forwarding packets?
211 	 */
212 
213 	M_ASSERTVALID(m);
214 	M_ASSERTPKTHDR(m);
215 
216 #ifdef ALTQ
217 	/*
218 	 * Is packet dropped by traffic conditioner?
219 	 */
220 	if (altq_input != NULL && (*altq_input)(m, AF_INET) == 0)
221 		goto drop;
222 #endif
223 
224 	/*
225 	 * Only IP packets without options
226 	 */
227 	ip = mtod(m, struct ip *);
228 
229 	if (ip->ip_hl != (sizeof(struct ip) >> 2)) {
230 		if (V_ip_doopts == 1)
231 			return m;
232 		else if (V_ip_doopts == 2) {
233 			icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_FILTER_PROHIB,
234 				0, 0);
235 			return NULL;	/* mbuf already free'd */
236 		}
237 		/* else ignore IP options and continue */
238 	}
239 
240 	/*
241 	 * Only unicast IP, not from loopback, no L2 or IP broadcast,
242 	 * no multicast, no INADDR_ANY
243 	 *
244 	 * XXX: Probably some of these checks could be direct drop
245 	 * conditions.  However it is not clear whether there are some
246 	 * hacks or obscure behaviours which make it necessary to
247 	 * let ip_input handle it.  We play safe here and let ip_input
248 	 * deal with it until it is proven that we can directly drop it.
249 	 */
250 	if ((m->m_flags & (M_BCAST|M_MCAST)) ||
251 	    (m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) ||
252 	    ntohl(ip->ip_src.s_addr) == (u_long)INADDR_BROADCAST ||
253 	    ntohl(ip->ip_dst.s_addr) == (u_long)INADDR_BROADCAST ||
254 	    IN_MULTICAST(ntohl(ip->ip_src.s_addr)) ||
255 	    IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) ||
256 	    IN_LINKLOCAL(ntohl(ip->ip_src.s_addr)) ||
257 	    IN_LINKLOCAL(ntohl(ip->ip_dst.s_addr)) ||
258 	    ip->ip_src.s_addr == INADDR_ANY ||
259 	    ip->ip_dst.s_addr == INADDR_ANY )
260 		return m;
261 
262 	/*
263 	 * Is it for a local address on this host?
264 	 */
265 	if (in_localip(ip->ip_dst))
266 		return m;
267 
268 	IPSTAT_INC(ips_total);
269 
270 	/*
271 	 * Step 3: incoming packet firewall processing
272 	 */
273 
274 	odest.s_addr = dest.s_addr = ip->ip_dst.s_addr;
275 
276 	/*
277 	 * Run through list of ipfilter hooks for input packets
278 	 */
279 	if (!PFIL_HOOKED_IN(V_inet_pfil_head))
280 		goto passin;
281 
282 	if (pfil_run_hooks(V_inet_pfil_head, &m, m->m_pkthdr.rcvif, PFIL_IN,
283 	    NULL) != PFIL_PASS)
284 		goto drop;
285 
286 	M_ASSERTVALID(m);
287 	M_ASSERTPKTHDR(m);
288 
289 	ip = mtod(m, struct ip *);	/* m may have changed by pfil hook */
290 	dest.s_addr = ip->ip_dst.s_addr;
291 
292 	/*
293 	 * Destination address changed?
294 	 */
295 	if (odest.s_addr != dest.s_addr) {
296 		/*
297 		 * Is it now for a local address on this host?
298 		 */
299 		if (in_localip(dest))
300 			goto forwardlocal;
301 		/*
302 		 * Go on with new destination address
303 		 */
304 	}
305 
306 	if (m->m_flags & M_FASTFWD_OURS) {
307 		/*
308 		 * ipfw changed it for a local address on this host.
309 		 */
310 		goto forwardlocal;
311 	}
312 
313 passin:
314 	/*
315 	 * Step 4: decrement TTL and look up route
316 	 */
317 
318 	/*
319 	 * Check TTL
320 	 */
321 #ifdef IPSTEALTH
322 	if (!V_ipstealth) {
323 #endif
324 	if (ip->ip_ttl <= IPTTLDEC) {
325 		icmp_error(m, ICMP_TIMXCEED, ICMP_TIMXCEED_INTRANS, 0, 0);
326 		return NULL;	/* mbuf already free'd */
327 	}
328 
329 	/*
330 	 * Decrement the TTL and incrementally change the IP header checksum.
331 	 * Don't bother doing this with hw checksum offloading, it's faster
332 	 * doing it right here.
333 	 */
334 	ip->ip_ttl -= IPTTLDEC;
335 	if (ip->ip_sum >= (u_int16_t) ~htons(IPTTLDEC << 8))
336 		ip->ip_sum -= ~htons(IPTTLDEC << 8);
337 	else
338 		ip->ip_sum += htons(IPTTLDEC << 8);
339 #ifdef IPSTEALTH
340 	}
341 #endif
342 
343 	/*
344 	 * Next hop forced by pfil(9) hook?
345 	 */
346 	if ((m->m_flags & M_IP_NEXTHOP) &&
347 	    ((fwd_tag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL)) != NULL)) {
348 		/*
349 		 * Now we will find route to forced destination.
350 		 */
351 		dest.s_addr = ((struct sockaddr_in *)
352 			    (fwd_tag + 1))->sin_addr.s_addr;
353 		m_tag_delete(m, fwd_tag);
354 		m->m_flags &= ~M_IP_NEXTHOP;
355 	}
356 
357 	/*
358 	 * Find route to destination.
359 	 */
360 	if (ip_findroute(&nh, dest, m) != 0)
361 		return (NULL);	/* icmp unreach already sent */
362 
363 	/*
364 	 * Avoid second route lookup by caching destination.
365 	 */
366 	rtdest.s_addr = dest.s_addr;
367 
368 	/*
369 	 * Step 5: outgoing firewall packet processing
370 	 */
371 	if (!PFIL_HOOKED_OUT(V_inet_pfil_head))
372 		goto passout;
373 
374 	if (pfil_run_hooks(V_inet_pfil_head, &m, nh->nh_ifp,
375 	    PFIL_OUT | PFIL_FWD, NULL) != PFIL_PASS)
376 		goto drop;
377 
378 	M_ASSERTVALID(m);
379 	M_ASSERTPKTHDR(m);
380 
381 	ip = mtod(m, struct ip *);
382 	dest.s_addr = ip->ip_dst.s_addr;
383 
384 	/*
385 	 * Destination address changed?
386 	 */
387 	if (m->m_flags & M_IP_NEXTHOP)
388 		fwd_tag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL);
389 	else
390 		fwd_tag = NULL;
391 	if (odest.s_addr != dest.s_addr || fwd_tag != NULL) {
392 		/*
393 		 * Is it now for a local address on this host?
394 		 */
395 		if (m->m_flags & M_FASTFWD_OURS || in_localip(dest)) {
396 forwardlocal:
397 			/*
398 			 * Return packet for processing by ip_input().
399 			 */
400 			m->m_flags |= M_FASTFWD_OURS;
401 			return (m);
402 		}
403 		/*
404 		 * Redo route lookup with new destination address
405 		 */
406 		if (fwd_tag) {
407 			dest.s_addr = ((struct sockaddr_in *)
408 				    (fwd_tag + 1))->sin_addr.s_addr;
409 			m_tag_delete(m, fwd_tag);
410 			m->m_flags &= ~M_IP_NEXTHOP;
411 		}
412 		if (dest.s_addr != rtdest.s_addr &&
413 		    ip_findroute(&nh, dest, m) != 0)
414 			return (NULL);	/* icmp unreach already sent */
415 	}
416 
417 passout:
418 	/*
419 	 * Step 6: send off the packet
420 	 */
421 	ip_len = ntohs(ip->ip_len);
422 	ip_off = ntohs(ip->ip_off);
423 
424 	bzero(&dst, sizeof(dst));
425 	dst.sin_family = AF_INET;
426 	dst.sin_len = sizeof(dst);
427 	if (nh->nh_flags & NHF_GATEWAY)
428 		dst.sin_addr = nh->gw4_sa.sin_addr;
429 	else
430 		dst.sin_addr = dest;
431 
432 	/*
433 	 * Handle redirect case.
434 	 */
435 	redest.s_addr = 0;
436 	if (V_ipsendredirects && (nh->nh_ifp == m->m_pkthdr.rcvif))
437 		mcopy = ip_redir_alloc(m, nh, ip, &redest.s_addr);
438 
439 	/*
440 	 * Check if packet fits MTU or if hardware will fragment for us
441 	 */
442 	if (ip_len <= nh->nh_mtu) {
443 		/*
444 		 * Avoid confusing lower layers.
445 		 */
446 		m_clrprotoflags(m);
447 		/*
448 		 * Send off the packet via outgoing interface
449 		 */
450 		IP_PROBE(send, NULL, NULL, ip, nh->nh_ifp, ip, NULL);
451 		error = (*nh->nh_ifp->if_output)(nh->nh_ifp, m,
452 		    (struct sockaddr *)&dst, NULL);
453 	} else {
454 		/*
455 		 * Handle EMSGSIZE with icmp reply needfrag for TCP MTU discovery
456 		 */
457 		if (ip_off & IP_DF) {
458 			IPSTAT_INC(ips_cantfrag);
459 			icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG,
460 				0, nh->nh_mtu);
461 			goto consumed;
462 		} else {
463 			/*
464 			 * We have to fragment the packet
465 			 */
466 			m->m_pkthdr.csum_flags |= CSUM_IP;
467 			if (ip_fragment(ip, &m, nh->nh_mtu,
468 			    nh->nh_ifp->if_hwassist) != 0)
469 				goto drop;
470 			KASSERT(m != NULL, ("null mbuf and no error"));
471 			/*
472 			 * Send off the fragments via outgoing interface
473 			 */
474 			error = 0;
475 			do {
476 				m0 = m->m_nextpkt;
477 				m->m_nextpkt = NULL;
478 				/*
479 				 * Avoid confusing lower layers.
480 				 */
481 				m_clrprotoflags(m);
482 
483 				IP_PROBE(send, NULL, NULL,
484 				    mtod(m, struct ip *), nh->nh_ifp,
485 				    mtod(m, struct ip *), NULL);
486 				error = (*nh->nh_ifp->if_output)(nh->nh_ifp, m,
487 				    (struct sockaddr *)&dst, NULL);
488 				if (error)
489 					break;
490 			} while ((m = m0) != NULL);
491 			if (error) {
492 				/* Reclaim remaining fragments */
493 				for (m = m0; m; m = m0) {
494 					m0 = m->m_nextpkt;
495 					m_freem(m);
496 				}
497 			} else
498 				IPSTAT_INC(ips_fragmented);
499 		}
500 	}
501 
502 	if (error != 0)
503 		IPSTAT_INC(ips_odropped);
504 	else {
505 		IPSTAT_INC(ips_forward);
506 		IPSTAT_INC(ips_fastforward);
507 	}
508 
509 	/* Send required redirect */
510 	if (mcopy != NULL) {
511 		icmp_error(mcopy, ICMP_REDIRECT, ICMP_REDIRECT_HOST, redest.s_addr, 0);
512 		mcopy = NULL; /* Freed by caller */
513 	}
514 
515 consumed:
516 	if (mcopy != NULL)
517 		m_freem(mcopy);
518 	return NULL;
519 drop:
520 	if (m)
521 		m_freem(m);
522 	return NULL;
523 }
524