xref: /freebsd/sys/net/route/nhop_ctl.c (revision a96ef4501919d7ac08e94e98dc34b0bdd744802b)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2020 Alexander V. Chernikov
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 #include "opt_inet.h"
31 #include "opt_route.h"
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/lock.h>
36 #include <sys/rwlock.h>
37 #include <sys/malloc.h>
38 #include <sys/socket.h>
39 #include <sys/sysctl.h>
40 #include <sys/kernel.h>
41 #include <sys/epoch.h>
42 
43 #include <net/if.h>
44 #include <net/if_var.h>
45 #include <net/if_dl.h>
46 #include <net/route.h>
47 #include <net/route/route_ctl.h>
48 #include <net/route/route_var.h>
49 #include <net/route/nhop_utils.h>
50 #include <net/route/nhop.h>
51 #include <net/route/nhop_var.h>
52 #include <net/vnet.h>
53 
54 /*
55  * This file contains core functionality for the nexthop ("nhop") route subsystem.
56  * The business logic needed to create nexhop objects is implemented here.
57  *
58  * Nexthops in the original sense are the objects containing all the necessary
59  * information to forward the packet to the selected destination.
60  * In particular, nexthop is defined by a combination of
61  *  ifp, ifa, aifp, mtu, gw addr(if set), nh_type, nh_family, mask of rt_flags and
62  *    NHF_DEFAULT
63  *
64  * Additionally, each nexthop gets assigned its unique index (nexthop index).
65  * It serves two purposes: first one is to ease the ability of userland programs to
66  *  reference nexthops by their index. The second one allows lookup algorithms to
67  *  to store index instead of pointer (2 bytes vs 8) as a lookup result.
68  * All nexthops are stored in the resizable hash table.
69  *
70  * Basically, this file revolves around supporting 3 functions:
71  * 1) nhop_create_from_info / nhop_create_from_nhop, which contains all
72  *  business logic on filling the nexthop fields based on the provided request.
73  * 2) nhop_get(), which gets a usable referenced nexthops.
74  *
75  * Conventions:
76  * 1) non-exported functions start with verb
77  * 2) exported function starts with the subsystem prefix: "nhop"
78  */
79 
80 static int dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w);
81 
82 static struct nhop_priv *alloc_nhop_structure(void);
83 static int get_nhop(struct rib_head *rnh, struct rt_addrinfo *info,
84     struct nhop_priv **pnh_priv);
85 static int finalize_nhop(struct nh_control *ctl, struct rt_addrinfo *info,
86     struct nhop_priv *nh_priv);
87 static struct ifnet *get_aifp(const struct nhop_object *nh);
88 static void fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp);
89 
90 static void destroy_nhop_epoch(epoch_context_t ctx);
91 static void destroy_nhop(struct nhop_priv *nh_priv);
92 
93 static void print_nhop(const char *prefix, const struct nhop_object *nh);
94 
95 _Static_assert(__offsetof(struct nhop_object, nh_ifp) == 32,
96     "nhop_object: wrong nh_ifp offset");
97 _Static_assert(sizeof(struct nhop_object) <= 128,
98     "nhop_object: size exceeds 128 bytes");
99 
100 static uma_zone_t nhops_zone;	/* Global zone for each and every nexthop */
101 
102 #define	NHOP_OBJECT_ALIGNED_SIZE	roundup2(sizeof(struct nhop_object), \
103 							2 * CACHE_LINE_SIZE)
104 #define	NHOP_PRIV_ALIGNED_SIZE		roundup2(sizeof(struct nhop_priv), \
105 							2 * CACHE_LINE_SIZE)
106 void
107 nhops_init(void)
108 {
109 
110 	nhops_zone = uma_zcreate("routing nhops",
111 	    NHOP_OBJECT_ALIGNED_SIZE + NHOP_PRIV_ALIGNED_SIZE,
112 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
113 }
114 
115 /*
116  * Fetches the interface of source address used by the route.
117  * In all cases except interface-address-route it would be the
118  * same as the transmit interfaces.
119  * However, for the interface address this function will return
120  * this interface ifp instead of loopback. This is needed to support
121  * link-local IPv6 loopback communications.
122  *
123  * Returns found ifp.
124  */
125 static struct ifnet *
126 get_aifp(const struct nhop_object *nh)
127 {
128 	struct ifnet *aifp = NULL;
129 
130 	/*
131 	 * Adjust the "outgoing" interface.  If we're going to loop
132 	 * the packet back to ourselves, the ifp would be the loopback
133 	 * interface. However, we'd rather know the interface associated
134 	 * to the destination address (which should probably be one of
135 	 * our own addresses).
136 	 */
137 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) &&
138 			nh->gw_sa.sa_family == AF_LINK) {
139 		aifp = ifnet_byindex(nh->gwl_sa.sdl_index);
140 		if (aifp == NULL) {
141 			DPRINTF("unable to get aifp for %s index %d",
142 				if_name(nh->nh_ifp), nh->gwl_sa.sdl_index);
143 		}
144 	}
145 
146 	if (aifp == NULL)
147 		aifp = nh->nh_ifp;
148 
149 	return (aifp);
150 }
151 
152 int
153 cmp_priv(const struct nhop_priv *_one, const struct nhop_priv *_two)
154 {
155 
156 	if (memcmp(_one->nh, _two->nh, NHOP_END_CMP) != 0)
157 		return (0);
158 
159 	if (memcmp(_one, _two, NH_PRIV_END_CMP) != 0)
160 		return (0);
161 
162 	return (1);
163 }
164 
165 /*
166  * Conditionally sets @nh mtu data based on the @info data.
167  */
168 static void
169 set_nhop_mtu_from_info(struct nhop_object *nh, const struct rt_addrinfo *info)
170 {
171 
172 	if (info->rti_mflags & RTV_MTU) {
173 		if (info->rti_rmx->rmx_mtu != 0) {
174 			/*
175 			 * MTU was explicitly provided by user.
176 			 * Keep it.
177 			 */
178 
179 			nh->nh_priv->rt_flags |= RTF_FIXEDMTU;
180 		} else {
181 			/*
182 			 * User explicitly sets MTU to 0.
183 			 * Assume rollback to default.
184 			 */
185 			nh->nh_priv->rt_flags &= ~RTF_FIXEDMTU;
186 		}
187 		nh->nh_mtu = info->rti_rmx->rmx_mtu;
188 	}
189 }
190 
191 /*
192  * Fills in shorted link-level sockadd version suitable to be stored inside the
193  *  nexthop gateway buffer.
194  */
195 static void
196 fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp)
197 {
198 
199 	bzero(sdl, sizeof(struct sockaddr_dl_short));
200 	sdl->sdl_family = AF_LINK;
201 	sdl->sdl_len = sizeof(struct sockaddr_dl_short);
202 	sdl->sdl_index = ifp->if_index;
203 	sdl->sdl_type = ifp->if_type;
204 }
205 
206 static int
207 set_nhop_gw_from_info(struct nhop_object *nh, struct rt_addrinfo *info)
208 {
209 	struct sockaddr *gw;
210 
211 	gw = info->rti_info[RTAX_GATEWAY];
212 	KASSERT(gw != NULL, ("gw is NULL"));
213 
214 	if ((gw->sa_family == AF_LINK) && !(info->rti_flags & RTF_GATEWAY)) {
215 
216 		/*
217 		 * Interface route with interface specified by the interface
218 		 * index in sockadd_dl structure. It is used in the IPv6 loopback
219 		 * output code, where we need to preserve the original interface
220 		 * to maintain proper scoping.
221 		 * Despite the fact that nexthop code stores original interface
222 		 * in the separate field (nh_aifp, see below), write AF_LINK
223 		 * compatible sa with shorter total length.
224 		 */
225 		struct sockaddr_dl *sdl = (struct sockaddr_dl *)gw;
226 		struct ifnet *ifp = ifnet_byindex(sdl->sdl_index);
227 		if (ifp == NULL) {
228 			DPRINTF("invalid ifindex %d", sdl->sdl_index);
229 			return (EINVAL);
230 		}
231 		fill_sdl_from_ifp(&nh->gwl_sa, ifp);
232 	} else {
233 
234 		/*
235 		 * Multiple options here:
236 		 *
237 		 * 1) RTF_GATEWAY with IPv4/IPv6 gateway data
238 		 * 2) Interface route with IPv4/IPv6 address of the
239 		 *   matching interface. Some routing daemons do that
240 		 *   instead of specifying ifindex in AF_LINK.
241 		 *
242 		 * In both cases, save the original nexthop to make the callers
243 		 *   happy.
244 		 */
245 		if (gw->sa_len > sizeof(struct sockaddr_in6)) {
246 			DPRINTF("nhop SA size too big: AF %d len %u",
247 			    gw->sa_family, gw->sa_len);
248 			return (ENOMEM);
249 		}
250 		memcpy(&nh->gw_sa, gw, gw->sa_len);
251 	}
252 	return (0);
253 }
254 
255 static uint16_t
256 convert_rt_to_nh_flags(int rt_flags)
257 {
258 	uint16_t res;
259 
260 	res = (rt_flags & RTF_REJECT) ? NHF_REJECT : 0;
261 	res |= (rt_flags & RTF_HOST) ? NHF_HOST : 0;
262 	res |= (rt_flags & RTF_BLACKHOLE) ? NHF_BLACKHOLE : 0;
263 	res |= (rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) ? NHF_REDIRECT : 0;
264 	res |= (rt_flags & RTF_BROADCAST) ? NHF_BROADCAST : 0;
265 	res |= (rt_flags & RTF_GATEWAY) ? NHF_GATEWAY : 0;
266 
267 	return (res);
268 }
269 
270 static int
271 fill_nhop_from_info(struct nhop_priv *nh_priv, struct rt_addrinfo *info)
272 {
273 	int error, rt_flags;
274 	struct nhop_object *nh;
275 
276 	nh = nh_priv->nh;
277 
278 	rt_flags = info->rti_flags & NHOP_RT_FLAG_MASK;
279 
280 	nh->nh_priv->rt_flags = rt_flags;
281 	nh_priv->nh_family = info->rti_info[RTAX_DST]->sa_family;
282 	nh_priv->nh_type = 0; // hook responsibility to set nhop type
283 
284 	nh->nh_flags = convert_rt_to_nh_flags(rt_flags);
285 	set_nhop_mtu_from_info(nh, info);
286 	if ((error = set_nhop_gw_from_info(nh, info)) != 0)
287 		return (error);
288 
289 	nh->nh_ifp = info->rti_ifa->ifa_ifp;
290 	nh->nh_ifa = info->rti_ifa;
291 	/* depends on the gateway */
292 	nh->nh_aifp = get_aifp(nh);
293 
294 	/*
295 	 * Note some of the remaining data is set by the
296 	 * per-address-family pre-add hook.
297 	 */
298 
299 	return (0);
300 }
301 
302 /*
303  * Creates a new nexthop based on the information in @info.
304  *
305  * Returns:
306  * 0 on success, filling @nh_ret with the desired nexthop object ptr
307  * errno otherwise
308  */
309 int
310 nhop_create_from_info(struct rib_head *rnh, struct rt_addrinfo *info,
311     struct nhop_object **nh_ret)
312 {
313 	struct nhop_priv *nh_priv;
314 	int error;
315 
316 	NET_EPOCH_ASSERT();
317 
318 	if (info->rti_info[RTAX_GATEWAY] == NULL)
319 		return (EINVAL);
320 
321 	nh_priv = alloc_nhop_structure();
322 
323 	error = fill_nhop_from_info(nh_priv, info);
324 	if (error != 0) {
325 		uma_zfree(nhops_zone, nh_priv->nh);
326 		return (error);
327 	}
328 
329 	error = get_nhop(rnh, info, &nh_priv);
330 	if (error == 0)
331 		*nh_ret = nh_priv->nh;
332 
333 	return (error);
334 }
335 
336 /*
337  * Gets linked nhop using the provided @pnh_priv nexhop data.
338  * If linked nhop is found, returns it, freeing the provided one.
339  * If there is no such nexthop, attaches the remaining data to the
340  *  provided nexthop and links it.
341  *
342  * Returns 0 on success, storing referenced nexthop in @pnh_priv.
343  * Otherwise, errno is returned.
344  */
345 static int
346 get_nhop(struct rib_head *rnh, struct rt_addrinfo *info,
347     struct nhop_priv **pnh_priv)
348 {
349 	const struct sockaddr *dst, *gateway, *netmask;
350 	struct nhop_priv *nh_priv, *tmp_priv;
351 	int error;
352 
353 	nh_priv = *pnh_priv;
354 
355 	/* Give the protocols chance to augment the request data */
356 	dst = info->rti_info[RTAX_DST];
357 	netmask = info->rti_info[RTAX_NETMASK];
358 	gateway = info->rti_info[RTAX_GATEWAY];
359 
360 	error = rnh->rnh_preadd(rnh->rib_fibnum, dst, netmask, nh_priv->nh);
361 	if (error != 0) {
362 		uma_zfree(nhops_zone, nh_priv->nh);
363 		return (error);
364 	}
365 
366 	tmp_priv = find_nhop(rnh->nh_control, nh_priv);
367 	if (tmp_priv != NULL) {
368 		uma_zfree(nhops_zone, nh_priv->nh);
369 		*pnh_priv = tmp_priv;
370 		return (0);
371 	}
372 
373 	/*
374 	 * Existing nexthop not found, need to create new one.
375 	 * Note: multiple simultaneous get_nhop() requests
376 	 *  can result in multiple equal nexhops existing in the
377 	 *  nexthop table. This is not a not a problem until the
378 	 *  relative number of such nexthops is significant, which
379 	 *  is extremely unlikely.
380 	 */
381 
382 	error = finalize_nhop(rnh->nh_control, info, nh_priv);
383 	if (error != 0)
384 		return (error);
385 
386 	return (0);
387 }
388 
389 /*
390  * Update @nh with data supplied in @info.
391  * This is a helper function to support route changes.
392  *
393  * It limits the changes that can be done to the route to the following:
394  * 1) all combination of gateway changes (gw, interface, blackhole/reject)
395  * 2) route flags (FLAG[123],STATIC,BLACKHOLE,REJECT)
396  * 3) route MTU
397  *
398  * Returns:
399  * 0 on success
400  */
401 static int
402 alter_nhop_from_info(struct nhop_object *nh, struct rt_addrinfo *info)
403 {
404 	struct sockaddr *info_gw;
405 	int error;
406 
407 	/* Update MTU if set in the request*/
408 	set_nhop_mtu_from_info(nh, info);
409 
410 	/* XXX: allow only one of BLACKHOLE,REJECT,GATEWAY */
411 
412 	/* Allow some flags (FLAG1,STATIC,BLACKHOLE,REJECT) to be toggled on change. */
413 	nh->nh_priv->rt_flags &= ~RTF_FMASK;
414 	nh->nh_priv->rt_flags |= info->rti_flags & RTF_FMASK;
415 
416 	/* Consider gateway change */
417 	info_gw = info->rti_info[RTAX_GATEWAY];
418 	if (info_gw != NULL) {
419 		error = set_nhop_gw_from_info(nh, info);
420 		if (error != 0)
421 			return (error);
422 		/* Update RTF_GATEWAY flag status */
423 		nh->nh_priv->rt_flags &= ~RTF_GATEWAY;
424 		nh->nh_priv->rt_flags |= (RTF_GATEWAY & info->rti_flags);
425 	}
426 	/* Update datapath flags */
427 	nh->nh_flags = convert_rt_to_nh_flags(nh->nh_priv->rt_flags);
428 
429 	if (info->rti_ifa != NULL)
430 		nh->nh_ifa = info->rti_ifa;
431 	if (info->rti_ifp != NULL)
432 		nh->nh_ifp = info->rti_ifp;
433 	nh->nh_aifp = get_aifp(nh);
434 
435 	return (0);
436 }
437 
438 /*
439  * Creates new nexthop based on @nh_orig and augmentation data from @info.
440  * Helper function used in the route changes, please see
441  *   alter_nhop_from_info() comments for more details.
442  *
443  * Returns:
444  * 0 on success, filling @nh_ret with the desired nexthop object
445  * errno otherwise
446  */
447 int
448 nhop_create_from_nhop(struct rib_head *rnh, const struct nhop_object *nh_orig,
449     struct rt_addrinfo *info, struct nhop_object **pnh)
450 {
451 	struct nhop_priv *nh_priv;
452 	struct nhop_object *nh;
453 	int error;
454 
455 	NET_EPOCH_ASSERT();
456 
457 	nh_priv = alloc_nhop_structure();
458 	nh = nh_priv->nh;
459 
460 	/* Start with copying data from original nexthop */
461 	nh_priv->nh_family = nh_orig->nh_priv->nh_family;
462 	nh_priv->rt_flags = nh_orig->nh_priv->rt_flags;
463 	nh_priv->nh_type = nh_orig->nh_priv->nh_type;
464 
465 	nh->nh_ifp = nh_orig->nh_ifp;
466 	nh->nh_ifa = nh_orig->nh_ifa;
467 	nh->nh_aifp = nh_orig->nh_aifp;
468 	nh->nh_mtu = nh_orig->nh_mtu;
469 	nh->nh_flags = nh_orig->nh_flags;
470 	memcpy(&nh->gw_sa, &nh_orig->gw_sa, nh_orig->gw_sa.sa_len);
471 
472 	error = alter_nhop_from_info(nh, info);
473 	if (error != 0) {
474 		uma_zfree(nhops_zone, nh_priv->nh);
475 		return (error);
476 	}
477 
478 	error = get_nhop(rnh, info, &nh_priv);
479 	if (error == 0)
480 		*pnh = nh_priv->nh;
481 
482 	return (error);
483 }
484 
485 /*
486  * Allocates memory for public/private nexthop structures.
487  *
488  * Returns pointer to nhop_priv or NULL.
489  */
490 static struct nhop_priv *
491 alloc_nhop_structure()
492 {
493 	struct nhop_object *nh;
494 	struct nhop_priv *nh_priv;
495 
496 	nh = (struct nhop_object *)uma_zalloc(nhops_zone, M_NOWAIT | M_ZERO);
497 	if (nh == NULL)
498 		return (NULL);
499 	nh_priv = (struct nhop_priv *)((char *)nh + NHOP_OBJECT_ALIGNED_SIZE);
500 
501 	nh->nh_priv = nh_priv;
502 	nh_priv->nh = nh;
503 
504 	return (nh_priv);
505 }
506 
507 static bool
508 reference_nhop_deps(struct nhop_object *nh)
509 {
510 	if (!ifa_try_ref(nh->nh_ifa))
511 		return (false);
512 	nh->nh_aifp = get_aifp(nh);
513 	if (!if_try_ref(nh->nh_aifp)) {
514 		ifa_free(nh->nh_ifa);
515 		return (false);
516 	}
517 	DPRINTF("AIFP: %p nh_ifp %p", nh->nh_aifp, nh->nh_ifp);
518 	if (!if_try_ref(nh->nh_ifp)) {
519 		ifa_free(nh->nh_ifa);
520 		if_rele(nh->nh_aifp);
521 		return (false);
522 	}
523 
524 	return (true);
525 }
526 
527 /*
528  * Alocates/references the remaining bits of nexthop data and links
529  *  it to the hash table.
530  * Returns 0 if successful,
531  *  errno otherwise. @nh_priv is freed in case of error.
532  */
533 static int
534 finalize_nhop(struct nh_control *ctl, struct rt_addrinfo *info,
535     struct nhop_priv *nh_priv)
536 {
537 	struct nhop_object *nh = nh_priv->nh;
538 
539 	/* Allocate per-cpu packet counter */
540 	nh->nh_pksent = counter_u64_alloc(M_NOWAIT);
541 	if (nh->nh_pksent == NULL) {
542 		uma_zfree(nhops_zone, nh);
543 		RTSTAT_INC(rts_nh_alloc_failure);
544 		DPRINTF("nh_alloc_finalize failed");
545 		return (ENOMEM);
546 	}
547 
548 	if (!reference_nhop_deps(nh)) {
549 		counter_u64_free(nh->nh_pksent);
550 		uma_zfree(nhops_zone, nh);
551 		RTSTAT_INC(rts_nh_alloc_failure);
552 		DPRINTF("nh_alloc_finalize failed - reference failure");
553 		return (EAGAIN);
554 	}
555 
556 	/* Save vnet to ease destruction */
557 	nh_priv->nh_vnet = curvnet;
558 
559 	refcount_init(&nh_priv->nh_refcnt, 1);
560 
561 	/* Please see nhop_free() comments on the initial value */
562 	refcount_init(&nh_priv->nh_linked, 2);
563 
564 	print_nhop("FINALIZE", nh);
565 
566 	if (link_nhop(ctl, nh_priv) == 0) {
567 		/*
568 		 * Adding nexthop to the datastructures
569 		 *  failed. Call destructor w/o waiting for
570 		 *  the epoch end, as nexthop is not used
571 		 *  and return.
572 		 */
573 		DPRINTF("link_nhop failed!");
574 		destroy_nhop(nh_priv);
575 
576 		return (ENOBUFS);
577 	}
578 
579 	return (0);
580 }
581 
582 static void
583 print_nhop_sa(char *buf, size_t buflen, const struct sockaddr *sa)
584 {
585 
586 	if (sa->sa_family == AF_INET) {
587 		const struct sockaddr_in *sin4;
588 		sin4 = (const struct sockaddr_in *)sa;
589 		inet_ntop(AF_INET, &sin4->sin_addr, buf, buflen);
590 	} else if (sa->sa_family == AF_INET6) {
591 		const struct sockaddr_in6 *sin6;
592 		sin6 = (const struct sockaddr_in6 *)sa;
593 		inet_ntop(AF_INET6, &sin6->sin6_addr, buf, buflen);
594 	} else if (sa->sa_family == AF_LINK) {
595 		const struct sockaddr_dl *sdl;
596 		sdl = (const struct sockaddr_dl *)sa;
597 		snprintf(buf, buflen, "if#%d", sdl->sdl_index);
598 	} else
599 		snprintf(buf, buflen, "af:%d", sa->sa_family);
600 }
601 
602 static void
603 print_nhop(const char *prefix, const struct nhop_object *nh)
604 {
605 	char src_buf[INET6_ADDRSTRLEN], addr_buf[INET6_ADDRSTRLEN];
606 
607 	print_nhop_sa(src_buf, sizeof(src_buf), nh->nh_ifa->ifa_addr);
608 	print_nhop_sa(addr_buf, sizeof(addr_buf), &nh->gw_sa);
609 
610 	DPRINTF("%s nhop priv %p: AF %d ifp %p %s addr %s src %p %s aifp %p %s mtu %d nh_flags %X",
611 	    prefix, nh->nh_priv, nh->nh_priv->nh_family, nh->nh_ifp,
612 	    if_name(nh->nh_ifp), addr_buf, nh->nh_ifa, src_buf, nh->nh_aifp,
613 	    if_name(nh->nh_aifp), nh->nh_mtu, nh->nh_flags);
614 }
615 
616 static void
617 destroy_nhop(struct nhop_priv *nh_priv)
618 {
619 	struct nhop_object *nh;
620 
621 	nh = nh_priv->nh;
622 
623 	print_nhop("DEL", nh);
624 
625 	if_rele(nh->nh_ifp);
626 	if_rele(nh->nh_aifp);
627 	ifa_free(nh->nh_ifa);
628 	counter_u64_free(nh->nh_pksent);
629 
630 	uma_zfree(nhops_zone, nh);
631 }
632 
633 /*
634  * Epoch callback indicating nhop is safe to destroy
635  */
636 static void
637 destroy_nhop_epoch(epoch_context_t ctx)
638 {
639 	struct nhop_priv *nh_priv;
640 
641 	nh_priv = __containerof(ctx, struct nhop_priv, nh_epoch_ctx);
642 
643 	destroy_nhop(nh_priv);
644 }
645 
646 void
647 nhop_ref_object(struct nhop_object *nh)
648 {
649 	u_int old;
650 
651 	old = refcount_acquire(&nh->nh_priv->nh_refcnt);
652 	KASSERT(old > 0, ("%s: nhop object %p has 0 refs", __func__, nh));
653 }
654 
655 int
656 nhop_try_ref_object(struct nhop_object *nh)
657 {
658 
659 	return (refcount_acquire_if_not_zero(&nh->nh_priv->nh_refcnt));
660 }
661 
662 void
663 nhop_free(struct nhop_object *nh)
664 {
665 	struct nh_control *ctl;
666 	struct nhop_priv *nh_priv = nh->nh_priv;
667 	struct epoch_tracker et;
668 
669 	if (!refcount_release(&nh_priv->nh_refcnt))
670 		return;
671 
672 	/*
673 	 * There are only 2 places, where nh_linked can be decreased:
674 	 *  rib destroy (nhops_destroy_rib) and this function.
675 	 * nh_link can never be increased.
676 	 *
677 	 * Hence, use initial value of 2 to make use of
678 	 *  refcount_release_if_not_last().
679 	 *
680 	 * There can be two scenarious when calling this function:
681 	 *
682 	 * 1) nh_linked value is 2. This means that either
683 	 *  nhops_destroy_rib() has not been called OR it is running,
684 	 *  but we are guaranteed that nh_control won't be freed in
685 	 *  this epoch. Hence, nexthop can be safely unlinked.
686 	 *
687 	 * 2) nh_linked value is 1. In that case, nhops_destroy_rib()
688 	 *  has been called and nhop unlink can be skipped.
689 	 */
690 
691 	NET_EPOCH_ENTER(et);
692 	if (refcount_release_if_not_last(&nh_priv->nh_linked)) {
693 		ctl = nh_priv->nh_control;
694 		if (unlink_nhop(ctl, nh_priv) == NULL) {
695 			/* Do not try to reclaim */
696 			DPRINTF("Failed to unlink nexhop %p", nh_priv);
697 			NET_EPOCH_EXIT(et);
698 			return;
699 		}
700 	}
701 	NET_EPOCH_EXIT(et);
702 
703 	epoch_call(net_epoch_preempt, destroy_nhop_epoch,
704 	    &nh_priv->nh_epoch_ctx);
705 }
706 
707 void
708 nhop_ref_any(struct nhop_object *nh)
709 {
710 #ifdef ROUTE_MPATH
711 	if (!NH_IS_NHGRP(nh))
712 		nhop_ref_object(nh);
713 	else
714 		nhgrp_ref_object((struct nhgrp_object *)nh);
715 #else
716 	nhop_ref_object(nh);
717 #endif
718 }
719 
720 void
721 nhop_free_any(struct nhop_object *nh)
722 {
723 
724 #ifdef ROUTE_MPATH
725 	if (!NH_IS_NHGRP(nh))
726 		nhop_free(nh);
727 	else
728 		nhgrp_free((struct nhgrp_object *)nh);
729 #else
730 	nhop_free(nh);
731 #endif
732 }
733 
734 /* Helper functions */
735 
736 uint32_t
737 nhop_get_idx(const struct nhop_object *nh)
738 {
739 
740 	return (nh->nh_priv->nh_idx);
741 }
742 
743 enum nhop_type
744 nhop_get_type(const struct nhop_object *nh)
745 {
746 
747 	return (nh->nh_priv->nh_type);
748 }
749 
750 void
751 nhop_set_type(struct nhop_object *nh, enum nhop_type nh_type)
752 {
753 
754 	nh->nh_priv->nh_type = nh_type;
755 }
756 
757 int
758 nhop_get_rtflags(const struct nhop_object *nh)
759 {
760 
761 	return (nh->nh_priv->rt_flags);
762 }
763 
764 void
765 nhop_set_rtflags(struct nhop_object *nh, int rt_flags)
766 {
767 
768 	nh->nh_priv->rt_flags = rt_flags;
769 }
770 
771 struct vnet *
772 nhop_get_vnet(const struct nhop_object *nh)
773 {
774 
775 	return (nh->nh_priv->nh_vnet);
776 }
777 
778 struct nhop_object *
779 nhop_select_func(struct nhop_object *nh, uint32_t flowid)
780 {
781 
782 	return (nhop_select(nh, flowid));
783 }
784 
785 void
786 nhops_update_ifmtu(struct rib_head *rh, struct ifnet *ifp, uint32_t mtu)
787 {
788 	struct nh_control *ctl;
789 	struct nhop_priv *nh_priv;
790 	struct nhop_object *nh;
791 
792 	ctl = rh->nh_control;
793 
794 	NHOPS_WLOCK(ctl);
795 	CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) {
796 		nh = nh_priv->nh;
797 		if (nh->nh_ifp == ifp) {
798 			if ((nh_priv->rt_flags & RTF_FIXEDMTU) == 0 ||
799 			    nh->nh_mtu > mtu) {
800 				/* Update MTU directly */
801 				nh->nh_mtu = mtu;
802 			}
803 		}
804 	} CHT_SLIST_FOREACH_END;
805 	NHOPS_WUNLOCK(ctl);
806 
807 }
808 
809 /*
810  * Dumps a single entry to sysctl buffer.
811  *
812  * Layout:
813  *  rt_msghdr - generic RTM header to allow users to skip non-understood messages
814  *  nhop_external - nexhop description structure (with length)
815  *  nhop_addrs - structure encapsulating GW/SRC sockaddrs
816  */
817 static int
818 dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w)
819 {
820 	struct {
821 		struct rt_msghdr	rtm;
822 		struct nhop_external	nhe;
823 		struct nhop_addrs	na;
824 	} arpc;
825 	struct nhop_external *pnhe;
826 	struct sockaddr *gw_sa, *src_sa;
827 	struct sockaddr_storage ss;
828 	size_t addrs_len;
829 	int error;
830 
831 	//DPRINTF("Dumping: head %p nh %p flags %X req %p\n", rh, nh, nh->nh_flags, w);
832 
833 	memset(&arpc, 0, sizeof(arpc));
834 
835 	arpc.rtm.rtm_msglen = sizeof(arpc);
836 	arpc.rtm.rtm_version = RTM_VERSION;
837 	arpc.rtm.rtm_type = RTM_GET;
838 	//arpc.rtm.rtm_flags = RTF_UP;
839 	arpc.rtm.rtm_flags = nh->nh_priv->rt_flags;
840 
841 	/* nhop_external */
842 	pnhe = &arpc.nhe;
843 	pnhe->nh_len = sizeof(struct nhop_external);
844 	pnhe->nh_idx = nh->nh_priv->nh_idx;
845 	pnhe->nh_fib = rh->rib_fibnum;
846 	pnhe->ifindex = nh->nh_ifp->if_index;
847 	pnhe->aifindex = nh->nh_aifp->if_index;
848 	pnhe->nh_family = nh->nh_priv->nh_family;
849 	pnhe->nh_type = nh->nh_priv->nh_type;
850 	pnhe->nh_mtu = nh->nh_mtu;
851 	pnhe->nh_flags = nh->nh_flags;
852 
853 	memcpy(pnhe->nh_prepend, nh->nh_prepend, sizeof(nh->nh_prepend));
854 	pnhe->prepend_len = nh->nh_prepend_len;
855 	pnhe->nh_refcount = nh->nh_priv->nh_refcnt;
856 	pnhe->nh_pksent = counter_u64_fetch(nh->nh_pksent);
857 
858 	/* sockaddr container */
859 	addrs_len = sizeof(struct nhop_addrs);
860 	arpc.na.gw_sa_off = addrs_len;
861 	gw_sa = (struct sockaddr *)&nh->gw4_sa;
862 	addrs_len += gw_sa->sa_len;
863 
864 	src_sa = nh->nh_ifa->ifa_addr;
865 	if (src_sa->sa_family == AF_LINK) {
866 		/* Shorten structure */
867 		memset(&ss, 0, sizeof(struct sockaddr_storage));
868 		fill_sdl_from_ifp((struct sockaddr_dl_short *)&ss,
869 		    nh->nh_ifa->ifa_ifp);
870 		src_sa = (struct sockaddr *)&ss;
871 	}
872 	arpc.na.src_sa_off = addrs_len;
873 	addrs_len += src_sa->sa_len;
874 
875 	/* Write total container length */
876 	arpc.na.na_len = addrs_len;
877 
878 	arpc.rtm.rtm_msglen += arpc.na.na_len - sizeof(struct nhop_addrs);
879 
880 	error = SYSCTL_OUT(w, &arpc, sizeof(arpc));
881 	if (error == 0)
882 		error = SYSCTL_OUT(w, gw_sa, gw_sa->sa_len);
883 	if (error == 0)
884 		error = SYSCTL_OUT(w, src_sa, src_sa->sa_len);
885 
886 	return (error);
887 }
888 
889 uint32_t
890 nhops_get_count(struct rib_head *rh)
891 {
892 	struct nh_control *ctl;
893 	uint32_t count;
894 
895 	ctl = rh->nh_control;
896 
897 	NHOPS_RLOCK(ctl);
898 	count = ctl->nh_head.items_count;
899 	NHOPS_RUNLOCK(ctl);
900 
901 	return (count);
902 }
903 
904 int
905 nhops_dump_sysctl(struct rib_head *rh, struct sysctl_req *w)
906 {
907 	struct nh_control *ctl;
908 	struct nhop_priv *nh_priv;
909 	int error;
910 
911 	ctl = rh->nh_control;
912 
913 	NHOPS_RLOCK(ctl);
914 	DPRINTF("NHDUMP: count=%u", ctl->nh_head.items_count);
915 	CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) {
916 		error = dump_nhop_entry(rh, nh_priv->nh, w);
917 		if (error != 0) {
918 			NHOPS_RUNLOCK(ctl);
919 			return (error);
920 		}
921 	} CHT_SLIST_FOREACH_END;
922 	NHOPS_RUNLOCK(ctl);
923 
924 	return (0);
925 }
926