xref: /freebsd/sys/net/route/nhop_ctl.c (revision 5b0945b57059d1cde0831d3afea7ec56c7d79508)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2020 Alexander V. Chernikov
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 #include "opt_inet.h"
31 #include "opt_route.h"
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/lock.h>
36 #include <sys/rwlock.h>
37 #include <sys/malloc.h>
38 #include <sys/socket.h>
39 #include <sys/sysctl.h>
40 #include <sys/kernel.h>
41 #include <sys/epoch.h>
42 
43 #include <net/if.h>
44 #include <net/if_var.h>
45 #include <net/if_dl.h>
46 #include <net/route.h>
47 #include <net/route/route_ctl.h>
48 #include <net/route/route_var.h>
49 #include <net/route/nhop_utils.h>
50 #include <net/route/nhop.h>
51 #include <net/route/nhop_var.h>
52 #include <net/route/shared.h>
53 #include <net/vnet.h>
54 
55 /*
56  * This file contains core functionality for the nexthop ("nhop") route subsystem.
57  * The business logic needed to create nexhop objects is implemented here.
58  *
59  * Nexthops in the original sense are the objects containing all the necessary
60  * information to forward the packet to the selected destination.
61  * In particular, nexthop is defined by a combination of
62  *  ifp, ifa, aifp, mtu, gw addr(if set), nh_type, nh_family, mask of rt_flags and
63  *    NHF_DEFAULT
64  *
65  * Additionally, each nexthop gets assigned its unique index (nexthop index).
66  * It serves two purposes: first one is to ease the ability of userland programs to
67  *  reference nexthops by their index. The second one allows lookup algorithms to
68  *  to store index instead of pointer (2 bytes vs 8) as a lookup result.
69  * All nexthops are stored in the resizable hash table.
70  *
71  * Basically, this file revolves around supporting 3 functions:
72  * 1) nhop_create_from_info / nhop_create_from_nhop, which contains all
73  *  business logic on filling the nexthop fields based on the provided request.
74  * 2) nhop_get(), which gets a usable referenced nexthops.
75  *
76  * Conventions:
77  * 1) non-exported functions start with verb
78  * 2) exported function starts with the subsystem prefix: "nhop"
79  */
80 
81 static int dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w);
82 
83 static struct nhop_priv *alloc_nhop_structure(void);
84 static int get_nhop(struct rib_head *rnh, struct rt_addrinfo *info,
85     struct nhop_priv **pnh_priv);
86 static int finalize_nhop(struct nh_control *ctl, struct rt_addrinfo *info,
87     struct nhop_priv *nh_priv);
88 static struct ifnet *get_aifp(const struct nhop_object *nh, int reference);
89 static void fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp);
90 
91 static void destroy_nhop_epoch(epoch_context_t ctx);
92 static void destroy_nhop(struct nhop_priv *nh_priv);
93 
94 static void print_nhop(const char *prefix, const struct nhop_object *nh);
95 
96 _Static_assert(__offsetof(struct nhop_object, nh_ifp) == 32,
97     "nhop_object: wrong nh_ifp offset");
98 _Static_assert(sizeof(struct nhop_object) <= 128,
99     "nhop_object: size exceeds 128 bytes");
100 
101 static uma_zone_t nhops_zone;	/* Global zone for each and every nexthop */
102 
103 
104 #define	NHOP_OBJECT_ALIGNED_SIZE	roundup2(sizeof(struct nhop_object), \
105 							2 * CACHE_LINE_SIZE)
106 #define	NHOP_PRIV_ALIGNED_SIZE		roundup2(sizeof(struct nhop_priv), \
107 							2 * CACHE_LINE_SIZE)
108 void
109 nhops_init(void)
110 {
111 
112 	nhops_zone = uma_zcreate("routing nhops",
113 	    NHOP_OBJECT_ALIGNED_SIZE + NHOP_PRIV_ALIGNED_SIZE,
114 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
115 }
116 
117 /*
118  * Fetches the interface of source address used by the route.
119  * In all cases except interface-address-route it would be the
120  * same as the transmit interfaces.
121  * However, for the interface address this function will return
122  * this interface ifp instead of loopback. This is needed to support
123  * link-local IPv6 loopback communications.
124  *
125  * If @reference is non-zero, found ifp is referenced.
126  *
127  * Returns found ifp.
128  */
129 static struct ifnet *
130 get_aifp(const struct nhop_object *nh, int reference)
131 {
132 	struct ifnet *aifp = NULL;
133 
134 	/*
135 	 * Adjust the "outgoing" interface.  If we're going to loop
136 	 * the packet back to ourselves, the ifp would be the loopback
137 	 * interface. However, we'd rather know the interface associated
138 	 * to the destination address (which should probably be one of
139 	 * our own addresses).
140 	 */
141 	if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) &&
142 			nh->gw_sa.sa_family == AF_LINK) {
143 		if (reference)
144 			aifp = ifnet_byindex_ref(nh->gwl_sa.sdl_index);
145 		else
146 			aifp = ifnet_byindex(nh->gwl_sa.sdl_index);
147 		if (aifp == NULL) {
148 			DPRINTF("unable to get aifp for %s index %d",
149 				if_name(nh->nh_ifp), nh->gwl_sa.sdl_index);
150 		}
151 	}
152 
153 	if (aifp == NULL) {
154 		aifp = nh->nh_ifp;
155 		if (reference)
156 			if_ref(aifp);
157 	}
158 
159 	return (aifp);
160 }
161 
162 int
163 cmp_priv(const struct nhop_priv *_one, const struct nhop_priv *_two)
164 {
165 
166 	if (memcmp(_one->nh, _two->nh, NHOP_END_CMP) != 0)
167 		return (0);
168 
169 	if ((_one->nh_type != _two->nh_type) ||
170 	    (_one->nh_family != _two->nh_family))
171 		return (0);
172 
173 	return (1);
174 }
175 
176 /*
177  * Conditionally sets @nh mtu data based on the @info data.
178  */
179 static void
180 set_nhop_mtu_from_info(struct nhop_object *nh, const struct rt_addrinfo *info)
181 {
182 
183 	if (info->rti_mflags & RTV_MTU) {
184 		if (info->rti_rmx->rmx_mtu != 0) {
185 
186 			/*
187 			 * MTU was explicitly provided by user.
188 			 * Keep it.
189 			 */
190 
191 			nh->nh_priv->rt_flags |= RTF_FIXEDMTU;
192 		} else {
193 
194 			/*
195 			 * User explicitly sets MTU to 0.
196 			 * Assume rollback to default.
197 			 */
198 			nh->nh_priv->rt_flags &= ~RTF_FIXEDMTU;
199 		}
200 		nh->nh_mtu = info->rti_rmx->rmx_mtu;
201 	}
202 }
203 
204 /*
205  * Fills in shorted link-level sockadd version suitable to be stored inside the
206  *  nexthop gateway buffer.
207  */
208 static void
209 fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp)
210 {
211 
212 	sdl->sdl_family = AF_LINK;
213 	sdl->sdl_len = sizeof(struct sockaddr_dl_short);
214 	sdl->sdl_index = ifp->if_index;
215 	sdl->sdl_type = ifp->if_type;
216 }
217 
218 static int
219 set_nhop_gw_from_info(struct nhop_object *nh, struct rt_addrinfo *info)
220 {
221 	struct sockaddr *gw;
222 
223 	gw = info->rti_info[RTAX_GATEWAY];
224 	if (info->rti_flags & RTF_GATEWAY) {
225 		if (gw->sa_len > sizeof(struct sockaddr_in6)) {
226 			DPRINTF("nhop SA size too big: AF %d len %u",
227 			    gw->sa_family, gw->sa_len);
228 			return (ENOMEM);
229 		}
230 		memcpy(&nh->gw_sa, gw, gw->sa_len);
231 	} else {
232 		/*
233 		 * Interface route. Currently the route.c code adds
234 		 * sa of type AF_LINK, which is 56 bytes long. The only
235 		 * meaningful data there is the interface index. It is used
236 		 * used is the IPv6 loopback output, where we need to preserve
237 		 * the original interface to maintain proper scoping.
238 		 * Despite the fact that nexthop code stores original interface
239 		 * in the separate field (nh_aifp, see below), write AF_LINK
240 		 * compatible sa with shorter total length.
241 		 */
242 		fill_sdl_from_ifp(&nh->gwl_sa, nh->nh_ifp);
243 	}
244 
245 	return (0);
246 }
247 
248 static int
249 fill_nhop_from_info(struct nhop_priv *nh_priv, struct rt_addrinfo *info)
250 {
251 	int error, rt_flags;
252 	struct nhop_object *nh;
253 
254 	nh = nh_priv->nh;
255 
256 	rt_flags = info->rti_flags & NHOP_RT_FLAG_MASK;
257 
258 	nh->nh_priv->rt_flags = rt_flags;
259 	nh_priv->nh_family = info->rti_info[RTAX_DST]->sa_family;
260 	nh_priv->nh_type = 0; // hook responsibility to set nhop type
261 
262 	nh->nh_flags = fib_rte_to_nh_flags(rt_flags);
263 	set_nhop_mtu_from_info(nh, info);
264 	nh->nh_ifp = info->rti_ifa->ifa_ifp;
265 	nh->nh_ifa = info->rti_ifa;
266 	nh->nh_aifp = get_aifp(nh, 0);
267 
268 	if ((error = set_nhop_gw_from_info(nh, info)) != 0)
269 		return (error);
270 
271 	/*
272 	 * Note some of the remaining data is set by the
273 	 * per-address-family pre-add hook.
274 	 */
275 
276 	return (0);
277 }
278 
279 /*
280  * Creates a new nexthop based on the information in @info.
281  *
282  * Returns:
283  * 0 on success, filling @nh_ret with the desired nexthop object ptr
284  * errno otherwise
285  */
286 int
287 nhop_create_from_info(struct rib_head *rnh, struct rt_addrinfo *info,
288     struct nhop_object **nh_ret)
289 {
290 	struct nhop_priv *nh_priv;
291 	int error;
292 
293 	NET_EPOCH_ASSERT();
294 
295 	nh_priv = alloc_nhop_structure();
296 
297 	error = fill_nhop_from_info(nh_priv, info);
298 	if (error != 0) {
299 		uma_zfree(nhops_zone, nh_priv->nh);
300 		return (error);
301 	}
302 
303 	error = get_nhop(rnh, info, &nh_priv);
304 	if (error == 0)
305 		*nh_ret = nh_priv->nh;
306 
307 	return (error);
308 }
309 
310 /*
311  * Gets linked nhop using the provided @pnh_priv nexhop data.
312  * If linked nhop is found, returns it, freeing the provided one.
313  * If there is no such nexthop, attaches the remaining data to the
314  *  provided nexthop and links it.
315  *
316  * Returns 0 on success, storing referenced nexthop in @pnh_priv.
317  * Otherwise, errno is returned.
318  */
319 static int
320 get_nhop(struct rib_head *rnh, struct rt_addrinfo *info,
321     struct nhop_priv **pnh_priv)
322 {
323 	const struct sockaddr *dst, *gateway, *netmask;
324 	struct nhop_priv *nh_priv, *tmp_priv;
325 	int error;
326 
327 	nh_priv = *pnh_priv;
328 
329 	/* Give the protocols chance to augment the request data */
330 	dst = info->rti_info[RTAX_DST];
331 	netmask = info->rti_info[RTAX_NETMASK];
332 	gateway = info->rti_info[RTAX_GATEWAY];
333 
334 	error = rnh->rnh_preadd(rnh->rib_fibnum, dst, netmask, nh_priv->nh);
335 	if (error != 0) {
336 		uma_zfree(nhops_zone, nh_priv->nh);
337 		return (error);
338 	}
339 
340 	tmp_priv = find_nhop(rnh->nh_control, nh_priv);
341 	if (tmp_priv != NULL) {
342 		uma_zfree(nhops_zone, nh_priv->nh);
343 		*pnh_priv = tmp_priv;
344 		return (0);
345 	}
346 
347 	/*
348 	 * Existing nexthop not found, need to create new one.
349 	 * Note: multiple simultaneous get_nhop() requests
350 	 *  can result in multiple equal nexhops existing in the
351 	 *  nexthop table. This is not a not a problem until the
352 	 *  relative number of such nexthops is significant, which
353 	 *  is extremely unlikely.
354 	 */
355 
356 	error = finalize_nhop(rnh->nh_control, info, nh_priv);
357 	if (error != 0)
358 		return (error);
359 
360 	return (0);
361 }
362 
363 /*
364  * Update @nh with data supplied in @info.
365  * This is a helper function to support route changes.
366  *
367  * It limits the changes that can be done to the route to the following:
368  * 1) all combination of gateway changes (gw, interface, blackhole/reject)
369  * 2) route flags (FLAG[123],STATIC,BLACKHOLE,REJECT)
370  * 3) route MTU
371  *
372  * Returns:
373  * 0 on success
374  */
375 static int
376 alter_nhop_from_info(struct nhop_object *nh, struct rt_addrinfo *info)
377 {
378 	struct sockaddr *info_gw;
379 	int error;
380 
381 	/* Update MTU if set in the request*/
382 	set_nhop_mtu_from_info(nh, info);
383 
384 	/* XXX: allow only one of BLACKHOLE,REJECT,GATEWAY */
385 
386 	/* Allow some flags (FLAG1,STATIC,BLACKHOLE,REJECT) to be toggled on change. */
387 	nh->nh_priv->rt_flags &= ~RTF_FMASK;
388 	nh->nh_priv->rt_flags |= info->rti_flags & RTF_FMASK;
389 
390 	/* Consider gateway change */
391 	info_gw = info->rti_info[RTAX_GATEWAY];
392 	if (info_gw != NULL) {
393 		error = set_nhop_gw_from_info(nh, info);
394 		if (error != 0)
395 			return (error);
396 		/* Update RTF_GATEWAY flag status */
397 		nh->nh_priv->rt_flags &= ~RTF_GATEWAY;
398 		nh->nh_priv->rt_flags |= (RTF_GATEWAY & info->rti_flags);
399 	}
400 	/* Update datapath flags */
401 	nh->nh_flags = fib_rte_to_nh_flags(nh->nh_priv->rt_flags);
402 
403 	if (info->rti_ifa != NULL)
404 		nh->nh_ifa = info->rti_ifa;
405 	if (info->rti_ifp != NULL)
406 		nh->nh_ifp = info->rti_ifp;
407 	nh->nh_aifp = get_aifp(nh, 0);
408 
409 	return (0);
410 }
411 
412 /*
413  * Creates new nexthop based on @nh_orig and augmentation data from @info.
414  * Helper function used in the route changes, please see
415  *   alter_nhop_from_info() comments for more details.
416  *
417  * Returns:
418  * 0 on success, filling @nh_ret with the desired nexthop object
419  * errno otherwise
420  */
421 int
422 nhop_create_from_nhop(struct rib_head *rnh, const struct nhop_object *nh_orig,
423     struct rt_addrinfo *info, struct nhop_object **pnh)
424 {
425 	struct nhop_priv *nh_priv;
426 	struct nhop_object *nh;
427 	int error;
428 
429 	NET_EPOCH_ASSERT();
430 
431 	nh_priv = alloc_nhop_structure();
432 	nh = nh_priv->nh;
433 
434 	/* Start with copying data from original nexthop */
435 	nh_priv->nh_family = nh_orig->nh_priv->nh_family;
436 	nh_priv->rt_flags = nh_orig->nh_priv->rt_flags;
437 	nh_priv->nh_type = nh_orig->nh_priv->nh_type;
438 
439 	nh->nh_ifp = nh_orig->nh_ifp;
440 	nh->nh_ifa = nh_orig->nh_ifa;
441 	nh->nh_aifp = nh_orig->nh_aifp;
442 	nh->nh_mtu = nh_orig->nh_mtu;
443 	nh->nh_flags = nh_orig->nh_flags;
444 	memcpy(&nh->gw_sa, &nh_orig->gw_sa, nh_orig->gw_sa.sa_len);
445 
446 	error = alter_nhop_from_info(nh, info);
447 	if (error != 0) {
448 		uma_zfree(nhops_zone, nh_priv->nh);
449 		return (error);
450 	}
451 
452 	error = get_nhop(rnh, info, &nh_priv);
453 	if (error == 0)
454 		*pnh = nh_priv->nh;
455 
456 	return (error);
457 }
458 
459 /*
460  * Allocates memory for public/private nexthop structures.
461  *
462  * Returns pointer to nhop_priv or NULL.
463  */
464 static struct nhop_priv *
465 alloc_nhop_structure()
466 {
467 	struct nhop_object *nh;
468 	struct nhop_priv *nh_priv;
469 
470 	nh = (struct nhop_object *)uma_zalloc(nhops_zone, M_NOWAIT | M_ZERO);
471 	if (nh == NULL)
472 		return (NULL);
473 	nh_priv = (struct nhop_priv *)((char *)nh + NHOP_OBJECT_ALIGNED_SIZE);
474 
475 	nh->nh_priv = nh_priv;
476 	nh_priv->nh = nh;
477 
478 	return (nh_priv);
479 }
480 
481 /*
482  * Alocates/references the remaining bits of nexthop data and links
483  *  it to the hash table.
484  * Returns 0 if successful,
485  *  errno otherwise. @nh_priv is freed in case of error.
486  */
487 static int
488 finalize_nhop(struct nh_control *ctl, struct rt_addrinfo *info,
489     struct nhop_priv *nh_priv)
490 {
491 	struct nhop_object *nh;
492 
493 	nh = nh_priv->nh;
494 
495 	/* Allocate per-cpu packet counter */
496 	nh->nh_pksent = counter_u64_alloc(M_NOWAIT);
497 	if (nh->nh_pksent == NULL) {
498 		uma_zfree(nhops_zone, nh);
499 		RTSTAT_INC(rts_nh_alloc_failure);
500 		DPRINTF("nh_alloc_finalize failed");
501 		return (ENOMEM);
502 	}
503 
504 	/* Save vnet to ease destruction */
505 	nh_priv->nh_vnet = curvnet;
506 
507 	/* Reference external objects and calculate (referenced) ifa */
508 	if_ref(nh->nh_ifp);
509 	ifa_ref(nh->nh_ifa);
510 	nh->nh_aifp = get_aifp(nh, 1);
511 	DPRINTF("AIFP: %p nh_ifp %p", nh->nh_aifp, nh->nh_ifp);
512 
513 	refcount_init(&nh_priv->nh_refcnt, 1);
514 
515 	/* Please see nhop_free() comments on the initial value */
516 	refcount_init(&nh_priv->nh_linked, 2);
517 
518 	print_nhop("FINALIZE", nh);
519 
520 	if (link_nhop(ctl, nh_priv) == 0) {
521 
522 		/*
523 		 * Adding nexthop to the datastructures
524 		 *  failed. Call destructor w/o waiting for
525 		 *  the epoch end, as nexthop is not used
526 		 *  and return.
527 		 */
528 		DPRINTF("link_nhop failed!");
529 		destroy_nhop(nh_priv);
530 
531 		return (ENOBUFS);
532 	}
533 
534 	return (0);
535 }
536 
537 static void
538 print_nhop_sa(char *buf, size_t buflen, const struct sockaddr *sa)
539 {
540 
541 	if (sa->sa_family == AF_INET) {
542 		const struct sockaddr_in *sin4;
543 		sin4 = (const struct sockaddr_in *)sa;
544 		inet_ntop(AF_INET, &sin4->sin_addr, buf, buflen);
545 	} else if (sa->sa_family == AF_INET6) {
546 		const struct sockaddr_in6 *sin6;
547 		sin6 = (const struct sockaddr_in6 *)sa;
548 		inet_ntop(AF_INET6, &sin6->sin6_addr, buf, buflen);
549 	} else if (sa->sa_family == AF_LINK) {
550 		const struct sockaddr_dl *sdl;
551 		sdl = (const struct sockaddr_dl *)sa;
552 		snprintf(buf, buflen, "if#%d", sdl->sdl_index);
553 	} else
554 		snprintf(buf, buflen, "af:%d", sa->sa_family);
555 }
556 
557 static void
558 print_nhop(const char *prefix, const struct nhop_object *nh)
559 {
560 	char src_buf[INET6_ADDRSTRLEN], addr_buf[INET6_ADDRSTRLEN];
561 
562 	print_nhop_sa(src_buf, sizeof(src_buf), nh->nh_ifa->ifa_addr);
563 	print_nhop_sa(addr_buf, sizeof(addr_buf), &nh->gw_sa);
564 
565 	DPRINTF("%s nhop priv %p: AF %d ifp %p %s addr %s src %p %s aifp %p %s mtu %d nh_flags %X",
566 	    prefix, nh->nh_priv, nh->nh_priv->nh_family, nh->nh_ifp,
567 	    if_name(nh->nh_ifp), addr_buf, nh->nh_ifa, src_buf, nh->nh_aifp,
568 	    if_name(nh->nh_aifp), nh->nh_mtu, nh->nh_flags);
569 }
570 
571 static void
572 destroy_nhop(struct nhop_priv *nh_priv)
573 {
574 	struct nhop_object *nh;
575 
576 	nh = nh_priv->nh;
577 
578 	print_nhop("DEL", nh);
579 
580 	if_rele(nh->nh_ifp);
581 	if_rele(nh->nh_aifp);
582 	ifa_free(nh->nh_ifa);
583 	counter_u64_free(nh->nh_pksent);
584 
585 	uma_zfree(nhops_zone, nh);
586 }
587 
588 /*
589  * Epoch callback indicating nhop is safe to destroy
590  */
591 static void
592 destroy_nhop_epoch(epoch_context_t ctx)
593 {
594 	struct nhop_priv *nh_priv;
595 
596 	nh_priv = __containerof(ctx, struct nhop_priv, nh_epoch_ctx);
597 
598 	destroy_nhop(nh_priv);
599 }
600 
601 void
602 nhop_ref_object(struct nhop_object *nh)
603 {
604 	u_int old;
605 
606 	old = refcount_acquire(&nh->nh_priv->nh_refcnt);
607 	KASSERT(old > 0, ("%s: nhop object %p has 0 refs", __func__, nh));
608 }
609 
610 int
611 nhop_try_ref_object(struct nhop_object *nh)
612 {
613 
614 	return (refcount_acquire_if_not_zero(&nh->nh_priv->nh_refcnt));
615 }
616 
617 void
618 nhop_free(struct nhop_object *nh)
619 {
620 	struct nh_control *ctl;
621 	struct nhop_priv *nh_priv = nh->nh_priv;
622 	struct epoch_tracker et;
623 
624 	if (!refcount_release(&nh_priv->nh_refcnt))
625 		return;
626 
627 	/*
628 	 * There are only 2 places, where nh_linked can be decreased:
629 	 *  rib destroy (nhops_destroy_rib) and this function.
630 	 * nh_link can never be increased.
631 	 *
632 	 * Hence, use initial value of 2 to make use of
633 	 *  refcount_release_if_not_last().
634 	 *
635 	 * There can be two scenarious when calling this function:
636 	 *
637 	 * 1) nh_linked value is 2. This means that either
638 	 *  nhops_destroy_rib() has not been called OR it is running,
639 	 *  but we are guaranteed that nh_control won't be freed in
640 	 *  this epoch. Hence, nexthop can be safely unlinked.
641 	 *
642 	 * 2) nh_linked value is 1. In that case, nhops_destroy_rib()
643 	 *  has been called and nhop unlink can be skipped.
644 	 */
645 
646 	NET_EPOCH_ENTER(et);
647 	if (refcount_release_if_not_last(&nh_priv->nh_linked)) {
648 		ctl = nh_priv->nh_control;
649 		if (unlink_nhop(ctl, nh_priv) == NULL) {
650 			/* Do not try to reclaim */
651 			DPRINTF("Failed to unlink nexhop %p", nh_priv);
652 			NET_EPOCH_EXIT(et);
653 			return;
654 		}
655 	}
656 	NET_EPOCH_EXIT(et);
657 
658 	epoch_call(net_epoch_preempt, destroy_nhop_epoch,
659 	    &nh_priv->nh_epoch_ctx);
660 }
661 
662 int
663 nhop_ref_any(struct nhop_object *nh)
664 {
665 
666 	return (nhop_try_ref_object(nh));
667 }
668 
669 void
670 nhop_free_any(struct nhop_object *nh)
671 {
672 
673 	nhop_free(nh);
674 }
675 
676 
677 /* Helper functions */
678 
679 uint32_t
680 nhop_get_idx(const struct nhop_object *nh)
681 {
682 
683 	return (nh->nh_priv->nh_idx);
684 }
685 
686 enum nhop_type
687 nhop_get_type(const struct nhop_object *nh)
688 {
689 
690 	return (nh->nh_priv->nh_type);
691 }
692 
693 void
694 nhop_set_type(struct nhop_object *nh, enum nhop_type nh_type)
695 {
696 
697 	nh->nh_priv->nh_type = nh_type;
698 }
699 
700 int
701 nhop_get_rtflags(const struct nhop_object *nh)
702 {
703 
704 	return (nh->nh_priv->rt_flags);
705 }
706 
707 void
708 nhop_set_rtflags(struct nhop_object *nh, int rt_flags)
709 {
710 
711 	nh->nh_priv->rt_flags = rt_flags;
712 }
713 
714 struct vnet *
715 nhop_get_vnet(const struct nhop_object *nh)
716 {
717 
718 	return (nh->nh_priv->nh_vnet);
719 }
720 
721 void
722 nhops_update_ifmtu(struct rib_head *rh, struct ifnet *ifp, uint32_t mtu)
723 {
724 	struct nh_control *ctl;
725 	struct nhop_priv *nh_priv;
726 	struct nhop_object *nh;
727 
728 	ctl = rh->nh_control;
729 
730 	NHOPS_WLOCK(ctl);
731 	CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) {
732 		nh = nh_priv->nh;
733 		if (nh->nh_ifp == ifp) {
734 			if ((nh_priv->rt_flags & RTF_FIXEDMTU) == 0 ||
735 			    nh->nh_mtu > mtu) {
736 				/* Update MTU directly */
737 				nh->nh_mtu = mtu;
738 			}
739 		}
740 	} CHT_SLIST_FOREACH_END;
741 	NHOPS_WUNLOCK(ctl);
742 
743 }
744 
745 /*
746  * Dumps a single entry to sysctl buffer.
747  *
748  * Layout:
749  *  rt_msghdr - generic RTM header to allow users to skip non-understood messages
750  *  nhop_external - nexhop description structure (with length)
751  *  nhop_addrs - structure encapsulating GW/SRC sockaddrs
752  */
753 static int
754 dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w)
755 {
756 	struct {
757 		struct rt_msghdr	rtm;
758 		struct nhop_external	nhe;
759 		struct nhop_addrs	na;
760 	} arpc;
761 	struct nhop_external *pnhe;
762 	struct sockaddr *gw_sa, *src_sa;
763 	struct sockaddr_storage ss;
764 	size_t addrs_len;
765 	int error;
766 
767 	//DPRINTF("Dumping: head %p nh %p flags %X req %p\n", rh, nh, nh->nh_flags, w);
768 
769 	memset(&arpc, 0, sizeof(arpc));
770 
771 	arpc.rtm.rtm_msglen = sizeof(arpc);
772 	arpc.rtm.rtm_version = RTM_VERSION;
773 	arpc.rtm.rtm_type = RTM_GET;
774 	//arpc.rtm.rtm_flags = RTF_UP;
775 	arpc.rtm.rtm_flags = nh->nh_priv->rt_flags;
776 
777 	/* nhop_external */
778 	pnhe = &arpc.nhe;
779 	pnhe->nh_len = sizeof(struct nhop_external);
780 	pnhe->nh_idx = nh->nh_priv->nh_idx;
781 	pnhe->nh_fib = rh->rib_fibnum;
782 	pnhe->ifindex = nh->nh_ifp->if_index;
783 	pnhe->aifindex = nh->nh_aifp->if_index;
784 	pnhe->nh_family = nh->nh_priv->nh_family;
785 	pnhe->nh_type = nh->nh_priv->nh_type;
786 	pnhe->nh_mtu = nh->nh_mtu;
787 	pnhe->nh_flags = nh->nh_flags;
788 
789 	memcpy(pnhe->nh_prepend, nh->nh_prepend, sizeof(nh->nh_prepend));
790 	pnhe->prepend_len = nh->nh_prepend_len;
791 	pnhe->nh_refcount = nh->nh_priv->nh_refcnt;
792 	pnhe->nh_pksent = counter_u64_fetch(nh->nh_pksent);
793 
794 	/* sockaddr container */
795 	addrs_len = sizeof(struct nhop_addrs);
796 	arpc.na.gw_sa_off = addrs_len;
797 	gw_sa = (struct sockaddr *)&nh->gw4_sa;
798 	addrs_len += gw_sa->sa_len;
799 
800 	src_sa = nh->nh_ifa->ifa_addr;
801 	if (src_sa->sa_family == AF_LINK) {
802 		/* Shorten structure */
803 		memset(&ss, 0, sizeof(struct sockaddr_storage));
804 		fill_sdl_from_ifp((struct sockaddr_dl_short *)&ss,
805 		    nh->nh_ifa->ifa_ifp);
806 		src_sa = (struct sockaddr *)&ss;
807 	}
808 	arpc.na.src_sa_off = addrs_len;
809 	addrs_len += src_sa->sa_len;
810 
811 	/* Write total container length */
812 	arpc.na.na_len = addrs_len;
813 
814 	arpc.rtm.rtm_msglen += arpc.na.na_len - sizeof(struct nhop_addrs);
815 
816 	error = SYSCTL_OUT(w, &arpc, sizeof(arpc));
817 	if (error == 0)
818 		error = SYSCTL_OUT(w, gw_sa, gw_sa->sa_len);
819 	if (error == 0)
820 		error = SYSCTL_OUT(w, src_sa, src_sa->sa_len);
821 
822 	return (error);
823 }
824 
825 int
826 nhops_dump_sysctl(struct rib_head *rh, struct sysctl_req *w)
827 {
828 	struct nh_control *ctl;
829 	struct nhop_priv *nh_priv;
830 	int error;
831 
832 	ctl = rh->nh_control;
833 
834 	NHOPS_RLOCK(ctl);
835 	DPRINTF("NHDUMP: count=%u", ctl->nh_head.items_count);
836 	CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) {
837 		error = dump_nhop_entry(rh, nh_priv->nh, w);
838 		if (error != 0) {
839 			NHOPS_RUNLOCK(ctl);
840 			return (error);
841 		}
842 	} CHT_SLIST_FOREACH_END;
843 	NHOPS_RUNLOCK(ctl);
844 
845 	return (0);
846 }
847 
848