58 #define for_nexthops(fi) {						\  argument
60 	for (nhsel = 0, nh = (fi)->fib_nh;				\
61 	     nhsel < fib_info_num_path((fi));				\
64 #define change_nexthops(fi) {						\  argument
66 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
67 	     nhsel < fib_info_num_path((fi));				\
74 #define for_nexthops(fi) {						\  argument
75 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
78 #define change_nexthops(fi) {						\  argument
80 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
85 #define endfor_nexthops(fi) }  argument
224 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
226 	if (fi->nh) {  in free_fib_info_rcu()
227 		nexthop_put(fi->nh);  in free_fib_info_rcu()
229 		change_nexthops(fi) {  in free_fib_info_rcu()
230 			fib_nh_release(fi->fib_net, nexthop_nh);  in free_fib_info_rcu()
231 		} endfor_nexthops(fi);  in free_fib_info_rcu()
234 	ip_fib_metrics_put(fi->fib_metrics);  in free_fib_info_rcu()
236 	kfree(fi);  in free_fib_info_rcu()
239 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
241 	if (fi->fib_dead == 0) {  in free_fib_info()
242 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
246 	call_rcu_hurry(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
250 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
253 	if (fi && refcount_dec_and_test(&fi->fib_treeref)) {  in fib_release_info()
254 		hlist_del(&fi->fib_hash);  in fib_release_info()
255 		fi->fib_net->ipv4.fib_info_cnt--;  in fib_release_info()
257 		if (fi->fib_prefsrc)  in fib_release_info()
258 			hlist_del(&fi->fib_lhash);  in fib_release_info()
259 		if (fi->nh) {  in fib_release_info()
260 			list_del(&fi->nh_list);  in fib_release_info()
262 			change_nexthops(fi) {  in fib_release_info()
266 			} endfor_nexthops(fi)  in fib_release_info()
269 		WRITE_ONCE(fi->fib_dead, 1);  in fib_release_info()
270 		fib_info_put(fi);  in fib_release_info()
274 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)  in nh_comp()  argument
278 	if (fi->nh || ofi->nh)  in nh_comp()
279 		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;  in nh_comp()
284 	for_nexthops(fi) {  in nh_comp()
307 	} endfor_nexthops(fi);  in nh_comp()
334 static struct hlist_head *fib_info_hash_bucket(struct fib_info *fi)  in fib_info_hash_bucket()  argument
336 	struct net *net = fi->fib_net;  in fib_info_hash_bucket()
339 	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,  in fib_info_hash_bucket()
340 				fi->fib_scope, (__force u32)fi->fib_prefsrc,  in fib_info_hash_bucket()
341 				fi->fib_priority);  in fib_info_hash_bucket()
343 	if (fi->nh) {  in fib_info_hash_bucket()
344 		val ^= fi->nh->id;  in fib_info_hash_bucket()
346 		for_nexthops(fi) {  in fib_info_hash_bucket()
348 		} endfor_nexthops(fi)  in fib_info_hash_bucket()
397 		struct fib_info *fi;  in fib_info_hash_grow()  local
399 		hlist_for_each_entry_safe(fi, n, head, fib_hash)  in fib_info_hash_grow()
400 			hlist_add_head(&fi->fib_hash, fib_info_hash_bucket(fi));  in fib_info_hash_grow()
406 		struct fib_info *fi;  in fib_info_hash_grow()  local
408 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash)  in fib_info_hash_grow()
409 			hlist_add_head(&fi->fib_lhash,  in fib_info_hash_grow()
410 				       fib_info_laddrhash_bucket(fi->fib_net,  in fib_info_hash_grow()
411 								 fi->fib_prefsrc));  in fib_info_hash_grow()
422 	struct fib_info *fi;  in fib_find_info_nh()  local
432 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info_nh()
433 		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)  in fib_find_info_nh()
436 		if (cfg->fc_protocol == fi->fib_protocol &&  in fib_find_info_nh()
437 		    cfg->fc_scope == fi->fib_scope &&  in fib_find_info_nh()
438 		    cfg->fc_prefsrc == fi->fib_prefsrc &&  in fib_find_info_nh()
439 		    cfg->fc_priority == fi->fib_priority &&  in fib_find_info_nh()
440 		    cfg->fc_type == fi->fib_type &&  in fib_find_info_nh()
441 		    cfg->fc_table == fi->fib_tb_id &&  in fib_find_info_nh()
442 		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))  in fib_find_info_nh()
443 			return fi;  in fib_find_info_nh()
452 	struct fib_info *fi;  in fib_find_info()  local
454 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
455 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
458 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
459 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
460 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
461 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
462 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
463 		    nfi->fib_tb_id == fi->fib_tb_id &&  in fib_find_info()
464 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
466 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
467 		    nh_comp(fi, nfi) == 0)  in fib_find_info()
468 			return fi;  in fib_find_info()
495 size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
503 	unsigned int nhs = fib_info_num_path(fi);  in fib_nlmsg_size()
508 	if (fi->nh)  in fib_nlmsg_size()
523 		for (i = 0; i < fib_info_num_path(fi); i++) {  in fib_nlmsg_size()
524 			struct fib_nh_common *nhc = fib_info_nhc(fi, i);  in fib_nlmsg_size()
556 	fri.fi = fa->fa_info;  in rtmsg_fib()
579 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
583 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_detect_death()
607 		*last_resort = fi;  in fib_detect_death()
715 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
719 	struct net *net = fi->fib_net;  in fib_get_nhs()
724 	change_nexthops(fi) {  in fib_get_nhs()
793 	} endfor_nexthops(fi);  in fib_get_nhs()
796 	nh = fib_info_nh(fi, 0);  in fib_get_nhs()
826 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
831 	if (fib_info_num_path(fi) < 2)  in fib_rebalance()
835 	for_nexthops(fi) {  in fib_rebalance()
844 	} endfor_nexthops(fi);  in fib_rebalance()
847 	change_nexthops(fi) {  in fib_rebalance()
862 	} endfor_nexthops(fi);  in fib_rebalance()
866 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
875 #define fib_rebalance(fi) do { } while (0)  argument
901 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
909 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
913 		if (fi->nh && cfg->fc_nh_id == fi->nh->id)  in fib_nh_match()
918 	if (fi->nh) {  in fib_nh_match()
927 		nh = fib_info_nh(fi, 0);  in fib_nh_match()
961 	for_nexthops(fi) {  in fib_nh_match()
1029 	} endfor_nexthops(fi);  in fib_nh_match()
1034 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
1064 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
1310 	if (res->fi->fib_prefsrc)  in fib_result_prefsrc()
1311 		return res->fi->fib_prefsrc;  in fib_result_prefsrc()
1322 	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);  in fib_result_prefsrc()
1353 	struct fib_info *fi = NULL;  in fib_create_info()  local
1377 			fi = fib_find_info_nh(net, cfg);  in fib_create_info()
1378 			if (fi) {  in fib_create_info()
1379 				refcount_inc(&fi->fib_treeref);  in fib_create_info()
1380 				return fi;  in fib_create_info()
1402 	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);  in fib_create_info()
1403 	if (!fi) {  in fib_create_info()
1408 	fi->fib_metrics = ip_fib_metrics_init(cfg->fc_mx, cfg->fc_mx_len, extack);  in fib_create_info()
1409 	if (IS_ERR(fi->fib_metrics)) {  in fib_create_info()
1410 		err = PTR_ERR(fi->fib_metrics);  in fib_create_info()
1411 		kfree(fi);  in fib_create_info()
1415 	fi->fib_net = net;  in fib_create_info()
1416 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1417 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1418 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1419 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1420 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1421 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1422 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1424 	fi->fib_nhs = nhs;  in fib_create_info()
1431 			fi->nh = nh;  in fib_create_info()
1434 		change_nexthops(fi) {  in fib_create_info()
1435 			nexthop_nh->nh_parent = fi;  in fib_create_info()
1436 		} endfor_nexthops(fi)  in fib_create_info()
1439 			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,  in fib_create_info()
1442 			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);  in fib_create_info()
1474 	if (fi->nh) {  in fib_create_info()
1475 		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);  in fib_create_info()
1479 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1502 		change_nexthops(fi) {  in fib_create_info()
1510 		} endfor_nexthops(fi)  in fib_create_info()
1511 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1512 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1515 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1520 	if (!fi->nh) {  in fib_create_info()
1521 		change_nexthops(fi) {  in fib_create_info()
1523 						  fi->fib_scope);  in fib_create_info()
1525 				fi->fib_nh_is_v6 = true;  in fib_create_info()
1526 		} endfor_nexthops(fi)  in fib_create_info()
1528 		fib_rebalance(fi);  in fib_create_info()
1532 	ofi = fib_find_info(fi);  in fib_create_info()
1534 		/* fib_table_lookup() should not see @fi yet. */  in fib_create_info()
1535 		fi->fib_dead = 1;  in fib_create_info()
1536 		free_fib_info(fi);  in fib_create_info()
1541 	refcount_set(&fi->fib_treeref, 1);  in fib_create_info()
1542 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1545 	hlist_add_head(&fi->fib_hash, fib_info_hash_bucket(fi));  in fib_create_info()
1547 	if (fi->fib_prefsrc) {  in fib_create_info()
1550 		head = fib_info_laddrhash_bucket(net, fi->fib_prefsrc);  in fib_create_info()
1551 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1553 	if (fi->nh) {  in fib_create_info()
1554 		list_add(&fi->nh_list, &nh->fi_list);  in fib_create_info()
1556 		change_nexthops(fi) {  in fib_create_info()
1563 		} endfor_nexthops(fi)  in fib_create_info()
1565 	return fi;  in fib_create_info()
1571 	if (fi) {  in fib_create_info()
1572 		/* fib_table_lookup() should not see @fi yet. */  in fib_create_info()
1573 		fi->fib_dead = 1;  in fib_create_info()
1574 		free_fib_info(fi);  in fib_create_info()
1684 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1692 	if (unlikely(fi->nh)) {  in fib_add_multipath()
1693 		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)  in fib_add_multipath()
1698 	for_nexthops(fi) {  in fib_add_multipath()
1706 	} endfor_nexthops(fi);  in fib_add_multipath()
1717 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1726 	unsigned int nhs = fib_info_num_path(fri->fi);  in fib_dump_info()
1727 	struct fib_info *fi = fri->fi;  in fib_dump_info()  local
1748 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1749 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1750 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1755 	if (fi->fib_priority &&  in fib_dump_info()
1756 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1758 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1761 	if (fi->fib_prefsrc &&  in fib_dump_info()
1762 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1765 	if (fi->nh) {  in fib_dump_info()
1766 		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))  in fib_dump_info()
1768 		if (nexthop_is_blackhole(fi->nh))  in fib_dump_info()
1770 		if (!READ_ONCE(fi->fib_net->ipv4.sysctl_nexthop_compat_mode))  in fib_dump_info()
1775 		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_dump_info()
1793 		if (fib_add_multipath(skb, fi) < 0)  in fib_dump_info()
1824 	struct fib_info *fi;  in fib_sync_down_addr()  local
1831 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1832 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1833 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1835 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1836 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1837 			fi->pfsrc_removed = true;  in fib_sync_down_addr()
1942 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1945 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1947 		if (fi == prev_fi)  in fib_sync_down_dev()
1949 		prev_fi = fi;  in fib_sync_down_dev()
1951 		change_nexthops(fi) {  in fib_sync_down_dev()
1972 				dead = fi->fib_nhs;  in fib_sync_down_dev()
1976 		} endfor_nexthops(fi)  in fib_sync_down_dev()
1977 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
1981 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
1984 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
1990 		fib_rebalance(fi);  in fib_sync_down_dev()
1999 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
2005 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
2039 		if (!fi) {  in fib_select_default()
2040 			if (next_fi != res->fi)  in fib_select_default()
2043 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
2045 			fib_result_assign(res, fi);  in fib_select_default()
2049 		fi = next_fi;  in fib_select_default()
2053 	if (order <= 0 || !fi) {  in fib_select_default()
2059 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
2061 		fib_result_assign(res, fi);  in fib_select_default()
2101 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
2104 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
2106 		if (fi == prev_fi)  in fib_sync_up()
2109 		prev_fi = fi;  in fib_sync_up()
2111 		change_nexthops(fi) {  in fib_sync_up()
2125 		} endfor_nexthops(fi)  in fib_sync_up()
2128 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
2132 		fib_rebalance(fi);  in fib_sync_up()
2168 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
2169 	struct net *net = fi->fib_net;  in fib_select_multipath()
2174 	if (unlikely(res->fi->nh)) {  in fib_select_multipath()
2182 	change_nexthops(fi) {  in fib_select_multipath()
2211 	} endfor_nexthops(fi);  in fib_select_multipath()
2222 	if (fib_info_num_path(res->fi) > 1) {  in fib_select_path()