416 	const struct rtable *rt = container_of(dst, struct rtable, dst);  in ipv4_neigh_lookup()  local
422 	if (likely(rt->rt_gw_family == AF_INET)) {  in ipv4_neigh_lookup()
423 		n = ip_neigh_gw4(dev, rt->rt_gw4);  in ipv4_neigh_lookup()
424 	} else if (rt->rt_gw_family == AF_INET6) {  in ipv4_neigh_lookup()
425 		n = ip_neigh_gw6(dev, &rt->rt_gw6);  in ipv4_neigh_lookup()
443 	const struct rtable *rt = container_of(dst, struct rtable, dst);  in ipv4_confirm_neigh()  local
447 	if (rt->rt_gw_family == AF_INET) {  in ipv4_confirm_neigh()
448 		pkey = (const __be32 *)&rt->rt_gw4;  in ipv4_confirm_neigh()
449 	} else if (IS_ENABLED(CONFIG_IPV6) && rt->rt_gw_family == AF_INET6) {  in ipv4_confirm_neigh()
450 		return __ipv6_confirm_neigh(dev, &rt->rt_gw6);  in ipv4_confirm_neigh()
452 		 (rt->rt_flags &  in ipv4_confirm_neigh()
578 	struct rtable *rt;  in fnhe_flush_routes()  local
580 	rt = rcu_dereference(fnhe->fnhe_rth_input);  in fnhe_flush_routes()
581 	if (rt) {  in fnhe_flush_routes()
583 		dst_dev_put(&rt->dst);  in fnhe_flush_routes()
584 		dst_release(&rt->dst);  in fnhe_flush_routes()
586 	rt = rcu_dereference(fnhe->fnhe_rth_output);  in fnhe_flush_routes()
587 	if (rt) {  in fnhe_flush_routes()
589 		dst_dev_put(&rt->dst);  in fnhe_flush_routes()
590 		dst_release(&rt->dst);  in fnhe_flush_routes()
630 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)  in fill_route_from_fnhe()  argument
632 	rt->rt_pmtu = fnhe->fnhe_pmtu;  in fill_route_from_fnhe()
633 	rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;  in fill_route_from_fnhe()
634 	rt->dst.expires = fnhe->fnhe_expires;  in fill_route_from_fnhe()
637 		rt->rt_flags |= RTCF_REDIRECTED;  in fill_route_from_fnhe()
638 		rt->rt_uses_gateway = 1;  in fill_route_from_fnhe()
639 		rt->rt_gw_family = AF_INET;  in fill_route_from_fnhe()
640 		rt->rt_gw4 = fnhe->fnhe_gw;  in fill_route_from_fnhe()
650 	struct rtable *rt;  in update_or_create_fnhe()  local
689 		rt = rcu_dereference(fnhe->fnhe_rth_input);  in update_or_create_fnhe()
690 		if (rt)  in update_or_create_fnhe()
691 			fill_route_from_fnhe(rt, fnhe);  in update_or_create_fnhe()
692 		rt = rcu_dereference(fnhe->fnhe_rth_output);  in update_or_create_fnhe()
693 		if (rt)  in update_or_create_fnhe()
694 			fill_route_from_fnhe(rt, fnhe);  in update_or_create_fnhe()
724 		rt = rcu_dereference(nhc->nhc_rth_input);  in update_or_create_fnhe()
725 		if (rt)  in update_or_create_fnhe()
726 			WRITE_ONCE(rt->dst.obsolete, DST_OBSOLETE_KILL);  in update_or_create_fnhe()
732 			rt = rcu_dereference(*prt);  in update_or_create_fnhe()
733 			if (rt)  in update_or_create_fnhe()
734 				WRITE_ONCE(rt->dst.obsolete, DST_OBSOLETE_KILL);  in update_or_create_fnhe()
744 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,  in __ip_do_redirect()  argument
766 	if (rt->rt_gw_family != AF_INET || rt->rt_gw4 != old_gw)  in __ip_do_redirect()
789 	n = __ipv4_neigh_lookup(rt->dst.dev, (__force u32)new_gw);  in __ip_do_redirect()
791 		n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);  in __ip_do_redirect()
806 				WRITE_ONCE(rt->dst.obsolete, DST_OBSOLETE_KILL);  in __ip_do_redirect()
831 	struct rtable *rt;  in ip_do_redirect()  local
840 	rt = dst_rtable(dst);  in ip_do_redirect()
843 	__ip_do_redirect(rt, skb, &fl4, true);  in ip_do_redirect()
849 	struct rtable *rt = dst_rtable(dst);  in ipv4_negative_advice()  local
852 	    (rt->rt_flags & RTCF_REDIRECTED) ||  in ipv4_negative_advice()
853 	    READ_ONCE(rt->dst.expires))  in ipv4_negative_advice()
875 	struct rtable *rt = skb_rtable(skb);  in ip_rt_send_redirect()  local
883 	in_dev = __in_dev_get_rcu(rt->dst.dev);  in ip_rt_send_redirect()
889 	vif = l3mdev_master_ifindex_rcu(rt->dst.dev);  in ip_rt_send_redirect()
891 	net = dev_net(rt->dst.dev);  in ip_rt_send_redirect()
896 			  rt_nexthop(rt, ip_hdr(skb)->daddr));  in ip_rt_send_redirect()
923 		__be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);  in ip_rt_send_redirect()
940 	struct rtable *rt = skb_rtable(skb);  in ip_error()  local
962 	net = dev_net(rt->dst.dev);  in ip_error()
964 		switch (rt->dst.error) {  in ip_error()
978 	switch (rt->dst.error) {  in ip_error()
1019 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)  in __ip_rt_update_pmtu()  argument
1021 	struct dst_entry *dst = &rt->dst;  in __ip_rt_update_pmtu()
1041 	if (rt->rt_pmtu == mtu && !lock &&  in __ip_rt_update_pmtu()
1074 	struct rtable *rt = dst_rtable(dst);  in ip_rt_update_pmtu()  local
1083 	__ip_rt_update_pmtu(rt, &fl4, mtu);  in ip_rt_update_pmtu()
1091 	struct rtable *rt;  in ipv4_update_pmtu()  local
1096 	rt = __ip_route_output_key(net, &fl4);  in ipv4_update_pmtu()
1097 	if (!IS_ERR(rt)) {  in ipv4_update_pmtu()
1098 		__ip_rt_update_pmtu(rt, &fl4, mtu);  in ipv4_update_pmtu()
1099 		ip_rt_put(rt);  in ipv4_update_pmtu()
1108 	struct rtable *rt;  in __ipv4_sk_update_pmtu()  local
1115 	rt = __ip_route_output_key(sock_net(sk), &fl4);  in __ipv4_sk_update_pmtu()
1116 	if (!IS_ERR(rt)) {  in __ipv4_sk_update_pmtu()
1117 		__ip_rt_update_pmtu(rt, &fl4, mtu);  in __ipv4_sk_update_pmtu()
1118 		ip_rt_put(rt);  in __ipv4_sk_update_pmtu()
1126 	struct rtable *rt;  in ipv4_sk_update_pmtu()  local
1145 	rt = dst_rtable(odst);  in ipv4_sk_update_pmtu()
1147 		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);  in ipv4_sk_update_pmtu()
1148 		if (IS_ERR(rt))  in ipv4_sk_update_pmtu()
1154 	__ip_rt_update_pmtu(dst_rtable(xfrm_dst_path(&rt->dst)), &fl4, mtu);  in ipv4_sk_update_pmtu()
1156 	if (!dst_check(&rt->dst, 0)) {  in ipv4_sk_update_pmtu()
1158 			dst_release(&rt->dst);  in ipv4_sk_update_pmtu()
1160 		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);  in ipv4_sk_update_pmtu()
1161 		if (IS_ERR(rt))  in ipv4_sk_update_pmtu()
1168 		sk_dst_set(sk, &rt->dst);  in ipv4_sk_update_pmtu()
1181 	struct rtable *rt;  in ipv4_redirect()  local
1184 	rt = __ip_route_output_key(net, &fl4);  in ipv4_redirect()
1185 	if (!IS_ERR(rt)) {  in ipv4_redirect()
1186 		__ip_do_redirect(rt, skb, &fl4, false);  in ipv4_redirect()
1187 		ip_rt_put(rt);  in ipv4_redirect()
1196 	struct rtable *rt;  in ipv4_sk_redirect()  local
1200 	rt = __ip_route_output_key(net, &fl4);  in ipv4_sk_redirect()
1201 	if (!IS_ERR(rt)) {  in ipv4_sk_redirect()
1202 		__ip_do_redirect(rt, skb, &fl4, false);  in ipv4_sk_redirect()
1203 		ip_rt_put(rt);  in ipv4_sk_redirect()
1211 	struct rtable *rt = dst_rtable(dst);  in ipv4_dst_check()  local
1222 	    rt_is_expired(rt))  in ipv4_dst_check()
1260 	struct rtable *rt;  in ipv4_link_failure()  local
1264 	rt = skb_rtable(skb);  in ipv4_link_failure()
1265 	if (rt)  in ipv4_link_failure()
1266 		dst_set_expires(&rt->dst, 0);  in ipv4_link_failure()
1288 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)  in ip_rt_get_source()  argument
1292 	if (rt_is_output_route(rt))  in ip_rt_get_source()
1301 			.flowi4_oif = rt->dst.dev->ifindex,  in ip_rt_get_source()
1307 		if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)  in ip_rt_get_source()
1308 			src = fib_result_prefsrc(dev_net(rt->dst.dev), &res);  in ip_rt_get_source()
1310 			src = inet_select_addr(rt->dst.dev,  in ip_rt_get_source()
1311 					       rt_nexthop(rt, iph->daddr),  in ip_rt_get_source()
1319 static void set_class_tag(struct rtable *rt, u32 tag)  in set_class_tag()  argument
1321 	if (!(rt->dst.tclassid & 0xFFFF))  in set_class_tag()
1322 		rt->dst.tclassid |= tag & 0xFFFF;  in set_class_tag()
1323 	if (!(rt->dst.tclassid & 0xFFFF0000))  in set_class_tag()
1324 		rt->dst.tclassid |= tag & 0xFFFF0000;  in set_class_tag()
1440 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,  in rt_bind_exception()  argument
1450 		int genid = fnhe_genid(dev_net(rt->dst.dev));  in rt_bind_exception()
1452 		if (rt_is_input_route(rt))  in rt_bind_exception()
1467 		fill_route_from_fnhe(rt, fnhe);  in rt_bind_exception()
1468 		if (!rt->rt_gw4) {  in rt_bind_exception()
1469 			rt->rt_gw4 = daddr;  in rt_bind_exception()
1470 			rt->rt_gw_family = AF_INET;  in rt_bind_exception()
1474 			dst_hold(&rt->dst);  in rt_bind_exception()
1475 			rcu_assign_pointer(*porig, rt);  in rt_bind_exception()
1490 static bool rt_cache_route(struct fib_nh_common *nhc, struct rtable *rt)  in rt_cache_route()  argument
1495 	if (rt_is_input_route(rt)) {  in rt_cache_route()
1505 	dst_hold(&rt->dst);  in rt_cache_route()
1506 	prev = cmpxchg(p, orig, rt);  in rt_cache_route()
1513 		dst_release(&rt->dst);  in rt_cache_route()
1527 void rt_add_uncached_list(struct rtable *rt)  in rt_add_uncached_list()  argument
1531 	rt->dst.rt_uncached_list = ul;  in rt_add_uncached_list()
1534 	list_add_tail(&rt->dst.rt_uncached, &ul->head);  in rt_add_uncached_list()
1538 void rt_del_uncached_list(struct rtable *rt)  in rt_del_uncached_list()  argument
1540 	struct uncached_list *ul = rt->dst.rt_uncached_list;  in rt_del_uncached_list()
1544 		list_del_init(&rt->dst.rt_uncached);  in rt_del_uncached_list()
1557 	struct rtable *rt, *safe;  in rt_flush_dev()  local
1567 		list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) {  in rt_flush_dev()
1568 			if (rt->dst.dev != dev)  in rt_flush_dev()
1570 			rt->dst.dev = blackhole_netdev;  in rt_flush_dev()
1572 					   &rt->dst.dev_tracker, GFP_ATOMIC);  in rt_flush_dev()
1573 			list_del_init(&rt->dst.rt_uncached);  in rt_flush_dev()
1579 static bool rt_cache_valid(const struct rtable *rt)  in rt_cache_valid()  argument
1581 	return	rt &&  in rt_cache_valid()
1582 		READ_ONCE(rt->dst.obsolete) == DST_OBSOLETE_FORCE_CHK &&  in rt_cache_valid()
1583 		!rt_is_expired(rt);  in rt_cache_valid()
1586 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,  in rt_set_nexthop()  argument
1598 			rt->rt_uses_gateway = 1;  in rt_set_nexthop()
1599 			rt->rt_gw_family = nhc->nhc_gw_family;  in rt_set_nexthop()
1602 				rt->rt_gw4 = nhc->nhc_gw.ipv4;  in rt_set_nexthop()
1604 				rt->rt_gw6 = nhc->nhc_gw.ipv6;  in rt_set_nexthop()
1607 		ip_dst_init_metrics(&rt->dst, fi->fib_metrics);  in rt_set_nexthop()
1614 			rt->dst.tclassid = nh->nh_tclassid;  in rt_set_nexthop()
1617 		rt->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate);  in rt_set_nexthop()
1619 			cached = rt_bind_exception(rt, fnhe, daddr, do_cache);  in rt_set_nexthop()
1621 			cached = rt_cache_route(nhc, rt);  in rt_set_nexthop()
1628 			if (!rt->rt_gw4) {  in rt_set_nexthop()
1629 				rt->rt_gw_family = AF_INET;  in rt_set_nexthop()
1630 				rt->rt_gw4 = daddr;  in rt_set_nexthop()
1632 			rt_add_uncached_list(rt);  in rt_set_nexthop()
1635 		rt_add_uncached_list(rt);  in rt_set_nexthop()
1639 	set_class_tag(rt, res->tclassid);  in rt_set_nexthop()
1641 	set_class_tag(rt, itag);  in rt_set_nexthop()
1649 	struct rtable *rt;  in rt_dst_alloc()  local
1651 	rt = dst_alloc(&ipv4_dst_ops, dev, DST_OBSOLETE_FORCE_CHK,  in rt_dst_alloc()
1654 	if (rt) {  in rt_dst_alloc()
1655 		rt->rt_genid = rt_genid_ipv4(dev_net(dev));  in rt_dst_alloc()
1656 		rt->rt_flags = flags;  in rt_dst_alloc()
1657 		rt->rt_type = type;  in rt_dst_alloc()
1658 		rt->rt_is_input = 0;  in rt_dst_alloc()
1659 		rt->rt_iif = 0;  in rt_dst_alloc()
1660 		rt->rt_pmtu = 0;  in rt_dst_alloc()
1661 		rt->rt_mtu_locked = 0;  in rt_dst_alloc()
1662 		rt->rt_uses_gateway = 0;  in rt_dst_alloc()
1663 		rt->rt_gw_family = 0;  in rt_dst_alloc()
1664 		rt->rt_gw4 = 0;  in rt_dst_alloc()
1666 		rt->dst.output = ip_output;  in rt_dst_alloc()
1668 			rt->dst.input = ip_local_deliver;  in rt_dst_alloc()
1671 	return rt;  in rt_dst_alloc()
1675 struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt)  in rt_dst_clone()  argument
1680 			   rt->dst.flags);  in rt_dst_clone()
1684 		new_rt->rt_flags = rt->rt_flags;  in rt_dst_clone()
1685 		new_rt->rt_type = rt->rt_type;  in rt_dst_clone()
1686 		new_rt->rt_is_input = rt->rt_is_input;  in rt_dst_clone()
1687 		new_rt->rt_iif = rt->rt_iif;  in rt_dst_clone()
1688 		new_rt->rt_pmtu = rt->rt_pmtu;  in rt_dst_clone()
1689 		new_rt->rt_mtu_locked = rt->rt_mtu_locked;  in rt_dst_clone()
1690 		new_rt->rt_gw_family = rt->rt_gw_family;  in rt_dst_clone()
1691 		if (rt->rt_gw_family == AF_INET)  in rt_dst_clone()
1692 			new_rt->rt_gw4 = rt->rt_gw4;  in rt_dst_clone()
1693 		else if (rt->rt_gw_family == AF_INET6)  in rt_dst_clone()
1694 			new_rt->rt_gw6 = rt->rt_gw6;  in rt_dst_clone()
1696 		new_rt->dst.input = READ_ONCE(rt->dst.input);  in rt_dst_clone()
1697 		new_rt->dst.output = READ_ONCE(rt->dst.output);  in rt_dst_clone()
1698 		new_rt->dst.error = rt->dst.error;  in rt_dst_clone()
1700 		new_rt->dst.lwtstate = lwtstate_get(rt->dst.lwtstate);  in rt_dst_clone()
2197 	struct rtable *rt = skb_rtable(hint);  in ip_route_use_hint()  local
2219 	if (!(rt->rt_flags & RTCF_LOCAL))  in ip_route_use_hint()
2895 	struct rtable *rt;  in ipv4_blackhole_route()  local
2897 	rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, DST_OBSOLETE_DEAD, 0);  in ipv4_blackhole_route()
2898 	if (rt) {  in ipv4_blackhole_route()
2899 		struct dst_entry *new = &rt->dst;  in ipv4_blackhole_route()
2908 		rt->rt_is_input = ort->rt_is_input;  in ipv4_blackhole_route()
2909 		rt->rt_iif = ort->rt_iif;  in ipv4_blackhole_route()
2910 		rt->rt_pmtu = ort->rt_pmtu;  in ipv4_blackhole_route()
2911 		rt->rt_mtu_locked = ort->rt_mtu_locked;  in ipv4_blackhole_route()
2913 		rt->rt_genid = rt_genid_ipv4(net);  in ipv4_blackhole_route()
2914 		rt->rt_flags = ort->rt_flags;  in ipv4_blackhole_route()
2915 		rt->rt_type = ort->rt_type;  in ipv4_blackhole_route()
2916 		rt->rt_uses_gateway = ort->rt_uses_gateway;  in ipv4_blackhole_route()
2917 		rt->rt_gw_family = ort->rt_gw_family;  in ipv4_blackhole_route()
2918 		if (rt->rt_gw_family == AF_INET)  in ipv4_blackhole_route()
2919 			rt->rt_gw4 = ort->rt_gw4;  in ipv4_blackhole_route()
2920 		else if (rt->rt_gw_family == AF_INET6)  in ipv4_blackhole_route()
2921 			rt->rt_gw6 = ort->rt_gw6;  in ipv4_blackhole_route()
2926 	return rt ? &rt->dst : ERR_PTR(-ENOMEM);  in ipv4_blackhole_route()
2932 	struct rtable *rt = __ip_route_output_key(net, flp4);  in ip_route_output_flow()  local
2934 	if (IS_ERR(rt))  in ip_route_output_flow()
2935 		return rt;  in ip_route_output_flow()
2938 		flp4->flowi4_oif = rt->dst.dev->ifindex;  in ip_route_output_flow()
2939 		rt = dst_rtable(xfrm_lookup_route(net, &rt->dst,  in ip_route_output_flow()
2944 	return rt;  in ip_route_output_flow()
2950 			struct rtable *rt, u32 table_id, dscp_t dscp,  in rt_fill_info()  argument
2972 	r->rtm_type	= rt->rt_type;  in rt_fill_info()
2975 	r->rtm_flags	= (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;  in rt_fill_info()
2976 	if (rt->rt_flags & RTCF_NOTIFY)  in rt_fill_info()
2988 	if (rt->dst.dev &&  in rt_fill_info()
2989 	    nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))  in rt_fill_info()
2991 	if (lwtunnel_fill_encap(skb, rt->dst.lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0)  in rt_fill_info()
2994 	if (rt->dst.tclassid &&  in rt_fill_info()
2995 	    nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))  in rt_fill_info()
2998 	if (fl4 && !rt_is_input_route(rt) &&  in rt_fill_info()
3003 	if (rt->rt_uses_gateway) {  in rt_fill_info()
3004 		if (rt->rt_gw_family == AF_INET &&  in rt_fill_info()
3005 		    nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gw4)) {  in rt_fill_info()
3007 		} else if (rt->rt_gw_family == AF_INET6) {  in rt_fill_info()
3018 			memcpy(via->rtvia_addr, &rt->rt_gw6, alen);  in rt_fill_info()
3022 	expires = READ_ONCE(rt->dst.expires);  in rt_fill_info()
3032 	memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));  in rt_fill_info()
3033 	if (rt->rt_pmtu && expires)  in rt_fill_info()
3034 		metrics[RTAX_MTU - 1] = rt->rt_pmtu;  in rt_fill_info()
3035 	if (rt->rt_mtu_locked && expires)  in rt_fill_info()
3051 		if (rt_is_input_route(rt)) {  in rt_fill_info()
3072 	error = rt->dst.error;  in rt_fill_info()
3074 	if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)  in rt_fill_info()
3097 			struct rtable *rt;  in fnhe_dump_bucket()  local
3110 			rt = rcu_dereference(fnhe->fnhe_rth_input);  in fnhe_dump_bucket()
3111 			if (!rt)  in fnhe_dump_bucket()
3112 				rt = rcu_dereference(fnhe->fnhe_rth_output);  in fnhe_dump_bucket()
3113 			if (!rt)  in fnhe_dump_bucket()
3116 			err = rt_fill_info(net, fnhe->fnhe_daddr, 0, rt,  in fnhe_dump_bucket()
3299 	struct rtable *rt = NULL;  in inet_rtm_getroute()  local
3372 		rt = skb_rtable(skb);  in inet_rtm_getroute()
3373 		if (err == 0 && rt->dst.error)  in inet_rtm_getroute()
3374 			err = -rt->dst.error;  in inet_rtm_getroute()
3378 		rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);  in inet_rtm_getroute()
3380 		if (IS_ERR(rt))  in inet_rtm_getroute()
3381 			err = PTR_ERR(rt);  in inet_rtm_getroute()
3383 			skb_dst_set(skb, &rt->dst);  in inet_rtm_getroute()
3390 		rt->rt_flags |= RTCF_NOTIFY;  in inet_rtm_getroute()
3415 		fri.type = rt->rt_type;  in inet_rtm_getroute()
3441 		err = rt_fill_info(net, dst, src, rt, table_id, res.dscp, &fl4,  in inet_rtm_getroute()