1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux INET6 implementation 4 * FIB front-end. 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 */ 9 10 /* Changes: 11 * 12 * YOSHIFUJI Hideaki @USAGI 13 * reworked default router selection. 14 * - respect outgoing interface 15 * - select from (probably) reachable routers (i.e. 16 * routers in REACHABLE, STALE, DELAY or PROBE states). 17 * - always select the same router if it is (probably) 18 * reachable. otherwise, round-robin the list. 19 * Ville Nuorvala 20 * Fixed routing subtrees. 21 */ 22 23 #define pr_fmt(fmt) "IPv6: " fmt 24 25 #include <linux/capability.h> 26 #include <linux/errno.h> 27 #include <linux/export.h> 28 #include <linux/types.h> 29 #include <linux/times.h> 30 #include <linux/socket.h> 31 #include <linux/sockios.h> 32 #include <linux/net.h> 33 #include <linux/route.h> 34 #include <linux/netdevice.h> 35 #include <linux/in6.h> 36 #include <linux/mroute6.h> 37 #include <linux/init.h> 38 #include <linux/if_arp.h> 39 #include <linux/proc_fs.h> 40 #include <linux/seq_file.h> 41 #include <linux/nsproxy.h> 42 #include <linux/slab.h> 43 #include <linux/jhash.h> 44 #include <net/net_namespace.h> 45 #include <net/snmp.h> 46 #include <net/ipv6.h> 47 #include <net/ip6_fib.h> 48 #include <net/ip6_route.h> 49 #include <net/ndisc.h> 50 #include <net/addrconf.h> 51 #include <net/tcp.h> 52 #include <linux/rtnetlink.h> 53 #include <net/dst.h> 54 #include <net/dst_metadata.h> 55 #include <net/xfrm.h> 56 #include <net/netevent.h> 57 #include <net/netlink.h> 58 #include <net/rtnh.h> 59 #include <net/lwtunnel.h> 60 #include <net/ip_tunnels.h> 61 #include <net/l3mdev.h> 62 #include <net/ip.h> 63 #include <linux/uaccess.h> 64 #include <linux/btf_ids.h> 65 66 #ifdef CONFIG_SYSCTL 67 #include <linux/sysctl.h> 68 #endif 69 70 static int ip6_rt_type_to_error(u8 fib6_type); 71 72 #define CREATE_TRACE_POINTS 73 #include <trace/events/fib6.h> 74 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup); 75 #undef CREATE_TRACE_POINTS 76 77 enum rt6_nud_state { 78 RT6_NUD_FAIL_HARD = -3, 79 RT6_NUD_FAIL_PROBE = -2, 80 RT6_NUD_FAIL_DO_RR = -1, 81 RT6_NUD_SUCCEED = 1 82 }; 83 84 INDIRECT_CALLABLE_SCOPE 85 struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie); 86 static unsigned int ip6_default_advmss(const struct dst_entry *dst); 87 INDIRECT_CALLABLE_SCOPE 88 unsigned int ip6_mtu(const struct dst_entry *dst); 89 static struct dst_entry *ip6_negative_advice(struct dst_entry *); 90 static void ip6_dst_destroy(struct dst_entry *); 91 static void ip6_dst_ifdown(struct dst_entry *, 92 struct net_device *dev, int how); 93 static int ip6_dst_gc(struct dst_ops *ops); 94 95 static int ip6_pkt_discard(struct sk_buff *skb); 96 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb); 97 static int ip6_pkt_prohibit(struct sk_buff *skb); 98 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb); 99 static void ip6_link_failure(struct sk_buff *skb); 100 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 101 struct sk_buff *skb, u32 mtu, 102 bool confirm_neigh); 103 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, 104 struct sk_buff *skb); 105 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 106 int strict); 107 static size_t rt6_nlmsg_size(struct fib6_info *f6i); 108 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 109 struct fib6_info *rt, struct dst_entry *dst, 110 struct in6_addr *dest, struct in6_addr *src, 111 int iif, int type, u32 portid, u32 seq, 112 unsigned int flags); 113 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 114 const struct in6_addr *daddr, 115 const struct in6_addr *saddr); 116 117 #ifdef CONFIG_IPV6_ROUTE_INFO 118 static struct fib6_info *rt6_add_route_info(struct net *net, 119 const struct in6_addr *prefix, int prefixlen, 120 const struct in6_addr *gwaddr, 121 struct net_device *dev, 122 unsigned int pref); 123 static struct fib6_info *rt6_get_route_info(struct net *net, 124 const struct in6_addr *prefix, int prefixlen, 125 const struct in6_addr *gwaddr, 126 struct net_device *dev); 127 #endif 128 129 struct uncached_list { 130 spinlock_t lock; 131 struct list_head head; 132 }; 133 134 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); 135 136 void rt6_uncached_list_add(struct rt6_info *rt) 137 { 138 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); 139 140 rt->rt6i_uncached_list = ul; 141 142 spin_lock_bh(&ul->lock); 143 list_add_tail(&rt->rt6i_uncached, &ul->head); 144 spin_unlock_bh(&ul->lock); 145 } 146 147 void rt6_uncached_list_del(struct rt6_info *rt) 148 { 149 if (!list_empty(&rt->rt6i_uncached)) { 150 struct uncached_list *ul = rt->rt6i_uncached_list; 151 struct net *net = dev_net(rt->dst.dev); 152 153 spin_lock_bh(&ul->lock); 154 list_del(&rt->rt6i_uncached); 155 atomic_dec(&net->ipv6.rt6_stats->fib_rt_uncache); 156 spin_unlock_bh(&ul->lock); 157 } 158 } 159 160 static void rt6_uncached_list_flush_dev(struct net *net, struct net_device *dev) 161 { 162 struct net_device *loopback_dev = net->loopback_dev; 163 int cpu; 164 165 if (dev == loopback_dev) 166 return; 167 168 for_each_possible_cpu(cpu) { 169 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 170 struct rt6_info *rt; 171 172 spin_lock_bh(&ul->lock); 173 list_for_each_entry(rt, &ul->head, rt6i_uncached) { 174 struct inet6_dev *rt_idev = rt->rt6i_idev; 175 struct net_device *rt_dev = rt->dst.dev; 176 177 if (rt_idev->dev == dev) { 178 rt->rt6i_idev = in6_dev_get(loopback_dev); 179 in6_dev_put(rt_idev); 180 } 181 182 if (rt_dev == dev) { 183 rt->dst.dev = blackhole_netdev; 184 dev_hold(rt->dst.dev); 185 dev_put(rt_dev); 186 } 187 } 188 spin_unlock_bh(&ul->lock); 189 } 190 } 191 192 static inline const void *choose_neigh_daddr(const struct in6_addr *p, 193 struct sk_buff *skb, 194 const void *daddr) 195 { 196 if (!ipv6_addr_any(p)) 197 return (const void *) p; 198 else if (skb) 199 return &ipv6_hdr(skb)->daddr; 200 return daddr; 201 } 202 203 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw, 204 struct net_device *dev, 205 struct sk_buff *skb, 206 const void *daddr) 207 { 208 struct neighbour *n; 209 210 daddr = choose_neigh_daddr(gw, skb, daddr); 211 n = __ipv6_neigh_lookup(dev, daddr); 212 if (n) 213 return n; 214 215 n = neigh_create(&nd_tbl, daddr, dev); 216 return IS_ERR(n) ? NULL : n; 217 } 218 219 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst, 220 struct sk_buff *skb, 221 const void *daddr) 222 { 223 const struct rt6_info *rt = container_of(dst, struct rt6_info, dst); 224 225 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any), 226 dst->dev, skb, daddr); 227 } 228 229 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr) 230 { 231 struct net_device *dev = dst->dev; 232 struct rt6_info *rt = (struct rt6_info *)dst; 233 234 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr); 235 if (!daddr) 236 return; 237 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) 238 return; 239 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr)) 240 return; 241 __ipv6_confirm_neigh(dev, daddr); 242 } 243 244 static struct dst_ops ip6_dst_ops_template = { 245 .family = AF_INET6, 246 .gc = ip6_dst_gc, 247 .gc_thresh = 1024, 248 .check = ip6_dst_check, 249 .default_advmss = ip6_default_advmss, 250 .mtu = ip6_mtu, 251 .cow_metrics = dst_cow_metrics_generic, 252 .destroy = ip6_dst_destroy, 253 .ifdown = ip6_dst_ifdown, 254 .negative_advice = ip6_negative_advice, 255 .link_failure = ip6_link_failure, 256 .update_pmtu = ip6_rt_update_pmtu, 257 .redirect = rt6_do_redirect, 258 .local_out = __ip6_local_out, 259 .neigh_lookup = ip6_dst_neigh_lookup, 260 .confirm_neigh = ip6_confirm_neigh, 261 }; 262 263 static struct dst_ops ip6_dst_blackhole_ops = { 264 .family = AF_INET6, 265 .default_advmss = ip6_default_advmss, 266 .neigh_lookup = ip6_dst_neigh_lookup, 267 .check = ip6_dst_check, 268 .destroy = ip6_dst_destroy, 269 .cow_metrics = dst_cow_metrics_generic, 270 .update_pmtu = dst_blackhole_update_pmtu, 271 .redirect = dst_blackhole_redirect, 272 .mtu = dst_blackhole_mtu, 273 }; 274 275 static const u32 ip6_template_metrics[RTAX_MAX] = { 276 [RTAX_HOPLIMIT - 1] = 0, 277 }; 278 279 static const struct fib6_info fib6_null_entry_template = { 280 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP), 281 .fib6_protocol = RTPROT_KERNEL, 282 .fib6_metric = ~(u32)0, 283 .fib6_ref = REFCOUNT_INIT(1), 284 .fib6_type = RTN_UNREACHABLE, 285 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics, 286 }; 287 288 static const struct rt6_info ip6_null_entry_template = { 289 .dst = { 290 .__refcnt = ATOMIC_INIT(1), 291 .__use = 1, 292 .obsolete = DST_OBSOLETE_FORCE_CHK, 293 .error = -ENETUNREACH, 294 .input = ip6_pkt_discard, 295 .output = ip6_pkt_discard_out, 296 }, 297 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 298 }; 299 300 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 301 302 static const struct rt6_info ip6_prohibit_entry_template = { 303 .dst = { 304 .__refcnt = ATOMIC_INIT(1), 305 .__use = 1, 306 .obsolete = DST_OBSOLETE_FORCE_CHK, 307 .error = -EACCES, 308 .input = ip6_pkt_prohibit, 309 .output = ip6_pkt_prohibit_out, 310 }, 311 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 312 }; 313 314 static const struct rt6_info ip6_blk_hole_entry_template = { 315 .dst = { 316 .__refcnt = ATOMIC_INIT(1), 317 .__use = 1, 318 .obsolete = DST_OBSOLETE_FORCE_CHK, 319 .error = -EINVAL, 320 .input = dst_discard, 321 .output = dst_discard_out, 322 }, 323 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 324 }; 325 326 #endif 327 328 static void rt6_info_init(struct rt6_info *rt) 329 { 330 struct dst_entry *dst = &rt->dst; 331 332 memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst)); 333 INIT_LIST_HEAD(&rt->rt6i_uncached); 334 } 335 336 /* allocate dst with ip6_dst_ops */ 337 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, 338 int flags) 339 { 340 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev, 341 1, DST_OBSOLETE_FORCE_CHK, flags); 342 343 if (rt) { 344 rt6_info_init(rt); 345 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 346 } 347 348 return rt; 349 } 350 EXPORT_SYMBOL(ip6_dst_alloc); 351 352 static void ip6_dst_destroy(struct dst_entry *dst) 353 { 354 struct rt6_info *rt = (struct rt6_info *)dst; 355 struct fib6_info *from; 356 struct inet6_dev *idev; 357 358 ip_dst_metrics_put(dst); 359 rt6_uncached_list_del(rt); 360 361 idev = rt->rt6i_idev; 362 if (idev) { 363 rt->rt6i_idev = NULL; 364 in6_dev_put(idev); 365 } 366 367 from = xchg((__force struct fib6_info **)&rt->from, NULL); 368 fib6_info_release(from); 369 } 370 371 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev, 372 int how) 373 { 374 struct rt6_info *rt = (struct rt6_info *)dst; 375 struct inet6_dev *idev = rt->rt6i_idev; 376 struct net_device *loopback_dev = 377 dev_net(dev)->loopback_dev; 378 379 if (idev && idev->dev != loopback_dev) { 380 struct inet6_dev *loopback_idev = in6_dev_get(loopback_dev); 381 if (loopback_idev) { 382 rt->rt6i_idev = loopback_idev; 383 in6_dev_put(idev); 384 } 385 } 386 } 387 388 static bool __rt6_check_expired(const struct rt6_info *rt) 389 { 390 if (rt->rt6i_flags & RTF_EXPIRES) 391 return time_after(jiffies, rt->dst.expires); 392 else 393 return false; 394 } 395 396 static bool rt6_check_expired(const struct rt6_info *rt) 397 { 398 struct fib6_info *from; 399 400 from = rcu_dereference(rt->from); 401 402 if (rt->rt6i_flags & RTF_EXPIRES) { 403 if (time_after(jiffies, rt->dst.expires)) 404 return true; 405 } else if (from) { 406 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK || 407 fib6_check_expired(from); 408 } 409 return false; 410 } 411 412 void fib6_select_path(const struct net *net, struct fib6_result *res, 413 struct flowi6 *fl6, int oif, bool have_oif_match, 414 const struct sk_buff *skb, int strict) 415 { 416 struct fib6_info *sibling, *next_sibling; 417 struct fib6_info *match = res->f6i; 418 419 if (!match->nh && (!match->fib6_nsiblings || have_oif_match)) 420 goto out; 421 422 if (match->nh && have_oif_match && res->nh) 423 return; 424 425 /* We might have already computed the hash for ICMPv6 errors. In such 426 * case it will always be non-zero. Otherwise now is the time to do it. 427 */ 428 if (!fl6->mp_hash && 429 (!match->nh || nexthop_is_multipath(match->nh))) 430 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL); 431 432 if (unlikely(match->nh)) { 433 nexthop_path_fib6_result(res, fl6->mp_hash); 434 return; 435 } 436 437 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound)) 438 goto out; 439 440 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings, 441 fib6_siblings) { 442 const struct fib6_nh *nh = sibling->fib6_nh; 443 int nh_upper_bound; 444 445 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound); 446 if (fl6->mp_hash > nh_upper_bound) 447 continue; 448 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0) 449 break; 450 match = sibling; 451 break; 452 } 453 454 out: 455 res->f6i = match; 456 res->nh = match->fib6_nh; 457 } 458 459 /* 460 * Route lookup. rcu_read_lock() should be held. 461 */ 462 463 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh, 464 const struct in6_addr *saddr, int oif, int flags) 465 { 466 const struct net_device *dev; 467 468 if (nh->fib_nh_flags & RTNH_F_DEAD) 469 return false; 470 471 dev = nh->fib_nh_dev; 472 if (oif) { 473 if (dev->ifindex == oif) 474 return true; 475 } else { 476 if (ipv6_chk_addr(net, saddr, dev, 477 flags & RT6_LOOKUP_F_IFACE)) 478 return true; 479 } 480 481 return false; 482 } 483 484 struct fib6_nh_dm_arg { 485 struct net *net; 486 const struct in6_addr *saddr; 487 int oif; 488 int flags; 489 struct fib6_nh *nh; 490 }; 491 492 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg) 493 { 494 struct fib6_nh_dm_arg *arg = _arg; 495 496 arg->nh = nh; 497 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif, 498 arg->flags); 499 } 500 501 /* returns fib6_nh from nexthop or NULL */ 502 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh, 503 struct fib6_result *res, 504 const struct in6_addr *saddr, 505 int oif, int flags) 506 { 507 struct fib6_nh_dm_arg arg = { 508 .net = net, 509 .saddr = saddr, 510 .oif = oif, 511 .flags = flags, 512 }; 513 514 if (nexthop_is_blackhole(nh)) 515 return NULL; 516 517 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg)) 518 return arg.nh; 519 520 return NULL; 521 } 522 523 static void rt6_device_match(struct net *net, struct fib6_result *res, 524 const struct in6_addr *saddr, int oif, int flags) 525 { 526 struct fib6_info *f6i = res->f6i; 527 struct fib6_info *spf6i; 528 struct fib6_nh *nh; 529 530 if (!oif && ipv6_addr_any(saddr)) { 531 if (unlikely(f6i->nh)) { 532 nh = nexthop_fib6_nh(f6i->nh); 533 if (nexthop_is_blackhole(f6i->nh)) 534 goto out_blackhole; 535 } else { 536 nh = f6i->fib6_nh; 537 } 538 if (!(nh->fib_nh_flags & RTNH_F_DEAD)) 539 goto out; 540 } 541 542 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) { 543 bool matched = false; 544 545 if (unlikely(spf6i->nh)) { 546 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr, 547 oif, flags); 548 if (nh) 549 matched = true; 550 } else { 551 nh = spf6i->fib6_nh; 552 if (__rt6_device_match(net, nh, saddr, oif, flags)) 553 matched = true; 554 } 555 if (matched) { 556 res->f6i = spf6i; 557 goto out; 558 } 559 } 560 561 if (oif && flags & RT6_LOOKUP_F_IFACE) { 562 res->f6i = net->ipv6.fib6_null_entry; 563 nh = res->f6i->fib6_nh; 564 goto out; 565 } 566 567 if (unlikely(f6i->nh)) { 568 nh = nexthop_fib6_nh(f6i->nh); 569 if (nexthop_is_blackhole(f6i->nh)) 570 goto out_blackhole; 571 } else { 572 nh = f6i->fib6_nh; 573 } 574 575 if (nh->fib_nh_flags & RTNH_F_DEAD) { 576 res->f6i = net->ipv6.fib6_null_entry; 577 nh = res->f6i->fib6_nh; 578 } 579 out: 580 res->nh = nh; 581 res->fib6_type = res->f6i->fib6_type; 582 res->fib6_flags = res->f6i->fib6_flags; 583 return; 584 585 out_blackhole: 586 res->fib6_flags |= RTF_REJECT; 587 res->fib6_type = RTN_BLACKHOLE; 588 res->nh = nh; 589 } 590 591 #ifdef CONFIG_IPV6_ROUTER_PREF 592 struct __rt6_probe_work { 593 struct work_struct work; 594 struct in6_addr target; 595 struct net_device *dev; 596 }; 597 598 static void rt6_probe_deferred(struct work_struct *w) 599 { 600 struct in6_addr mcaddr; 601 struct __rt6_probe_work *work = 602 container_of(w, struct __rt6_probe_work, work); 603 604 addrconf_addr_solict_mult(&work->target, &mcaddr); 605 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); 606 dev_put(work->dev); 607 kfree(work); 608 } 609 610 static void rt6_probe(struct fib6_nh *fib6_nh) 611 { 612 struct __rt6_probe_work *work = NULL; 613 const struct in6_addr *nh_gw; 614 unsigned long last_probe; 615 struct neighbour *neigh; 616 struct net_device *dev; 617 struct inet6_dev *idev; 618 619 /* 620 * Okay, this does not seem to be appropriate 621 * for now, however, we need to check if it 622 * is really so; aka Router Reachability Probing. 623 * 624 * Router Reachability Probe MUST be rate-limited 625 * to no more than one per minute. 626 */ 627 if (!fib6_nh->fib_nh_gw_family) 628 return; 629 630 nh_gw = &fib6_nh->fib_nh_gw6; 631 dev = fib6_nh->fib_nh_dev; 632 rcu_read_lock_bh(); 633 last_probe = READ_ONCE(fib6_nh->last_probe); 634 idev = __in6_dev_get(dev); 635 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); 636 if (neigh) { 637 if (neigh->nud_state & NUD_VALID) 638 goto out; 639 640 write_lock(&neigh->lock); 641 if (!(neigh->nud_state & NUD_VALID) && 642 time_after(jiffies, 643 neigh->updated + idev->cnf.rtr_probe_interval)) { 644 work = kmalloc(sizeof(*work), GFP_ATOMIC); 645 if (work) 646 __neigh_set_probe_once(neigh); 647 } 648 write_unlock(&neigh->lock); 649 } else if (time_after(jiffies, last_probe + 650 idev->cnf.rtr_probe_interval)) { 651 work = kmalloc(sizeof(*work), GFP_ATOMIC); 652 } 653 654 if (!work || cmpxchg(&fib6_nh->last_probe, 655 last_probe, jiffies) != last_probe) { 656 kfree(work); 657 } else { 658 INIT_WORK(&work->work, rt6_probe_deferred); 659 work->target = *nh_gw; 660 dev_hold(dev); 661 work->dev = dev; 662 schedule_work(&work->work); 663 } 664 665 out: 666 rcu_read_unlock_bh(); 667 } 668 #else 669 static inline void rt6_probe(struct fib6_nh *fib6_nh) 670 { 671 } 672 #endif 673 674 /* 675 * Default Router Selection (RFC 2461 6.3.6) 676 */ 677 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh) 678 { 679 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD; 680 struct neighbour *neigh; 681 682 rcu_read_lock_bh(); 683 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev, 684 &fib6_nh->fib_nh_gw6); 685 if (neigh) { 686 read_lock(&neigh->lock); 687 if (neigh->nud_state & NUD_VALID) 688 ret = RT6_NUD_SUCCEED; 689 #ifdef CONFIG_IPV6_ROUTER_PREF 690 else if (!(neigh->nud_state & NUD_FAILED)) 691 ret = RT6_NUD_SUCCEED; 692 else 693 ret = RT6_NUD_FAIL_PROBE; 694 #endif 695 read_unlock(&neigh->lock); 696 } else { 697 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ? 698 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR; 699 } 700 rcu_read_unlock_bh(); 701 702 return ret; 703 } 704 705 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 706 int strict) 707 { 708 int m = 0; 709 710 if (!oif || nh->fib_nh_dev->ifindex == oif) 711 m = 2; 712 713 if (!m && (strict & RT6_LOOKUP_F_IFACE)) 714 return RT6_NUD_FAIL_HARD; 715 #ifdef CONFIG_IPV6_ROUTER_PREF 716 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2; 717 #endif 718 if ((strict & RT6_LOOKUP_F_REACHABLE) && 719 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) { 720 int n = rt6_check_neigh(nh); 721 if (n < 0) 722 return n; 723 } 724 return m; 725 } 726 727 static bool find_match(struct fib6_nh *nh, u32 fib6_flags, 728 int oif, int strict, int *mpri, bool *do_rr) 729 { 730 bool match_do_rr = false; 731 bool rc = false; 732 int m; 733 734 if (nh->fib_nh_flags & RTNH_F_DEAD) 735 goto out; 736 737 if (ip6_ignore_linkdown(nh->fib_nh_dev) && 738 nh->fib_nh_flags & RTNH_F_LINKDOWN && 739 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE)) 740 goto out; 741 742 m = rt6_score_route(nh, fib6_flags, oif, strict); 743 if (m == RT6_NUD_FAIL_DO_RR) { 744 match_do_rr = true; 745 m = 0; /* lowest valid score */ 746 } else if (m == RT6_NUD_FAIL_HARD) { 747 goto out; 748 } 749 750 if (strict & RT6_LOOKUP_F_REACHABLE) 751 rt6_probe(nh); 752 753 /* note that m can be RT6_NUD_FAIL_PROBE at this point */ 754 if (m > *mpri) { 755 *do_rr = match_do_rr; 756 *mpri = m; 757 rc = true; 758 } 759 out: 760 return rc; 761 } 762 763 struct fib6_nh_frl_arg { 764 u32 flags; 765 int oif; 766 int strict; 767 int *mpri; 768 bool *do_rr; 769 struct fib6_nh *nh; 770 }; 771 772 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg) 773 { 774 struct fib6_nh_frl_arg *arg = _arg; 775 776 arg->nh = nh; 777 return find_match(nh, arg->flags, arg->oif, arg->strict, 778 arg->mpri, arg->do_rr); 779 } 780 781 static void __find_rr_leaf(struct fib6_info *f6i_start, 782 struct fib6_info *nomatch, u32 metric, 783 struct fib6_result *res, struct fib6_info **cont, 784 int oif, int strict, bool *do_rr, int *mpri) 785 { 786 struct fib6_info *f6i; 787 788 for (f6i = f6i_start; 789 f6i && f6i != nomatch; 790 f6i = rcu_dereference(f6i->fib6_next)) { 791 bool matched = false; 792 struct fib6_nh *nh; 793 794 if (cont && f6i->fib6_metric != metric) { 795 *cont = f6i; 796 return; 797 } 798 799 if (fib6_check_expired(f6i)) 800 continue; 801 802 if (unlikely(f6i->nh)) { 803 struct fib6_nh_frl_arg arg = { 804 .flags = f6i->fib6_flags, 805 .oif = oif, 806 .strict = strict, 807 .mpri = mpri, 808 .do_rr = do_rr 809 }; 810 811 if (nexthop_is_blackhole(f6i->nh)) { 812 res->fib6_flags = RTF_REJECT; 813 res->fib6_type = RTN_BLACKHOLE; 814 res->f6i = f6i; 815 res->nh = nexthop_fib6_nh(f6i->nh); 816 return; 817 } 818 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match, 819 &arg)) { 820 matched = true; 821 nh = arg.nh; 822 } 823 } else { 824 nh = f6i->fib6_nh; 825 if (find_match(nh, f6i->fib6_flags, oif, strict, 826 mpri, do_rr)) 827 matched = true; 828 } 829 if (matched) { 830 res->f6i = f6i; 831 res->nh = nh; 832 res->fib6_flags = f6i->fib6_flags; 833 res->fib6_type = f6i->fib6_type; 834 } 835 } 836 } 837 838 static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf, 839 struct fib6_info *rr_head, int oif, int strict, 840 bool *do_rr, struct fib6_result *res) 841 { 842 u32 metric = rr_head->fib6_metric; 843 struct fib6_info *cont = NULL; 844 int mpri = -1; 845 846 __find_rr_leaf(rr_head, NULL, metric, res, &cont, 847 oif, strict, do_rr, &mpri); 848 849 __find_rr_leaf(leaf, rr_head, metric, res, &cont, 850 oif, strict, do_rr, &mpri); 851 852 if (res->f6i || !cont) 853 return; 854 855 __find_rr_leaf(cont, NULL, metric, res, NULL, 856 oif, strict, do_rr, &mpri); 857 } 858 859 static void rt6_select(struct net *net, struct fib6_node *fn, int oif, 860 struct fib6_result *res, int strict) 861 { 862 struct fib6_info *leaf = rcu_dereference(fn->leaf); 863 struct fib6_info *rt0; 864 bool do_rr = false; 865 int key_plen; 866 867 /* make sure this function or its helpers sets f6i */ 868 res->f6i = NULL; 869 870 if (!leaf || leaf == net->ipv6.fib6_null_entry) 871 goto out; 872 873 rt0 = rcu_dereference(fn->rr_ptr); 874 if (!rt0) 875 rt0 = leaf; 876 877 /* Double check to make sure fn is not an intermediate node 878 * and fn->leaf does not points to its child's leaf 879 * (This might happen if all routes under fn are deleted from 880 * the tree and fib6_repair_tree() is called on the node.) 881 */ 882 key_plen = rt0->fib6_dst.plen; 883 #ifdef CONFIG_IPV6_SUBTREES 884 if (rt0->fib6_src.plen) 885 key_plen = rt0->fib6_src.plen; 886 #endif 887 if (fn->fn_bit != key_plen) 888 goto out; 889 890 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res); 891 if (do_rr) { 892 struct fib6_info *next = rcu_dereference(rt0->fib6_next); 893 894 /* no entries matched; do round-robin */ 895 if (!next || next->fib6_metric != rt0->fib6_metric) 896 next = leaf; 897 898 if (next != rt0) { 899 spin_lock_bh(&leaf->fib6_table->tb6_lock); 900 /* make sure next is not being deleted from the tree */ 901 if (next->fib6_node) 902 rcu_assign_pointer(fn->rr_ptr, next); 903 spin_unlock_bh(&leaf->fib6_table->tb6_lock); 904 } 905 } 906 907 out: 908 if (!res->f6i) { 909 res->f6i = net->ipv6.fib6_null_entry; 910 res->nh = res->f6i->fib6_nh; 911 res->fib6_flags = res->f6i->fib6_flags; 912 res->fib6_type = res->f6i->fib6_type; 913 } 914 } 915 916 static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res) 917 { 918 return (res->f6i->fib6_flags & RTF_NONEXTHOP) || 919 res->nh->fib_nh_gw_family; 920 } 921 922 #ifdef CONFIG_IPV6_ROUTE_INFO 923 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, 924 const struct in6_addr *gwaddr) 925 { 926 struct net *net = dev_net(dev); 927 struct route_info *rinfo = (struct route_info *) opt; 928 struct in6_addr prefix_buf, *prefix; 929 unsigned int pref; 930 unsigned long lifetime; 931 struct fib6_info *rt; 932 933 if (len < sizeof(struct route_info)) { 934 return -EINVAL; 935 } 936 937 /* Sanity check for prefix_len and length */ 938 if (rinfo->length > 3) { 939 return -EINVAL; 940 } else if (rinfo->prefix_len > 128) { 941 return -EINVAL; 942 } else if (rinfo->prefix_len > 64) { 943 if (rinfo->length < 2) { 944 return -EINVAL; 945 } 946 } else if (rinfo->prefix_len > 0) { 947 if (rinfo->length < 1) { 948 return -EINVAL; 949 } 950 } 951 952 pref = rinfo->route_pref; 953 if (pref == ICMPV6_ROUTER_PREF_INVALID) 954 return -EINVAL; 955 956 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ); 957 958 if (rinfo->length == 3) 959 prefix = (struct in6_addr *)rinfo->prefix; 960 else { 961 /* this function is safe */ 962 ipv6_addr_prefix(&prefix_buf, 963 (struct in6_addr *)rinfo->prefix, 964 rinfo->prefix_len); 965 prefix = &prefix_buf; 966 } 967 968 if (rinfo->prefix_len == 0) 969 rt = rt6_get_dflt_router(net, gwaddr, dev); 970 else 971 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len, 972 gwaddr, dev); 973 974 if (rt && !lifetime) { 975 ip6_del_rt(net, rt, false); 976 rt = NULL; 977 } 978 979 if (!rt && lifetime) 980 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, 981 dev, pref); 982 else if (rt) 983 rt->fib6_flags = RTF_ROUTEINFO | 984 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref); 985 986 if (rt) { 987 if (!addrconf_finite_timeout(lifetime)) 988 fib6_clean_expires(rt); 989 else 990 fib6_set_expires(rt, jiffies + HZ * lifetime); 991 992 fib6_info_release(rt); 993 } 994 return 0; 995 } 996 #endif 997 998 /* 999 * Misc support functions 1000 */ 1001 1002 /* called with rcu_lock held */ 1003 static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) 1004 { 1005 struct net_device *dev = res->nh->fib_nh_dev; 1006 1007 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 1008 /* for copies of local routes, dst->dev needs to be the 1009 * device if it is a master device, the master device if 1010 * device is enslaved, and the loopback as the default 1011 */ 1012 if (netif_is_l3_slave(dev) && 1013 !rt6_need_strict(&res->f6i->fib6_dst.addr)) 1014 dev = l3mdev_master_dev_rcu(dev); 1015 else if (!netif_is_l3_master(dev)) 1016 dev = dev_net(dev)->loopback_dev; 1017 /* last case is netif_is_l3_master(dev) is true in which 1018 * case we want dev returned to be dev 1019 */ 1020 } 1021 1022 return dev; 1023 } 1024 1025 static const int fib6_prop[RTN_MAX + 1] = { 1026 [RTN_UNSPEC] = 0, 1027 [RTN_UNICAST] = 0, 1028 [RTN_LOCAL] = 0, 1029 [RTN_BROADCAST] = 0, 1030 [RTN_ANYCAST] = 0, 1031 [RTN_MULTICAST] = 0, 1032 [RTN_BLACKHOLE] = -EINVAL, 1033 [RTN_UNREACHABLE] = -EHOSTUNREACH, 1034 [RTN_PROHIBIT] = -EACCES, 1035 [RTN_THROW] = -EAGAIN, 1036 [RTN_NAT] = -EINVAL, 1037 [RTN_XRESOLVE] = -EINVAL, 1038 }; 1039 1040 static int ip6_rt_type_to_error(u8 fib6_type) 1041 { 1042 return fib6_prop[fib6_type]; 1043 } 1044 1045 static unsigned short fib6_info_dst_flags(struct fib6_info *rt) 1046 { 1047 unsigned short flags = 0; 1048 1049 if (rt->dst_nocount) 1050 flags |= DST_NOCOUNT; 1051 if (rt->dst_nopolicy) 1052 flags |= DST_NOPOLICY; 1053 1054 return flags; 1055 } 1056 1057 static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type) 1058 { 1059 rt->dst.error = ip6_rt_type_to_error(fib6_type); 1060 1061 switch (fib6_type) { 1062 case RTN_BLACKHOLE: 1063 rt->dst.output = dst_discard_out; 1064 rt->dst.input = dst_discard; 1065 break; 1066 case RTN_PROHIBIT: 1067 rt->dst.output = ip6_pkt_prohibit_out; 1068 rt->dst.input = ip6_pkt_prohibit; 1069 break; 1070 case RTN_THROW: 1071 case RTN_UNREACHABLE: 1072 default: 1073 rt->dst.output = ip6_pkt_discard_out; 1074 rt->dst.input = ip6_pkt_discard; 1075 break; 1076 } 1077 } 1078 1079 static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res) 1080 { 1081 struct fib6_info *f6i = res->f6i; 1082 1083 if (res->fib6_flags & RTF_REJECT) { 1084 ip6_rt_init_dst_reject(rt, res->fib6_type); 1085 return; 1086 } 1087 1088 rt->dst.error = 0; 1089 rt->dst.output = ip6_output; 1090 1091 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) { 1092 rt->dst.input = ip6_input; 1093 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 1094 rt->dst.input = ip6_mc_input; 1095 } else { 1096 rt->dst.input = ip6_forward; 1097 } 1098 1099 if (res->nh->fib_nh_lws) { 1100 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws); 1101 lwtunnel_set_redirect(&rt->dst); 1102 } 1103 1104 rt->dst.lastuse = jiffies; 1105 } 1106 1107 /* Caller must already hold reference to @from */ 1108 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from) 1109 { 1110 rt->rt6i_flags &= ~RTF_EXPIRES; 1111 rcu_assign_pointer(rt->from, from); 1112 ip_dst_init_metrics(&rt->dst, from->fib6_metrics); 1113 } 1114 1115 /* Caller must already hold reference to f6i in result */ 1116 static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res) 1117 { 1118 const struct fib6_nh *nh = res->nh; 1119 const struct net_device *dev = nh->fib_nh_dev; 1120 struct fib6_info *f6i = res->f6i; 1121 1122 ip6_rt_init_dst(rt, res); 1123 1124 rt->rt6i_dst = f6i->fib6_dst; 1125 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL; 1126 rt->rt6i_flags = res->fib6_flags; 1127 if (nh->fib_nh_gw_family) { 1128 rt->rt6i_gateway = nh->fib_nh_gw6; 1129 rt->rt6i_flags |= RTF_GATEWAY; 1130 } 1131 rt6_set_from(rt, f6i); 1132 #ifdef CONFIG_IPV6_SUBTREES 1133 rt->rt6i_src = f6i->fib6_src; 1134 #endif 1135 } 1136 1137 static struct fib6_node* fib6_backtrack(struct fib6_node *fn, 1138 struct in6_addr *saddr) 1139 { 1140 struct fib6_node *pn, *sn; 1141 while (1) { 1142 if (fn->fn_flags & RTN_TL_ROOT) 1143 return NULL; 1144 pn = rcu_dereference(fn->parent); 1145 sn = FIB6_SUBTREE(pn); 1146 if (sn && sn != fn) 1147 fn = fib6_node_lookup(sn, NULL, saddr); 1148 else 1149 fn = pn; 1150 if (fn->fn_flags & RTN_RTINFO) 1151 return fn; 1152 } 1153 } 1154 1155 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt) 1156 { 1157 struct rt6_info *rt = *prt; 1158 1159 if (dst_hold_safe(&rt->dst)) 1160 return true; 1161 if (net) { 1162 rt = net->ipv6.ip6_null_entry; 1163 dst_hold(&rt->dst); 1164 } else { 1165 rt = NULL; 1166 } 1167 *prt = rt; 1168 return false; 1169 } 1170 1171 /* called with rcu_lock held */ 1172 static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res) 1173 { 1174 struct net_device *dev = res->nh->fib_nh_dev; 1175 struct fib6_info *f6i = res->f6i; 1176 unsigned short flags; 1177 struct rt6_info *nrt; 1178 1179 if (!fib6_info_hold_safe(f6i)) 1180 goto fallback; 1181 1182 flags = fib6_info_dst_flags(f6i); 1183 nrt = ip6_dst_alloc(dev_net(dev), dev, flags); 1184 if (!nrt) { 1185 fib6_info_release(f6i); 1186 goto fallback; 1187 } 1188 1189 ip6_rt_copy_init(nrt, res); 1190 return nrt; 1191 1192 fallback: 1193 nrt = dev_net(dev)->ipv6.ip6_null_entry; 1194 dst_hold(&nrt->dst); 1195 return nrt; 1196 } 1197 1198 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, 1199 struct fib6_table *table, 1200 struct flowi6 *fl6, 1201 const struct sk_buff *skb, 1202 int flags) 1203 { 1204 struct fib6_result res = {}; 1205 struct fib6_node *fn; 1206 struct rt6_info *rt; 1207 1208 if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) 1209 flags &= ~RT6_LOOKUP_F_IFACE; 1210 1211 rcu_read_lock(); 1212 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1213 restart: 1214 res.f6i = rcu_dereference(fn->leaf); 1215 if (!res.f6i) 1216 res.f6i = net->ipv6.fib6_null_entry; 1217 else 1218 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif, 1219 flags); 1220 1221 if (res.f6i == net->ipv6.fib6_null_entry) { 1222 fn = fib6_backtrack(fn, &fl6->saddr); 1223 if (fn) 1224 goto restart; 1225 1226 rt = net->ipv6.ip6_null_entry; 1227 dst_hold(&rt->dst); 1228 goto out; 1229 } else if (res.fib6_flags & RTF_REJECT) { 1230 goto do_create; 1231 } 1232 1233 fib6_select_path(net, &res, fl6, fl6->flowi6_oif, 1234 fl6->flowi6_oif != 0, skb, flags); 1235 1236 /* Search through exception table */ 1237 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1238 if (rt) { 1239 if (ip6_hold_safe(net, &rt)) 1240 dst_use_noref(&rt->dst, jiffies); 1241 } else { 1242 do_create: 1243 rt = ip6_create_rt_rcu(&res); 1244 } 1245 1246 out: 1247 trace_fib6_table_lookup(net, &res, table, fl6); 1248 1249 rcu_read_unlock(); 1250 1251 return rt; 1252 } 1253 1254 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6, 1255 const struct sk_buff *skb, int flags) 1256 { 1257 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup); 1258 } 1259 EXPORT_SYMBOL_GPL(ip6_route_lookup); 1260 1261 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr, 1262 const struct in6_addr *saddr, int oif, 1263 const struct sk_buff *skb, int strict) 1264 { 1265 struct flowi6 fl6 = { 1266 .flowi6_oif = oif, 1267 .daddr = *daddr, 1268 }; 1269 struct dst_entry *dst; 1270 int flags = strict ? RT6_LOOKUP_F_IFACE : 0; 1271 1272 if (saddr) { 1273 memcpy(&fl6.saddr, saddr, sizeof(*saddr)); 1274 flags |= RT6_LOOKUP_F_HAS_SADDR; 1275 } 1276 1277 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup); 1278 if (dst->error == 0) 1279 return (struct rt6_info *) dst; 1280 1281 dst_release(dst); 1282 1283 return NULL; 1284 } 1285 EXPORT_SYMBOL(rt6_lookup); 1286 1287 /* ip6_ins_rt is called with FREE table->tb6_lock. 1288 * It takes new route entry, the addition fails by any reason the 1289 * route is released. 1290 * Caller must hold dst before calling it. 1291 */ 1292 1293 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info, 1294 struct netlink_ext_ack *extack) 1295 { 1296 int err; 1297 struct fib6_table *table; 1298 1299 table = rt->fib6_table; 1300 spin_lock_bh(&table->tb6_lock); 1301 err = fib6_add(&table->tb6_root, rt, info, extack); 1302 spin_unlock_bh(&table->tb6_lock); 1303 1304 return err; 1305 } 1306 1307 int ip6_ins_rt(struct net *net, struct fib6_info *rt) 1308 { 1309 struct nl_info info = { .nl_net = net, }; 1310 1311 return __ip6_ins_rt(rt, &info, NULL); 1312 } 1313 1314 static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res, 1315 const struct in6_addr *daddr, 1316 const struct in6_addr *saddr) 1317 { 1318 struct fib6_info *f6i = res->f6i; 1319 struct net_device *dev; 1320 struct rt6_info *rt; 1321 1322 /* 1323 * Clone the route. 1324 */ 1325 1326 if (!fib6_info_hold_safe(f6i)) 1327 return NULL; 1328 1329 dev = ip6_rt_get_dev_rcu(res); 1330 rt = ip6_dst_alloc(dev_net(dev), dev, 0); 1331 if (!rt) { 1332 fib6_info_release(f6i); 1333 return NULL; 1334 } 1335 1336 ip6_rt_copy_init(rt, res); 1337 rt->rt6i_flags |= RTF_CACHE; 1338 rt->rt6i_dst.addr = *daddr; 1339 rt->rt6i_dst.plen = 128; 1340 1341 if (!rt6_is_gw_or_nonexthop(res)) { 1342 if (f6i->fib6_dst.plen != 128 && 1343 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr)) 1344 rt->rt6i_flags |= RTF_ANYCAST; 1345 #ifdef CONFIG_IPV6_SUBTREES 1346 if (rt->rt6i_src.plen && saddr) { 1347 rt->rt6i_src.addr = *saddr; 1348 rt->rt6i_src.plen = 128; 1349 } 1350 #endif 1351 } 1352 1353 return rt; 1354 } 1355 1356 static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res) 1357 { 1358 struct fib6_info *f6i = res->f6i; 1359 unsigned short flags = fib6_info_dst_flags(f6i); 1360 struct net_device *dev; 1361 struct rt6_info *pcpu_rt; 1362 1363 if (!fib6_info_hold_safe(f6i)) 1364 return NULL; 1365 1366 rcu_read_lock(); 1367 dev = ip6_rt_get_dev_rcu(res); 1368 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT); 1369 rcu_read_unlock(); 1370 if (!pcpu_rt) { 1371 fib6_info_release(f6i); 1372 return NULL; 1373 } 1374 ip6_rt_copy_init(pcpu_rt, res); 1375 pcpu_rt->rt6i_flags |= RTF_PCPU; 1376 1377 if (f6i->nh) 1378 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev)); 1379 1380 return pcpu_rt; 1381 } 1382 1383 static bool rt6_is_valid(const struct rt6_info *rt6) 1384 { 1385 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev)); 1386 } 1387 1388 /* It should be called with rcu_read_lock() acquired */ 1389 static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) 1390 { 1391 struct rt6_info *pcpu_rt; 1392 1393 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu); 1394 1395 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) { 1396 struct rt6_info *prev, **p; 1397 1398 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1399 prev = xchg(p, NULL); 1400 if (prev) { 1401 dst_dev_put(&prev->dst); 1402 dst_release(&prev->dst); 1403 } 1404 1405 pcpu_rt = NULL; 1406 } 1407 1408 return pcpu_rt; 1409 } 1410 1411 static struct rt6_info *rt6_make_pcpu_route(struct net *net, 1412 const struct fib6_result *res) 1413 { 1414 struct rt6_info *pcpu_rt, *prev, **p; 1415 1416 pcpu_rt = ip6_rt_pcpu_alloc(res); 1417 if (!pcpu_rt) 1418 return NULL; 1419 1420 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1421 prev = cmpxchg(p, NULL, pcpu_rt); 1422 BUG_ON(prev); 1423 1424 if (res->f6i->fib6_destroying) { 1425 struct fib6_info *from; 1426 1427 from = xchg((__force struct fib6_info **)&pcpu_rt->from, NULL); 1428 fib6_info_release(from); 1429 } 1430 1431 return pcpu_rt; 1432 } 1433 1434 /* exception hash table implementation 1435 */ 1436 static DEFINE_SPINLOCK(rt6_exception_lock); 1437 1438 /* Remove rt6_ex from hash table and free the memory 1439 * Caller must hold rt6_exception_lock 1440 */ 1441 static void rt6_remove_exception(struct rt6_exception_bucket *bucket, 1442 struct rt6_exception *rt6_ex) 1443 { 1444 struct fib6_info *from; 1445 struct net *net; 1446 1447 if (!bucket || !rt6_ex) 1448 return; 1449 1450 net = dev_net(rt6_ex->rt6i->dst.dev); 1451 net->ipv6.rt6_stats->fib_rt_cache--; 1452 1453 /* purge completely the exception to allow releasing the held resources: 1454 * some [sk] cache may keep the dst around for unlimited time 1455 */ 1456 from = xchg((__force struct fib6_info **)&rt6_ex->rt6i->from, NULL); 1457 fib6_info_release(from); 1458 dst_dev_put(&rt6_ex->rt6i->dst); 1459 1460 hlist_del_rcu(&rt6_ex->hlist); 1461 dst_release(&rt6_ex->rt6i->dst); 1462 kfree_rcu(rt6_ex, rcu); 1463 WARN_ON_ONCE(!bucket->depth); 1464 bucket->depth--; 1465 } 1466 1467 /* Remove oldest rt6_ex in bucket and free the memory 1468 * Caller must hold rt6_exception_lock 1469 */ 1470 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) 1471 { 1472 struct rt6_exception *rt6_ex, *oldest = NULL; 1473 1474 if (!bucket) 1475 return; 1476 1477 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 1478 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp)) 1479 oldest = rt6_ex; 1480 } 1481 rt6_remove_exception(bucket, oldest); 1482 } 1483 1484 static u32 rt6_exception_hash(const struct in6_addr *dst, 1485 const struct in6_addr *src) 1486 { 1487 static u32 seed __read_mostly; 1488 u32 val; 1489 1490 net_get_random_once(&seed, sizeof(seed)); 1491 val = jhash2((const u32 *)dst, sizeof(*dst)/sizeof(u32), seed); 1492 1493 #ifdef CONFIG_IPV6_SUBTREES 1494 if (src) 1495 val = jhash2((const u32 *)src, sizeof(*src)/sizeof(u32), val); 1496 #endif 1497 return hash_32(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT); 1498 } 1499 1500 /* Helper function to find the cached rt in the hash table 1501 * and update bucket pointer to point to the bucket for this 1502 * (daddr, saddr) pair 1503 * Caller must hold rt6_exception_lock 1504 */ 1505 static struct rt6_exception * 1506 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket, 1507 const struct in6_addr *daddr, 1508 const struct in6_addr *saddr) 1509 { 1510 struct rt6_exception *rt6_ex; 1511 u32 hval; 1512 1513 if (!(*bucket) || !daddr) 1514 return NULL; 1515 1516 hval = rt6_exception_hash(daddr, saddr); 1517 *bucket += hval; 1518 1519 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) { 1520 struct rt6_info *rt6 = rt6_ex->rt6i; 1521 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1522 1523 #ifdef CONFIG_IPV6_SUBTREES 1524 if (matched && saddr) 1525 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1526 #endif 1527 if (matched) 1528 return rt6_ex; 1529 } 1530 return NULL; 1531 } 1532 1533 /* Helper function to find the cached rt in the hash table 1534 * and update bucket pointer to point to the bucket for this 1535 * (daddr, saddr) pair 1536 * Caller must hold rcu_read_lock() 1537 */ 1538 static struct rt6_exception * 1539 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket, 1540 const struct in6_addr *daddr, 1541 const struct in6_addr *saddr) 1542 { 1543 struct rt6_exception *rt6_ex; 1544 u32 hval; 1545 1546 WARN_ON_ONCE(!rcu_read_lock_held()); 1547 1548 if (!(*bucket) || !daddr) 1549 return NULL; 1550 1551 hval = rt6_exception_hash(daddr, saddr); 1552 *bucket += hval; 1553 1554 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) { 1555 struct rt6_info *rt6 = rt6_ex->rt6i; 1556 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1557 1558 #ifdef CONFIG_IPV6_SUBTREES 1559 if (matched && saddr) 1560 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1561 #endif 1562 if (matched) 1563 return rt6_ex; 1564 } 1565 return NULL; 1566 } 1567 1568 static unsigned int fib6_mtu(const struct fib6_result *res) 1569 { 1570 const struct fib6_nh *nh = res->nh; 1571 unsigned int mtu; 1572 1573 if (res->f6i->fib6_pmtu) { 1574 mtu = res->f6i->fib6_pmtu; 1575 } else { 1576 struct net_device *dev = nh->fib_nh_dev; 1577 struct inet6_dev *idev; 1578 1579 rcu_read_lock(); 1580 idev = __in6_dev_get(dev); 1581 mtu = idev->cnf.mtu6; 1582 rcu_read_unlock(); 1583 } 1584 1585 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 1586 1587 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 1588 } 1589 1590 #define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL 1591 1592 /* used when the flushed bit is not relevant, only access to the bucket 1593 * (ie., all bucket users except rt6_insert_exception); 1594 * 1595 * called under rcu lock; sometimes called with rt6_exception_lock held 1596 */ 1597 static 1598 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh, 1599 spinlock_t *lock) 1600 { 1601 struct rt6_exception_bucket *bucket; 1602 1603 if (lock) 1604 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1605 lockdep_is_held(lock)); 1606 else 1607 bucket = rcu_dereference(nh->rt6i_exception_bucket); 1608 1609 /* remove bucket flushed bit if set */ 1610 if (bucket) { 1611 unsigned long p = (unsigned long)bucket; 1612 1613 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED; 1614 bucket = (struct rt6_exception_bucket *)p; 1615 } 1616 1617 return bucket; 1618 } 1619 1620 static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket) 1621 { 1622 unsigned long p = (unsigned long)bucket; 1623 1624 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED); 1625 } 1626 1627 /* called with rt6_exception_lock held */ 1628 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh, 1629 spinlock_t *lock) 1630 { 1631 struct rt6_exception_bucket *bucket; 1632 unsigned long p; 1633 1634 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1635 lockdep_is_held(lock)); 1636 1637 p = (unsigned long)bucket; 1638 p |= FIB6_EXCEPTION_BUCKET_FLUSHED; 1639 bucket = (struct rt6_exception_bucket *)p; 1640 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1641 } 1642 1643 static int rt6_insert_exception(struct rt6_info *nrt, 1644 const struct fib6_result *res) 1645 { 1646 struct net *net = dev_net(nrt->dst.dev); 1647 struct rt6_exception_bucket *bucket; 1648 struct fib6_info *f6i = res->f6i; 1649 struct in6_addr *src_key = NULL; 1650 struct rt6_exception *rt6_ex; 1651 struct fib6_nh *nh = res->nh; 1652 int err = 0; 1653 1654 spin_lock_bh(&rt6_exception_lock); 1655 1656 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1657 lockdep_is_held(&rt6_exception_lock)); 1658 if (!bucket) { 1659 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket), 1660 GFP_ATOMIC); 1661 if (!bucket) { 1662 err = -ENOMEM; 1663 goto out; 1664 } 1665 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1666 } else if (fib6_nh_excptn_bucket_flushed(bucket)) { 1667 err = -EINVAL; 1668 goto out; 1669 } 1670 1671 #ifdef CONFIG_IPV6_SUBTREES 1672 /* fib6_src.plen != 0 indicates f6i is in subtree 1673 * and exception table is indexed by a hash of 1674 * both fib6_dst and fib6_src. 1675 * Otherwise, the exception table is indexed by 1676 * a hash of only fib6_dst. 1677 */ 1678 if (f6i->fib6_src.plen) 1679 src_key = &nrt->rt6i_src.addr; 1680 #endif 1681 /* rt6_mtu_change() might lower mtu on f6i. 1682 * Only insert this exception route if its mtu 1683 * is less than f6i's mtu value. 1684 */ 1685 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) { 1686 err = -EINVAL; 1687 goto out; 1688 } 1689 1690 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr, 1691 src_key); 1692 if (rt6_ex) 1693 rt6_remove_exception(bucket, rt6_ex); 1694 1695 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC); 1696 if (!rt6_ex) { 1697 err = -ENOMEM; 1698 goto out; 1699 } 1700 rt6_ex->rt6i = nrt; 1701 rt6_ex->stamp = jiffies; 1702 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain); 1703 bucket->depth++; 1704 net->ipv6.rt6_stats->fib_rt_cache++; 1705 1706 if (bucket->depth > FIB6_MAX_DEPTH) 1707 rt6_exception_remove_oldest(bucket); 1708 1709 out: 1710 spin_unlock_bh(&rt6_exception_lock); 1711 1712 /* Update fn->fn_sernum to invalidate all cached dst */ 1713 if (!err) { 1714 spin_lock_bh(&f6i->fib6_table->tb6_lock); 1715 fib6_update_sernum(net, f6i); 1716 spin_unlock_bh(&f6i->fib6_table->tb6_lock); 1717 fib6_force_start_gc(net); 1718 } 1719 1720 return err; 1721 } 1722 1723 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from) 1724 { 1725 struct rt6_exception_bucket *bucket; 1726 struct rt6_exception *rt6_ex; 1727 struct hlist_node *tmp; 1728 int i; 1729 1730 spin_lock_bh(&rt6_exception_lock); 1731 1732 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1733 if (!bucket) 1734 goto out; 1735 1736 /* Prevent rt6_insert_exception() to recreate the bucket list */ 1737 if (!from) 1738 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock); 1739 1740 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 1741 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) { 1742 if (!from || 1743 rcu_access_pointer(rt6_ex->rt6i->from) == from) 1744 rt6_remove_exception(bucket, rt6_ex); 1745 } 1746 WARN_ON_ONCE(!from && bucket->depth); 1747 bucket++; 1748 } 1749 out: 1750 spin_unlock_bh(&rt6_exception_lock); 1751 } 1752 1753 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg) 1754 { 1755 struct fib6_info *f6i = arg; 1756 1757 fib6_nh_flush_exceptions(nh, f6i); 1758 1759 return 0; 1760 } 1761 1762 void rt6_flush_exceptions(struct fib6_info *f6i) 1763 { 1764 if (f6i->nh) 1765 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions, 1766 f6i); 1767 else 1768 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i); 1769 } 1770 1771 /* Find cached rt in the hash table inside passed in rt 1772 * Caller has to hold rcu_read_lock() 1773 */ 1774 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 1775 const struct in6_addr *daddr, 1776 const struct in6_addr *saddr) 1777 { 1778 const struct in6_addr *src_key = NULL; 1779 struct rt6_exception_bucket *bucket; 1780 struct rt6_exception *rt6_ex; 1781 struct rt6_info *ret = NULL; 1782 1783 #ifdef CONFIG_IPV6_SUBTREES 1784 /* fib6i_src.plen != 0 indicates f6i is in subtree 1785 * and exception table is indexed by a hash of 1786 * both fib6_dst and fib6_src. 1787 * However, the src addr used to create the hash 1788 * might not be exactly the passed in saddr which 1789 * is a /128 addr from the flow. 1790 * So we need to use f6i->fib6_src to redo lookup 1791 * if the passed in saddr does not find anything. 1792 * (See the logic in ip6_rt_cache_alloc() on how 1793 * rt->rt6i_src is updated.) 1794 */ 1795 if (res->f6i->fib6_src.plen) 1796 src_key = saddr; 1797 find_ex: 1798 #endif 1799 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL); 1800 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key); 1801 1802 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i)) 1803 ret = rt6_ex->rt6i; 1804 1805 #ifdef CONFIG_IPV6_SUBTREES 1806 /* Use fib6_src as src_key and redo lookup */ 1807 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) { 1808 src_key = &res->f6i->fib6_src.addr; 1809 goto find_ex; 1810 } 1811 #endif 1812 1813 return ret; 1814 } 1815 1816 /* Remove the passed in cached rt from the hash table that contains it */ 1817 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen, 1818 const struct rt6_info *rt) 1819 { 1820 const struct in6_addr *src_key = NULL; 1821 struct rt6_exception_bucket *bucket; 1822 struct rt6_exception *rt6_ex; 1823 int err; 1824 1825 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 1826 return -ENOENT; 1827 1828 spin_lock_bh(&rt6_exception_lock); 1829 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1830 1831 #ifdef CONFIG_IPV6_SUBTREES 1832 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1833 * and exception table is indexed by a hash of 1834 * both rt6i_dst and rt6i_src. 1835 * Otherwise, the exception table is indexed by 1836 * a hash of only rt6i_dst. 1837 */ 1838 if (plen) 1839 src_key = &rt->rt6i_src.addr; 1840 #endif 1841 rt6_ex = __rt6_find_exception_spinlock(&bucket, 1842 &rt->rt6i_dst.addr, 1843 src_key); 1844 if (rt6_ex) { 1845 rt6_remove_exception(bucket, rt6_ex); 1846 err = 0; 1847 } else { 1848 err = -ENOENT; 1849 } 1850 1851 spin_unlock_bh(&rt6_exception_lock); 1852 return err; 1853 } 1854 1855 struct fib6_nh_excptn_arg { 1856 struct rt6_info *rt; 1857 int plen; 1858 }; 1859 1860 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg) 1861 { 1862 struct fib6_nh_excptn_arg *arg = _arg; 1863 int err; 1864 1865 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt); 1866 if (err == 0) 1867 return 1; 1868 1869 return 0; 1870 } 1871 1872 static int rt6_remove_exception_rt(struct rt6_info *rt) 1873 { 1874 struct fib6_info *from; 1875 1876 from = rcu_dereference(rt->from); 1877 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1878 return -EINVAL; 1879 1880 if (from->nh) { 1881 struct fib6_nh_excptn_arg arg = { 1882 .rt = rt, 1883 .plen = from->fib6_src.plen 1884 }; 1885 int rc; 1886 1887 /* rc = 1 means an entry was found */ 1888 rc = nexthop_for_each_fib6_nh(from->nh, 1889 rt6_nh_remove_exception_rt, 1890 &arg); 1891 return rc ? 0 : -ENOENT; 1892 } 1893 1894 return fib6_nh_remove_exception(from->fib6_nh, 1895 from->fib6_src.plen, rt); 1896 } 1897 1898 /* Find rt6_ex which contains the passed in rt cache and 1899 * refresh its stamp 1900 */ 1901 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen, 1902 const struct rt6_info *rt) 1903 { 1904 const struct in6_addr *src_key = NULL; 1905 struct rt6_exception_bucket *bucket; 1906 struct rt6_exception *rt6_ex; 1907 1908 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 1909 #ifdef CONFIG_IPV6_SUBTREES 1910 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1911 * and exception table is indexed by a hash of 1912 * both rt6i_dst and rt6i_src. 1913 * Otherwise, the exception table is indexed by 1914 * a hash of only rt6i_dst. 1915 */ 1916 if (plen) 1917 src_key = &rt->rt6i_src.addr; 1918 #endif 1919 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key); 1920 if (rt6_ex) 1921 rt6_ex->stamp = jiffies; 1922 } 1923 1924 struct fib6_nh_match_arg { 1925 const struct net_device *dev; 1926 const struct in6_addr *gw; 1927 struct fib6_nh *match; 1928 }; 1929 1930 /* determine if fib6_nh has given device and gateway */ 1931 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg) 1932 { 1933 struct fib6_nh_match_arg *arg = _arg; 1934 1935 if (arg->dev != nh->fib_nh_dev || 1936 (arg->gw && !nh->fib_nh_gw_family) || 1937 (!arg->gw && nh->fib_nh_gw_family) || 1938 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6))) 1939 return 0; 1940 1941 arg->match = nh; 1942 1943 /* found a match, break the loop */ 1944 return 1; 1945 } 1946 1947 static void rt6_update_exception_stamp_rt(struct rt6_info *rt) 1948 { 1949 struct fib6_info *from; 1950 struct fib6_nh *fib6_nh; 1951 1952 rcu_read_lock(); 1953 1954 from = rcu_dereference(rt->from); 1955 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1956 goto unlock; 1957 1958 if (from->nh) { 1959 struct fib6_nh_match_arg arg = { 1960 .dev = rt->dst.dev, 1961 .gw = &rt->rt6i_gateway, 1962 }; 1963 1964 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg); 1965 1966 if (!arg.match) 1967 goto unlock; 1968 fib6_nh = arg.match; 1969 } else { 1970 fib6_nh = from->fib6_nh; 1971 } 1972 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt); 1973 unlock: 1974 rcu_read_unlock(); 1975 } 1976 1977 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, 1978 struct rt6_info *rt, int mtu) 1979 { 1980 /* If the new MTU is lower than the route PMTU, this new MTU will be the 1981 * lowest MTU in the path: always allow updating the route PMTU to 1982 * reflect PMTU decreases. 1983 * 1984 * If the new MTU is higher, and the route PMTU is equal to the local 1985 * MTU, this means the old MTU is the lowest in the path, so allow 1986 * updating it: if other nodes now have lower MTUs, PMTU discovery will 1987 * handle this. 1988 */ 1989 1990 if (dst_mtu(&rt->dst) >= mtu) 1991 return true; 1992 1993 if (dst_mtu(&rt->dst) == idev->cnf.mtu6) 1994 return true; 1995 1996 return false; 1997 } 1998 1999 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev, 2000 const struct fib6_nh *nh, int mtu) 2001 { 2002 struct rt6_exception_bucket *bucket; 2003 struct rt6_exception *rt6_ex; 2004 int i; 2005 2006 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2007 if (!bucket) 2008 return; 2009 2010 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2011 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 2012 struct rt6_info *entry = rt6_ex->rt6i; 2013 2014 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected 2015 * route), the metrics of its rt->from have already 2016 * been updated. 2017 */ 2018 if (dst_metric_raw(&entry->dst, RTAX_MTU) && 2019 rt6_mtu_change_route_allowed(idev, entry, mtu)) 2020 dst_metric_set(&entry->dst, RTAX_MTU, mtu); 2021 } 2022 bucket++; 2023 } 2024 } 2025 2026 #define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE) 2027 2028 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh, 2029 const struct in6_addr *gateway) 2030 { 2031 struct rt6_exception_bucket *bucket; 2032 struct rt6_exception *rt6_ex; 2033 struct hlist_node *tmp; 2034 int i; 2035 2036 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2037 return; 2038 2039 spin_lock_bh(&rt6_exception_lock); 2040 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2041 if (bucket) { 2042 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2043 hlist_for_each_entry_safe(rt6_ex, tmp, 2044 &bucket->chain, hlist) { 2045 struct rt6_info *entry = rt6_ex->rt6i; 2046 2047 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) == 2048 RTF_CACHE_GATEWAY && 2049 ipv6_addr_equal(gateway, 2050 &entry->rt6i_gateway)) { 2051 rt6_remove_exception(bucket, rt6_ex); 2052 } 2053 } 2054 bucket++; 2055 } 2056 } 2057 2058 spin_unlock_bh(&rt6_exception_lock); 2059 } 2060 2061 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket, 2062 struct rt6_exception *rt6_ex, 2063 struct fib6_gc_args *gc_args, 2064 unsigned long now) 2065 { 2066 struct rt6_info *rt = rt6_ex->rt6i; 2067 2068 /* we are pruning and obsoleting aged-out and non gateway exceptions 2069 * even if others have still references to them, so that on next 2070 * dst_check() such references can be dropped. 2071 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when 2072 * expired, independently from their aging, as per RFC 8201 section 4 2073 */ 2074 if (!(rt->rt6i_flags & RTF_EXPIRES)) { 2075 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) { 2076 RT6_TRACE("aging clone %p\n", rt); 2077 rt6_remove_exception(bucket, rt6_ex); 2078 return; 2079 } 2080 } else if (time_after(jiffies, rt->dst.expires)) { 2081 RT6_TRACE("purging expired route %p\n", rt); 2082 rt6_remove_exception(bucket, rt6_ex); 2083 return; 2084 } 2085 2086 if (rt->rt6i_flags & RTF_GATEWAY) { 2087 struct neighbour *neigh; 2088 2089 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway); 2090 2091 if (!(neigh && (neigh->flags & NTF_ROUTER))) { 2092 RT6_TRACE("purging route %p via non-router but gateway\n", 2093 rt); 2094 rt6_remove_exception(bucket, rt6_ex); 2095 return; 2096 } 2097 } 2098 2099 gc_args->more++; 2100 } 2101 2102 static void fib6_nh_age_exceptions(const struct fib6_nh *nh, 2103 struct fib6_gc_args *gc_args, 2104 unsigned long now) 2105 { 2106 struct rt6_exception_bucket *bucket; 2107 struct rt6_exception *rt6_ex; 2108 struct hlist_node *tmp; 2109 int i; 2110 2111 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2112 return; 2113 2114 rcu_read_lock_bh(); 2115 spin_lock(&rt6_exception_lock); 2116 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2117 if (bucket) { 2118 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2119 hlist_for_each_entry_safe(rt6_ex, tmp, 2120 &bucket->chain, hlist) { 2121 rt6_age_examine_exception(bucket, rt6_ex, 2122 gc_args, now); 2123 } 2124 bucket++; 2125 } 2126 } 2127 spin_unlock(&rt6_exception_lock); 2128 rcu_read_unlock_bh(); 2129 } 2130 2131 struct fib6_nh_age_excptn_arg { 2132 struct fib6_gc_args *gc_args; 2133 unsigned long now; 2134 }; 2135 2136 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg) 2137 { 2138 struct fib6_nh_age_excptn_arg *arg = _arg; 2139 2140 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now); 2141 return 0; 2142 } 2143 2144 void rt6_age_exceptions(struct fib6_info *f6i, 2145 struct fib6_gc_args *gc_args, 2146 unsigned long now) 2147 { 2148 if (f6i->nh) { 2149 struct fib6_nh_age_excptn_arg arg = { 2150 .gc_args = gc_args, 2151 .now = now 2152 }; 2153 2154 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions, 2155 &arg); 2156 } else { 2157 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now); 2158 } 2159 } 2160 2161 /* must be called with rcu lock held */ 2162 int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif, 2163 struct flowi6 *fl6, struct fib6_result *res, int strict) 2164 { 2165 struct fib6_node *fn, *saved_fn; 2166 2167 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2168 saved_fn = fn; 2169 2170 if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) 2171 oif = 0; 2172 2173 redo_rt6_select: 2174 rt6_select(net, fn, oif, res, strict); 2175 if (res->f6i == net->ipv6.fib6_null_entry) { 2176 fn = fib6_backtrack(fn, &fl6->saddr); 2177 if (fn) 2178 goto redo_rt6_select; 2179 else if (strict & RT6_LOOKUP_F_REACHABLE) { 2180 /* also consider unreachable route */ 2181 strict &= ~RT6_LOOKUP_F_REACHABLE; 2182 fn = saved_fn; 2183 goto redo_rt6_select; 2184 } 2185 } 2186 2187 trace_fib6_table_lookup(net, res, table, fl6); 2188 2189 return 0; 2190 } 2191 2192 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, 2193 int oif, struct flowi6 *fl6, 2194 const struct sk_buff *skb, int flags) 2195 { 2196 struct fib6_result res = {}; 2197 struct rt6_info *rt = NULL; 2198 int strict = 0; 2199 2200 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) && 2201 !rcu_read_lock_held()); 2202 2203 strict |= flags & RT6_LOOKUP_F_IFACE; 2204 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE; 2205 if (net->ipv6.devconf_all->forwarding == 0) 2206 strict |= RT6_LOOKUP_F_REACHABLE; 2207 2208 rcu_read_lock(); 2209 2210 fib6_table_lookup(net, table, oif, fl6, &res, strict); 2211 if (res.f6i == net->ipv6.fib6_null_entry) 2212 goto out; 2213 2214 fib6_select_path(net, &res, fl6, oif, false, skb, strict); 2215 2216 /*Search through exception table */ 2217 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 2218 if (rt) { 2219 goto out; 2220 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) && 2221 !res.nh->fib_nh_gw_family)) { 2222 /* Create a RTF_CACHE clone which will not be 2223 * owned by the fib6 tree. It is for the special case where 2224 * the daddr in the skb during the neighbor look-up is different 2225 * from the fl6->daddr used to look-up route here. 2226 */ 2227 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL); 2228 2229 if (rt) { 2230 /* 1 refcnt is taken during ip6_rt_cache_alloc(). 2231 * As rt6_uncached_list_add() does not consume refcnt, 2232 * this refcnt is always returned to the caller even 2233 * if caller sets RT6_LOOKUP_F_DST_NOREF flag. 2234 */ 2235 rt6_uncached_list_add(rt); 2236 atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache); 2237 rcu_read_unlock(); 2238 2239 return rt; 2240 } 2241 } else { 2242 /* Get a percpu copy */ 2243 local_bh_disable(); 2244 rt = rt6_get_pcpu_route(&res); 2245 2246 if (!rt) 2247 rt = rt6_make_pcpu_route(net, &res); 2248 2249 local_bh_enable(); 2250 } 2251 out: 2252 if (!rt) 2253 rt = net->ipv6.ip6_null_entry; 2254 if (!(flags & RT6_LOOKUP_F_DST_NOREF)) 2255 ip6_hold_safe(net, &rt); 2256 rcu_read_unlock(); 2257 2258 return rt; 2259 } 2260 EXPORT_SYMBOL_GPL(ip6_pol_route); 2261 2262 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, 2263 struct fib6_table *table, 2264 struct flowi6 *fl6, 2265 const struct sk_buff *skb, 2266 int flags) 2267 { 2268 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags); 2269 } 2270 2271 struct dst_entry *ip6_route_input_lookup(struct net *net, 2272 struct net_device *dev, 2273 struct flowi6 *fl6, 2274 const struct sk_buff *skb, 2275 int flags) 2276 { 2277 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG) 2278 flags |= RT6_LOOKUP_F_IFACE; 2279 2280 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input); 2281 } 2282 EXPORT_SYMBOL_GPL(ip6_route_input_lookup); 2283 2284 static void ip6_multipath_l3_keys(const struct sk_buff *skb, 2285 struct flow_keys *keys, 2286 struct flow_keys *flkeys) 2287 { 2288 const struct ipv6hdr *outer_iph = ipv6_hdr(skb); 2289 const struct ipv6hdr *key_iph = outer_iph; 2290 struct flow_keys *_flkeys = flkeys; 2291 const struct ipv6hdr *inner_iph; 2292 const struct icmp6hdr *icmph; 2293 struct ipv6hdr _inner_iph; 2294 struct icmp6hdr _icmph; 2295 2296 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6)) 2297 goto out; 2298 2299 icmph = skb_header_pointer(skb, skb_transport_offset(skb), 2300 sizeof(_icmph), &_icmph); 2301 if (!icmph) 2302 goto out; 2303 2304 if (!icmpv6_is_err(icmph->icmp6_type)) 2305 goto out; 2306 2307 inner_iph = skb_header_pointer(skb, 2308 skb_transport_offset(skb) + sizeof(*icmph), 2309 sizeof(_inner_iph), &_inner_iph); 2310 if (!inner_iph) 2311 goto out; 2312 2313 key_iph = inner_iph; 2314 _flkeys = NULL; 2315 out: 2316 if (_flkeys) { 2317 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src; 2318 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst; 2319 keys->tags.flow_label = _flkeys->tags.flow_label; 2320 keys->basic.ip_proto = _flkeys->basic.ip_proto; 2321 } else { 2322 keys->addrs.v6addrs.src = key_iph->saddr; 2323 keys->addrs.v6addrs.dst = key_iph->daddr; 2324 keys->tags.flow_label = ip6_flowlabel(key_iph); 2325 keys->basic.ip_proto = key_iph->nexthdr; 2326 } 2327 } 2328 2329 static u32 rt6_multipath_custom_hash_outer(const struct net *net, 2330 const struct sk_buff *skb, 2331 bool *p_has_inner) 2332 { 2333 u32 hash_fields = ip6_multipath_hash_fields(net); 2334 struct flow_keys keys, hash_keys; 2335 2336 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2337 return 0; 2338 2339 memset(&hash_keys, 0, sizeof(hash_keys)); 2340 skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP); 2341 2342 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2343 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2344 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2345 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2346 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2347 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2348 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2349 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2350 hash_keys.tags.flow_label = keys.tags.flow_label; 2351 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2352 hash_keys.ports.src = keys.ports.src; 2353 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2354 hash_keys.ports.dst = keys.ports.dst; 2355 2356 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION); 2357 return flow_hash_from_keys(&hash_keys); 2358 } 2359 2360 static u32 rt6_multipath_custom_hash_inner(const struct net *net, 2361 const struct sk_buff *skb, 2362 bool has_inner) 2363 { 2364 u32 hash_fields = ip6_multipath_hash_fields(net); 2365 struct flow_keys keys, hash_keys; 2366 2367 /* We assume the packet carries an encapsulation, but if none was 2368 * encountered during dissection of the outer flow, then there is no 2369 * point in calling the flow dissector again. 2370 */ 2371 if (!has_inner) 2372 return 0; 2373 2374 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)) 2375 return 0; 2376 2377 memset(&hash_keys, 0, sizeof(hash_keys)); 2378 skb_flow_dissect_flow_keys(skb, &keys, 0); 2379 2380 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION)) 2381 return 0; 2382 2383 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2384 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2385 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2386 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 2387 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2388 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 2389 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2390 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2391 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2392 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2393 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2394 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2395 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 2396 hash_keys.tags.flow_label = keys.tags.flow_label; 2397 } 2398 2399 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 2400 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2401 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 2402 hash_keys.ports.src = keys.ports.src; 2403 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 2404 hash_keys.ports.dst = keys.ports.dst; 2405 2406 return flow_hash_from_keys(&hash_keys); 2407 } 2408 2409 static u32 rt6_multipath_custom_hash_skb(const struct net *net, 2410 const struct sk_buff *skb) 2411 { 2412 u32 mhash, mhash_inner; 2413 bool has_inner = true; 2414 2415 mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner); 2416 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner); 2417 2418 return jhash_2words(mhash, mhash_inner, 0); 2419 } 2420 2421 static u32 rt6_multipath_custom_hash_fl6(const struct net *net, 2422 const struct flowi6 *fl6) 2423 { 2424 u32 hash_fields = ip6_multipath_hash_fields(net); 2425 struct flow_keys hash_keys; 2426 2427 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2428 return 0; 2429 2430 memset(&hash_keys, 0, sizeof(hash_keys)); 2431 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2432 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2433 hash_keys.addrs.v6addrs.src = fl6->saddr; 2434 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2435 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2436 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2437 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2438 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2439 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2440 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2441 hash_keys.ports.src = fl6->fl6_sport; 2442 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2443 hash_keys.ports.dst = fl6->fl6_dport; 2444 2445 return flow_hash_from_keys(&hash_keys); 2446 } 2447 2448 /* if skb is set it will be used and fl6 can be NULL */ 2449 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6, 2450 const struct sk_buff *skb, struct flow_keys *flkeys) 2451 { 2452 struct flow_keys hash_keys; 2453 u32 mhash = 0; 2454 2455 switch (ip6_multipath_hash_policy(net)) { 2456 case 0: 2457 memset(&hash_keys, 0, sizeof(hash_keys)); 2458 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2459 if (skb) { 2460 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2461 } else { 2462 hash_keys.addrs.v6addrs.src = fl6->saddr; 2463 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2464 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2465 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2466 } 2467 mhash = flow_hash_from_keys(&hash_keys); 2468 break; 2469 case 1: 2470 if (skb) { 2471 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 2472 struct flow_keys keys; 2473 2474 /* short-circuit if we already have L4 hash present */ 2475 if (skb->l4_hash) 2476 return skb_get_hash_raw(skb) >> 1; 2477 2478 memset(&hash_keys, 0, sizeof(hash_keys)); 2479 2480 if (!flkeys) { 2481 skb_flow_dissect_flow_keys(skb, &keys, flag); 2482 flkeys = &keys; 2483 } 2484 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2485 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2486 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2487 hash_keys.ports.src = flkeys->ports.src; 2488 hash_keys.ports.dst = flkeys->ports.dst; 2489 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2490 } else { 2491 memset(&hash_keys, 0, sizeof(hash_keys)); 2492 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2493 hash_keys.addrs.v6addrs.src = fl6->saddr; 2494 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2495 hash_keys.ports.src = fl6->fl6_sport; 2496 hash_keys.ports.dst = fl6->fl6_dport; 2497 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2498 } 2499 mhash = flow_hash_from_keys(&hash_keys); 2500 break; 2501 case 2: 2502 memset(&hash_keys, 0, sizeof(hash_keys)); 2503 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2504 if (skb) { 2505 struct flow_keys keys; 2506 2507 if (!flkeys) { 2508 skb_flow_dissect_flow_keys(skb, &keys, 0); 2509 flkeys = &keys; 2510 } 2511 2512 /* Inner can be v4 or v6 */ 2513 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2514 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2515 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 2516 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 2517 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2518 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2519 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2520 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2521 hash_keys.tags.flow_label = flkeys->tags.flow_label; 2522 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2523 } else { 2524 /* Same as case 0 */ 2525 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2526 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2527 } 2528 } else { 2529 /* Same as case 0 */ 2530 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2531 hash_keys.addrs.v6addrs.src = fl6->saddr; 2532 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2533 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2534 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2535 } 2536 mhash = flow_hash_from_keys(&hash_keys); 2537 break; 2538 case 3: 2539 if (skb) 2540 mhash = rt6_multipath_custom_hash_skb(net, skb); 2541 else 2542 mhash = rt6_multipath_custom_hash_fl6(net, fl6); 2543 break; 2544 } 2545 2546 return mhash >> 1; 2547 } 2548 2549 /* Called with rcu held */ 2550 void ip6_route_input(struct sk_buff *skb) 2551 { 2552 const struct ipv6hdr *iph = ipv6_hdr(skb); 2553 struct net *net = dev_net(skb->dev); 2554 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF; 2555 struct ip_tunnel_info *tun_info; 2556 struct flowi6 fl6 = { 2557 .flowi6_iif = skb->dev->ifindex, 2558 .daddr = iph->daddr, 2559 .saddr = iph->saddr, 2560 .flowlabel = ip6_flowinfo(iph), 2561 .flowi6_mark = skb->mark, 2562 .flowi6_proto = iph->nexthdr, 2563 }; 2564 struct flow_keys *flkeys = NULL, _flkeys; 2565 2566 tun_info = skb_tunnel_info(skb); 2567 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2568 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id; 2569 2570 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys)) 2571 flkeys = &_flkeys; 2572 2573 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6)) 2574 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys); 2575 skb_dst_drop(skb); 2576 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev, 2577 &fl6, skb, flags)); 2578 } 2579 2580 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, 2581 struct fib6_table *table, 2582 struct flowi6 *fl6, 2583 const struct sk_buff *skb, 2584 int flags) 2585 { 2586 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags); 2587 } 2588 2589 struct dst_entry *ip6_route_output_flags_noref(struct net *net, 2590 const struct sock *sk, 2591 struct flowi6 *fl6, int flags) 2592 { 2593 bool any_src; 2594 2595 if (ipv6_addr_type(&fl6->daddr) & 2596 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) { 2597 struct dst_entry *dst; 2598 2599 /* This function does not take refcnt on the dst */ 2600 dst = l3mdev_link_scope_lookup(net, fl6); 2601 if (dst) 2602 return dst; 2603 } 2604 2605 fl6->flowi6_iif = LOOPBACK_IFINDEX; 2606 2607 flags |= RT6_LOOKUP_F_DST_NOREF; 2608 any_src = ipv6_addr_any(&fl6->saddr); 2609 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) || 2610 (fl6->flowi6_oif && any_src)) 2611 flags |= RT6_LOOKUP_F_IFACE; 2612 2613 if (!any_src) 2614 flags |= RT6_LOOKUP_F_HAS_SADDR; 2615 else if (sk) 2616 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs); 2617 2618 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output); 2619 } 2620 EXPORT_SYMBOL_GPL(ip6_route_output_flags_noref); 2621 2622 struct dst_entry *ip6_route_output_flags(struct net *net, 2623 const struct sock *sk, 2624 struct flowi6 *fl6, 2625 int flags) 2626 { 2627 struct dst_entry *dst; 2628 struct rt6_info *rt6; 2629 2630 rcu_read_lock(); 2631 dst = ip6_route_output_flags_noref(net, sk, fl6, flags); 2632 rt6 = (struct rt6_info *)dst; 2633 /* For dst cached in uncached_list, refcnt is already taken. */ 2634 if (list_empty(&rt6->rt6i_uncached) && !dst_hold_safe(dst)) { 2635 dst = &net->ipv6.ip6_null_entry->dst; 2636 dst_hold(dst); 2637 } 2638 rcu_read_unlock(); 2639 2640 return dst; 2641 } 2642 EXPORT_SYMBOL_GPL(ip6_route_output_flags); 2643 2644 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2645 { 2646 struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig; 2647 struct net_device *loopback_dev = net->loopback_dev; 2648 struct dst_entry *new = NULL; 2649 2650 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1, 2651 DST_OBSOLETE_DEAD, 0); 2652 if (rt) { 2653 rt6_info_init(rt); 2654 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 2655 2656 new = &rt->dst; 2657 new->__use = 1; 2658 new->input = dst_discard; 2659 new->output = dst_discard_out; 2660 2661 dst_copy_metrics(new, &ort->dst); 2662 2663 rt->rt6i_idev = in6_dev_get(loopback_dev); 2664 rt->rt6i_gateway = ort->rt6i_gateway; 2665 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU; 2666 2667 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key)); 2668 #ifdef CONFIG_IPV6_SUBTREES 2669 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key)); 2670 #endif 2671 } 2672 2673 dst_release(dst_orig); 2674 return new ? new : ERR_PTR(-ENOMEM); 2675 } 2676 2677 /* 2678 * Destination cache support functions 2679 */ 2680 2681 static bool fib6_check(struct fib6_info *f6i, u32 cookie) 2682 { 2683 u32 rt_cookie = 0; 2684 2685 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie) 2686 return false; 2687 2688 if (fib6_check_expired(f6i)) 2689 return false; 2690 2691 return true; 2692 } 2693 2694 static struct dst_entry *rt6_check(struct rt6_info *rt, 2695 struct fib6_info *from, 2696 u32 cookie) 2697 { 2698 u32 rt_cookie = 0; 2699 2700 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) || 2701 rt_cookie != cookie) 2702 return NULL; 2703 2704 if (rt6_check_expired(rt)) 2705 return NULL; 2706 2707 return &rt->dst; 2708 } 2709 2710 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, 2711 struct fib6_info *from, 2712 u32 cookie) 2713 { 2714 if (!__rt6_check_expired(rt) && 2715 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 2716 fib6_check(from, cookie)) 2717 return &rt->dst; 2718 else 2719 return NULL; 2720 } 2721 2722 INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, 2723 u32 cookie) 2724 { 2725 struct dst_entry *dst_ret; 2726 struct fib6_info *from; 2727 struct rt6_info *rt; 2728 2729 rt = container_of(dst, struct rt6_info, dst); 2730 2731 if (rt->sernum) 2732 return rt6_is_valid(rt) ? dst : NULL; 2733 2734 rcu_read_lock(); 2735 2736 /* All IPV6 dsts are created with ->obsolete set to the value 2737 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 2738 * into this function always. 2739 */ 2740 2741 from = rcu_dereference(rt->from); 2742 2743 if (from && (rt->rt6i_flags & RTF_PCPU || 2744 unlikely(!list_empty(&rt->rt6i_uncached)))) 2745 dst_ret = rt6_dst_from_check(rt, from, cookie); 2746 else 2747 dst_ret = rt6_check(rt, from, cookie); 2748 2749 rcu_read_unlock(); 2750 2751 return dst_ret; 2752 } 2753 EXPORT_INDIRECT_CALLABLE(ip6_dst_check); 2754 2755 static struct dst_entry *ip6_negative_advice(struct dst_entry *dst) 2756 { 2757 struct rt6_info *rt = (struct rt6_info *) dst; 2758 2759 if (rt) { 2760 if (rt->rt6i_flags & RTF_CACHE) { 2761 rcu_read_lock(); 2762 if (rt6_check_expired(rt)) { 2763 rt6_remove_exception_rt(rt); 2764 dst = NULL; 2765 } 2766 rcu_read_unlock(); 2767 } else { 2768 dst_release(dst); 2769 dst = NULL; 2770 } 2771 } 2772 return dst; 2773 } 2774 2775 static void ip6_link_failure(struct sk_buff *skb) 2776 { 2777 struct rt6_info *rt; 2778 2779 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 2780 2781 rt = (struct rt6_info *) skb_dst(skb); 2782 if (rt) { 2783 rcu_read_lock(); 2784 if (rt->rt6i_flags & RTF_CACHE) { 2785 rt6_remove_exception_rt(rt); 2786 } else { 2787 struct fib6_info *from; 2788 struct fib6_node *fn; 2789 2790 from = rcu_dereference(rt->from); 2791 if (from) { 2792 fn = rcu_dereference(from->fib6_node); 2793 if (fn && (rt->rt6i_flags & RTF_DEFAULT)) 2794 fn->fn_sernum = -1; 2795 } 2796 } 2797 rcu_read_unlock(); 2798 } 2799 } 2800 2801 static void rt6_update_expires(struct rt6_info *rt0, int timeout) 2802 { 2803 if (!(rt0->rt6i_flags & RTF_EXPIRES)) { 2804 struct fib6_info *from; 2805 2806 rcu_read_lock(); 2807 from = rcu_dereference(rt0->from); 2808 if (from) 2809 rt0->dst.expires = from->expires; 2810 rcu_read_unlock(); 2811 } 2812 2813 dst_set_expires(&rt0->dst, timeout); 2814 rt0->rt6i_flags |= RTF_EXPIRES; 2815 } 2816 2817 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) 2818 { 2819 struct net *net = dev_net(rt->dst.dev); 2820 2821 dst_metric_set(&rt->dst, RTAX_MTU, mtu); 2822 rt->rt6i_flags |= RTF_MODIFIED; 2823 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); 2824 } 2825 2826 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) 2827 { 2828 return !(rt->rt6i_flags & RTF_CACHE) && 2829 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from)); 2830 } 2831 2832 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk, 2833 const struct ipv6hdr *iph, u32 mtu, 2834 bool confirm_neigh) 2835 { 2836 const struct in6_addr *daddr, *saddr; 2837 struct rt6_info *rt6 = (struct rt6_info *)dst; 2838 2839 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU) 2840 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it. 2841 * [see also comment in rt6_mtu_change_route()] 2842 */ 2843 2844 if (iph) { 2845 daddr = &iph->daddr; 2846 saddr = &iph->saddr; 2847 } else if (sk) { 2848 daddr = &sk->sk_v6_daddr; 2849 saddr = &inet6_sk(sk)->saddr; 2850 } else { 2851 daddr = NULL; 2852 saddr = NULL; 2853 } 2854 2855 if (confirm_neigh) 2856 dst_confirm_neigh(dst, daddr); 2857 2858 if (mtu < IPV6_MIN_MTU) 2859 return; 2860 if (mtu >= dst_mtu(dst)) 2861 return; 2862 2863 if (!rt6_cache_allowed_for_pmtu(rt6)) { 2864 rt6_do_update_pmtu(rt6, mtu); 2865 /* update rt6_ex->stamp for cache */ 2866 if (rt6->rt6i_flags & RTF_CACHE) 2867 rt6_update_exception_stamp_rt(rt6); 2868 } else if (daddr) { 2869 struct fib6_result res = {}; 2870 struct rt6_info *nrt6; 2871 2872 rcu_read_lock(); 2873 res.f6i = rcu_dereference(rt6->from); 2874 if (!res.f6i) 2875 goto out_unlock; 2876 2877 res.fib6_flags = res.f6i->fib6_flags; 2878 res.fib6_type = res.f6i->fib6_type; 2879 2880 if (res.f6i->nh) { 2881 struct fib6_nh_match_arg arg = { 2882 .dev = dst->dev, 2883 .gw = &rt6->rt6i_gateway, 2884 }; 2885 2886 nexthop_for_each_fib6_nh(res.f6i->nh, 2887 fib6_nh_find_match, &arg); 2888 2889 /* fib6_info uses a nexthop that does not have fib6_nh 2890 * using the dst->dev + gw. Should be impossible. 2891 */ 2892 if (!arg.match) 2893 goto out_unlock; 2894 2895 res.nh = arg.match; 2896 } else { 2897 res.nh = res.f6i->fib6_nh; 2898 } 2899 2900 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr); 2901 if (nrt6) { 2902 rt6_do_update_pmtu(nrt6, mtu); 2903 if (rt6_insert_exception(nrt6, &res)) 2904 dst_release_immediate(&nrt6->dst); 2905 } 2906 out_unlock: 2907 rcu_read_unlock(); 2908 } 2909 } 2910 2911 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 2912 struct sk_buff *skb, u32 mtu, 2913 bool confirm_neigh) 2914 { 2915 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu, 2916 confirm_neigh); 2917 } 2918 2919 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, 2920 int oif, u32 mark, kuid_t uid) 2921 { 2922 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 2923 struct dst_entry *dst; 2924 struct flowi6 fl6 = { 2925 .flowi6_oif = oif, 2926 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark), 2927 .daddr = iph->daddr, 2928 .saddr = iph->saddr, 2929 .flowlabel = ip6_flowinfo(iph), 2930 .flowi6_uid = uid, 2931 }; 2932 2933 dst = ip6_route_output(net, NULL, &fl6); 2934 if (!dst->error) 2935 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true); 2936 dst_release(dst); 2937 } 2938 EXPORT_SYMBOL_GPL(ip6_update_pmtu); 2939 2940 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu) 2941 { 2942 int oif = sk->sk_bound_dev_if; 2943 struct dst_entry *dst; 2944 2945 if (!oif && skb->dev) 2946 oif = l3mdev_master_ifindex(skb->dev); 2947 2948 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, sk->sk_mark, sk->sk_uid); 2949 2950 dst = __sk_dst_get(sk); 2951 if (!dst || !dst->obsolete || 2952 dst->ops->check(dst, inet6_sk(sk)->dst_cookie)) 2953 return; 2954 2955 bh_lock_sock(sk); 2956 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 2957 ip6_datagram_dst_update(sk, false); 2958 bh_unlock_sock(sk); 2959 } 2960 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu); 2961 2962 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst, 2963 const struct flowi6 *fl6) 2964 { 2965 #ifdef CONFIG_IPV6_SUBTREES 2966 struct ipv6_pinfo *np = inet6_sk(sk); 2967 #endif 2968 2969 ip6_dst_store(sk, dst, 2970 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ? 2971 &sk->sk_v6_daddr : NULL, 2972 #ifdef CONFIG_IPV6_SUBTREES 2973 ipv6_addr_equal(&fl6->saddr, &np->saddr) ? 2974 &np->saddr : 2975 #endif 2976 NULL); 2977 } 2978 2979 static bool ip6_redirect_nh_match(const struct fib6_result *res, 2980 struct flowi6 *fl6, 2981 const struct in6_addr *gw, 2982 struct rt6_info **ret) 2983 { 2984 const struct fib6_nh *nh = res->nh; 2985 2986 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family || 2987 fl6->flowi6_oif != nh->fib_nh_dev->ifindex) 2988 return false; 2989 2990 /* rt_cache's gateway might be different from its 'parent' 2991 * in the case of an ip redirect. 2992 * So we keep searching in the exception table if the gateway 2993 * is different. 2994 */ 2995 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) { 2996 struct rt6_info *rt_cache; 2997 2998 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr); 2999 if (rt_cache && 3000 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) { 3001 *ret = rt_cache; 3002 return true; 3003 } 3004 return false; 3005 } 3006 return true; 3007 } 3008 3009 struct fib6_nh_rd_arg { 3010 struct fib6_result *res; 3011 struct flowi6 *fl6; 3012 const struct in6_addr *gw; 3013 struct rt6_info **ret; 3014 }; 3015 3016 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg) 3017 { 3018 struct fib6_nh_rd_arg *arg = _arg; 3019 3020 arg->res->nh = nh; 3021 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret); 3022 } 3023 3024 /* Handle redirects */ 3025 struct ip6rd_flowi { 3026 struct flowi6 fl6; 3027 struct in6_addr gateway; 3028 }; 3029 3030 INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, 3031 struct fib6_table *table, 3032 struct flowi6 *fl6, 3033 const struct sk_buff *skb, 3034 int flags) 3035 { 3036 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6; 3037 struct rt6_info *ret = NULL; 3038 struct fib6_result res = {}; 3039 struct fib6_nh_rd_arg arg = { 3040 .res = &res, 3041 .fl6 = fl6, 3042 .gw = &rdfl->gateway, 3043 .ret = &ret 3044 }; 3045 struct fib6_info *rt; 3046 struct fib6_node *fn; 3047 3048 /* l3mdev_update_flow overrides oif if the device is enslaved; in 3049 * this case we must match on the real ingress device, so reset it 3050 */ 3051 if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) 3052 fl6->flowi6_oif = skb->dev->ifindex; 3053 3054 /* Get the "current" route for this destination and 3055 * check if the redirect has come from appropriate router. 3056 * 3057 * RFC 4861 specifies that redirects should only be 3058 * accepted if they come from the nexthop to the target. 3059 * Due to the way the routes are chosen, this notion 3060 * is a bit fuzzy and one might need to check all possible 3061 * routes. 3062 */ 3063 3064 rcu_read_lock(); 3065 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 3066 restart: 3067 for_each_fib6_node_rt_rcu(fn) { 3068 res.f6i = rt; 3069 if (fib6_check_expired(rt)) 3070 continue; 3071 if (rt->fib6_flags & RTF_REJECT) 3072 break; 3073 if (unlikely(rt->nh)) { 3074 if (nexthop_is_blackhole(rt->nh)) 3075 continue; 3076 /* on match, res->nh is filled in and potentially ret */ 3077 if (nexthop_for_each_fib6_nh(rt->nh, 3078 fib6_nh_redirect_match, 3079 &arg)) 3080 goto out; 3081 } else { 3082 res.nh = rt->fib6_nh; 3083 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway, 3084 &ret)) 3085 goto out; 3086 } 3087 } 3088 3089 if (!rt) 3090 rt = net->ipv6.fib6_null_entry; 3091 else if (rt->fib6_flags & RTF_REJECT) { 3092 ret = net->ipv6.ip6_null_entry; 3093 goto out; 3094 } 3095 3096 if (rt == net->ipv6.fib6_null_entry) { 3097 fn = fib6_backtrack(fn, &fl6->saddr); 3098 if (fn) 3099 goto restart; 3100 } 3101 3102 res.f6i = rt; 3103 res.nh = rt->fib6_nh; 3104 out: 3105 if (ret) { 3106 ip6_hold_safe(net, &ret); 3107 } else { 3108 res.fib6_flags = res.f6i->fib6_flags; 3109 res.fib6_type = res.f6i->fib6_type; 3110 ret = ip6_create_rt_rcu(&res); 3111 } 3112 3113 rcu_read_unlock(); 3114 3115 trace_fib6_table_lookup(net, &res, table, fl6); 3116 return ret; 3117 }; 3118 3119 static struct dst_entry *ip6_route_redirect(struct net *net, 3120 const struct flowi6 *fl6, 3121 const struct sk_buff *skb, 3122 const struct in6_addr *gateway) 3123 { 3124 int flags = RT6_LOOKUP_F_HAS_SADDR; 3125 struct ip6rd_flowi rdfl; 3126 3127 rdfl.fl6 = *fl6; 3128 rdfl.gateway = *gateway; 3129 3130 return fib6_rule_lookup(net, &rdfl.fl6, skb, 3131 flags, __ip6_route_redirect); 3132 } 3133 3134 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, 3135 kuid_t uid) 3136 { 3137 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 3138 struct dst_entry *dst; 3139 struct flowi6 fl6 = { 3140 .flowi6_iif = LOOPBACK_IFINDEX, 3141 .flowi6_oif = oif, 3142 .flowi6_mark = mark, 3143 .daddr = iph->daddr, 3144 .saddr = iph->saddr, 3145 .flowlabel = ip6_flowinfo(iph), 3146 .flowi6_uid = uid, 3147 }; 3148 3149 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr); 3150 rt6_do_redirect(dst, NULL, skb); 3151 dst_release(dst); 3152 } 3153 EXPORT_SYMBOL_GPL(ip6_redirect); 3154 3155 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif) 3156 { 3157 const struct ipv6hdr *iph = ipv6_hdr(skb); 3158 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb); 3159 struct dst_entry *dst; 3160 struct flowi6 fl6 = { 3161 .flowi6_iif = LOOPBACK_IFINDEX, 3162 .flowi6_oif = oif, 3163 .daddr = msg->dest, 3164 .saddr = iph->daddr, 3165 .flowi6_uid = sock_net_uid(net, NULL), 3166 }; 3167 3168 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr); 3169 rt6_do_redirect(dst, NULL, skb); 3170 dst_release(dst); 3171 } 3172 3173 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) 3174 { 3175 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark, 3176 sk->sk_uid); 3177 } 3178 EXPORT_SYMBOL_GPL(ip6_sk_redirect); 3179 3180 static unsigned int ip6_default_advmss(const struct dst_entry *dst) 3181 { 3182 struct net_device *dev = dst->dev; 3183 unsigned int mtu = dst_mtu(dst); 3184 struct net *net = dev_net(dev); 3185 3186 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 3187 3188 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) 3189 mtu = net->ipv6.sysctl.ip6_rt_min_advmss; 3190 3191 /* 3192 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and 3193 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size. 3194 * IPV6_MAXPLEN is also valid and means: "any MSS, 3195 * rely only on pmtu discovery" 3196 */ 3197 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr)) 3198 mtu = IPV6_MAXPLEN; 3199 return mtu; 3200 } 3201 3202 INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst) 3203 { 3204 return ip6_dst_mtu_maybe_forward(dst, false); 3205 } 3206 EXPORT_INDIRECT_CALLABLE(ip6_mtu); 3207 3208 /* MTU selection: 3209 * 1. mtu on route is locked - use it 3210 * 2. mtu from nexthop exception 3211 * 3. mtu from egress device 3212 * 3213 * based on ip6_dst_mtu_forward and exception logic of 3214 * rt6_find_cached_rt; called with rcu_read_lock 3215 */ 3216 u32 ip6_mtu_from_fib6(const struct fib6_result *res, 3217 const struct in6_addr *daddr, 3218 const struct in6_addr *saddr) 3219 { 3220 const struct fib6_nh *nh = res->nh; 3221 struct fib6_info *f6i = res->f6i; 3222 struct inet6_dev *idev; 3223 struct rt6_info *rt; 3224 u32 mtu = 0; 3225 3226 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) { 3227 mtu = f6i->fib6_pmtu; 3228 if (mtu) 3229 goto out; 3230 } 3231 3232 rt = rt6_find_cached_rt(res, daddr, saddr); 3233 if (unlikely(rt)) { 3234 mtu = dst_metric_raw(&rt->dst, RTAX_MTU); 3235 } else { 3236 struct net_device *dev = nh->fib_nh_dev; 3237 3238 mtu = IPV6_MIN_MTU; 3239 idev = __in6_dev_get(dev); 3240 if (idev && idev->cnf.mtu6 > mtu) 3241 mtu = idev->cnf.mtu6; 3242 } 3243 3244 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 3245 out: 3246 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 3247 } 3248 3249 struct dst_entry *icmp6_dst_alloc(struct net_device *dev, 3250 struct flowi6 *fl6) 3251 { 3252 struct dst_entry *dst; 3253 struct rt6_info *rt; 3254 struct inet6_dev *idev = in6_dev_get(dev); 3255 struct net *net = dev_net(dev); 3256 3257 if (unlikely(!idev)) 3258 return ERR_PTR(-ENODEV); 3259 3260 rt = ip6_dst_alloc(net, dev, 0); 3261 if (unlikely(!rt)) { 3262 in6_dev_put(idev); 3263 dst = ERR_PTR(-ENOMEM); 3264 goto out; 3265 } 3266 3267 rt->dst.input = ip6_input; 3268 rt->dst.output = ip6_output; 3269 rt->rt6i_gateway = fl6->daddr; 3270 rt->rt6i_dst.addr = fl6->daddr; 3271 rt->rt6i_dst.plen = 128; 3272 rt->rt6i_idev = idev; 3273 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0); 3274 3275 /* Add this dst into uncached_list so that rt6_disable_ip() can 3276 * do proper release of the net_device 3277 */ 3278 rt6_uncached_list_add(rt); 3279 atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache); 3280 3281 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0); 3282 3283 out: 3284 return dst; 3285 } 3286 3287 static int ip6_dst_gc(struct dst_ops *ops) 3288 { 3289 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); 3290 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; 3291 int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size; 3292 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; 3293 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; 3294 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; 3295 int entries; 3296 3297 entries = dst_entries_get_fast(ops); 3298 if (entries > rt_max_size) 3299 entries = dst_entries_get_slow(ops); 3300 3301 if (time_after(rt_last_gc + rt_min_interval, jiffies) && 3302 entries <= rt_max_size) 3303 goto out; 3304 3305 net->ipv6.ip6_rt_gc_expire++; 3306 fib6_run_gc(net->ipv6.ip6_rt_gc_expire, net, true); 3307 entries = dst_entries_get_slow(ops); 3308 if (entries < ops->gc_thresh) 3309 net->ipv6.ip6_rt_gc_expire = rt_gc_timeout>>1; 3310 out: 3311 net->ipv6.ip6_rt_gc_expire -= net->ipv6.ip6_rt_gc_expire>>rt_elasticity; 3312 return entries > rt_max_size; 3313 } 3314 3315 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg, 3316 const struct in6_addr *gw_addr, u32 tbid, 3317 int flags, struct fib6_result *res) 3318 { 3319 struct flowi6 fl6 = { 3320 .flowi6_oif = cfg->fc_ifindex, 3321 .daddr = *gw_addr, 3322 .saddr = cfg->fc_prefsrc, 3323 }; 3324 struct fib6_table *table; 3325 int err; 3326 3327 table = fib6_get_table(net, tbid); 3328 if (!table) 3329 return -EINVAL; 3330 3331 if (!ipv6_addr_any(&cfg->fc_prefsrc)) 3332 flags |= RT6_LOOKUP_F_HAS_SADDR; 3333 3334 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE; 3335 3336 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags); 3337 if (!err && res->f6i != net->ipv6.fib6_null_entry) 3338 fib6_select_path(net, res, &fl6, cfg->fc_ifindex, 3339 cfg->fc_ifindex != 0, NULL, flags); 3340 3341 return err; 3342 } 3343 3344 static int ip6_route_check_nh_onlink(struct net *net, 3345 struct fib6_config *cfg, 3346 const struct net_device *dev, 3347 struct netlink_ext_ack *extack) 3348 { 3349 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; 3350 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3351 struct fib6_result res = {}; 3352 int err; 3353 3354 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res); 3355 if (!err && !(res.fib6_flags & RTF_REJECT) && 3356 /* ignore match if it is the default route */ 3357 !ipv6_addr_any(&res.f6i->fib6_dst.addr) && 3358 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) { 3359 NL_SET_ERR_MSG(extack, 3360 "Nexthop has invalid gateway or device mismatch"); 3361 err = -EINVAL; 3362 } 3363 3364 return err; 3365 } 3366 3367 static int ip6_route_check_nh(struct net *net, 3368 struct fib6_config *cfg, 3369 struct net_device **_dev, 3370 struct inet6_dev **idev) 3371 { 3372 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3373 struct net_device *dev = _dev ? *_dev : NULL; 3374 int flags = RT6_LOOKUP_F_IFACE; 3375 struct fib6_result res = {}; 3376 int err = -EHOSTUNREACH; 3377 3378 if (cfg->fc_table) { 3379 err = ip6_nh_lookup_table(net, cfg, gw_addr, 3380 cfg->fc_table, flags, &res); 3381 /* gw_addr can not require a gateway or resolve to a reject 3382 * route. If a device is given, it must match the result. 3383 */ 3384 if (err || res.fib6_flags & RTF_REJECT || 3385 res.nh->fib_nh_gw_family || 3386 (dev && dev != res.nh->fib_nh_dev)) 3387 err = -EHOSTUNREACH; 3388 } 3389 3390 if (err < 0) { 3391 struct flowi6 fl6 = { 3392 .flowi6_oif = cfg->fc_ifindex, 3393 .daddr = *gw_addr, 3394 }; 3395 3396 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags); 3397 if (err || res.fib6_flags & RTF_REJECT || 3398 res.nh->fib_nh_gw_family) 3399 err = -EHOSTUNREACH; 3400 3401 if (err) 3402 return err; 3403 3404 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex, 3405 cfg->fc_ifindex != 0, NULL, flags); 3406 } 3407 3408 err = 0; 3409 if (dev) { 3410 if (dev != res.nh->fib_nh_dev) 3411 err = -EHOSTUNREACH; 3412 } else { 3413 *_dev = dev = res.nh->fib_nh_dev; 3414 dev_hold(dev); 3415 *idev = in6_dev_get(dev); 3416 } 3417 3418 return err; 3419 } 3420 3421 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg, 3422 struct net_device **_dev, struct inet6_dev **idev, 3423 struct netlink_ext_ack *extack) 3424 { 3425 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3426 int gwa_type = ipv6_addr_type(gw_addr); 3427 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true; 3428 const struct net_device *dev = *_dev; 3429 bool need_addr_check = !dev; 3430 int err = -EINVAL; 3431 3432 /* if gw_addr is local we will fail to detect this in case 3433 * address is still TENTATIVE (DAD in progress). rt6_lookup() 3434 * will return already-added prefix route via interface that 3435 * prefix route was assigned to, which might be non-loopback. 3436 */ 3437 if (dev && 3438 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3439 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3440 goto out; 3441 } 3442 3443 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) { 3444 /* IPv6 strictly inhibits using not link-local 3445 * addresses as nexthop address. 3446 * Otherwise, router will not able to send redirects. 3447 * It is very good, but in some (rare!) circumstances 3448 * (SIT, PtP, NBMA NOARP links) it is handy to allow 3449 * some exceptions. --ANK 3450 * We allow IPv4-mapped nexthops to support RFC4798-type 3451 * addressing 3452 */ 3453 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) { 3454 NL_SET_ERR_MSG(extack, "Invalid gateway address"); 3455 goto out; 3456 } 3457 3458 rcu_read_lock(); 3459 3460 if (cfg->fc_flags & RTNH_F_ONLINK) 3461 err = ip6_route_check_nh_onlink(net, cfg, dev, extack); 3462 else 3463 err = ip6_route_check_nh(net, cfg, _dev, idev); 3464 3465 rcu_read_unlock(); 3466 3467 if (err) 3468 goto out; 3469 } 3470 3471 /* reload in case device was changed */ 3472 dev = *_dev; 3473 3474 err = -EINVAL; 3475 if (!dev) { 3476 NL_SET_ERR_MSG(extack, "Egress device not specified"); 3477 goto out; 3478 } else if (dev->flags & IFF_LOOPBACK) { 3479 NL_SET_ERR_MSG(extack, 3480 "Egress device can not be loopback device for this route"); 3481 goto out; 3482 } 3483 3484 /* if we did not check gw_addr above, do so now that the 3485 * egress device has been resolved. 3486 */ 3487 if (need_addr_check && 3488 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3489 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3490 goto out; 3491 } 3492 3493 err = 0; 3494 out: 3495 return err; 3496 } 3497 3498 static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type) 3499 { 3500 if ((flags & RTF_REJECT) || 3501 (dev && (dev->flags & IFF_LOOPBACK) && 3502 !(addr_type & IPV6_ADDR_LOOPBACK) && 3503 !(flags & (RTF_ANYCAST | RTF_LOCAL)))) 3504 return true; 3505 3506 return false; 3507 } 3508 3509 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, 3510 struct fib6_config *cfg, gfp_t gfp_flags, 3511 struct netlink_ext_ack *extack) 3512 { 3513 struct net_device *dev = NULL; 3514 struct inet6_dev *idev = NULL; 3515 int addr_type; 3516 int err; 3517 3518 fib6_nh->fib_nh_family = AF_INET6; 3519 #ifdef CONFIG_IPV6_ROUTER_PREF 3520 fib6_nh->last_probe = jiffies; 3521 #endif 3522 if (cfg->fc_is_fdb) { 3523 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3524 fib6_nh->fib_nh_gw_family = AF_INET6; 3525 return 0; 3526 } 3527 3528 err = -ENODEV; 3529 if (cfg->fc_ifindex) { 3530 dev = dev_get_by_index(net, cfg->fc_ifindex); 3531 if (!dev) 3532 goto out; 3533 idev = in6_dev_get(dev); 3534 if (!idev) 3535 goto out; 3536 } 3537 3538 if (cfg->fc_flags & RTNH_F_ONLINK) { 3539 if (!dev) { 3540 NL_SET_ERR_MSG(extack, 3541 "Nexthop device required for onlink"); 3542 goto out; 3543 } 3544 3545 if (!(dev->flags & IFF_UP)) { 3546 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3547 err = -ENETDOWN; 3548 goto out; 3549 } 3550 3551 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK; 3552 } 3553 3554 fib6_nh->fib_nh_weight = 1; 3555 3556 /* We cannot add true routes via loopback here, 3557 * they would result in kernel looping; promote them to reject routes 3558 */ 3559 addr_type = ipv6_addr_type(&cfg->fc_dst); 3560 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { 3561 /* hold loopback dev/idev if we haven't done so. */ 3562 if (dev != net->loopback_dev) { 3563 if (dev) { 3564 dev_put(dev); 3565 in6_dev_put(idev); 3566 } 3567 dev = net->loopback_dev; 3568 dev_hold(dev); 3569 idev = in6_dev_get(dev); 3570 if (!idev) { 3571 err = -ENODEV; 3572 goto out; 3573 } 3574 } 3575 goto pcpu_alloc; 3576 } 3577 3578 if (cfg->fc_flags & RTF_GATEWAY) { 3579 err = ip6_validate_gw(net, cfg, &dev, &idev, extack); 3580 if (err) 3581 goto out; 3582 3583 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3584 fib6_nh->fib_nh_gw_family = AF_INET6; 3585 } 3586 3587 err = -ENODEV; 3588 if (!dev) 3589 goto out; 3590 3591 if (idev->cnf.disable_ipv6) { 3592 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device"); 3593 err = -EACCES; 3594 goto out; 3595 } 3596 3597 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) { 3598 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3599 err = -ENETDOWN; 3600 goto out; 3601 } 3602 3603 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) && 3604 !netif_carrier_ok(dev)) 3605 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 3606 3607 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap, 3608 cfg->fc_encap_type, cfg, gfp_flags, extack); 3609 if (err) 3610 goto out; 3611 3612 pcpu_alloc: 3613 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags); 3614 if (!fib6_nh->rt6i_pcpu) { 3615 err = -ENOMEM; 3616 goto out; 3617 } 3618 3619 fib6_nh->fib_nh_dev = dev; 3620 fib6_nh->fib_nh_oif = dev->ifindex; 3621 err = 0; 3622 out: 3623 if (idev) 3624 in6_dev_put(idev); 3625 3626 if (err) { 3627 lwtstate_put(fib6_nh->fib_nh_lws); 3628 fib6_nh->fib_nh_lws = NULL; 3629 if (dev) 3630 dev_put(dev); 3631 } 3632 3633 return err; 3634 } 3635 3636 void fib6_nh_release(struct fib6_nh *fib6_nh) 3637 { 3638 struct rt6_exception_bucket *bucket; 3639 3640 rcu_read_lock(); 3641 3642 fib6_nh_flush_exceptions(fib6_nh, NULL); 3643 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL); 3644 if (bucket) { 3645 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL); 3646 kfree(bucket); 3647 } 3648 3649 rcu_read_unlock(); 3650 3651 if (fib6_nh->rt6i_pcpu) { 3652 int cpu; 3653 3654 for_each_possible_cpu(cpu) { 3655 struct rt6_info **ppcpu_rt; 3656 struct rt6_info *pcpu_rt; 3657 3658 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); 3659 pcpu_rt = *ppcpu_rt; 3660 if (pcpu_rt) { 3661 dst_dev_put(&pcpu_rt->dst); 3662 dst_release(&pcpu_rt->dst); 3663 *ppcpu_rt = NULL; 3664 } 3665 } 3666 3667 free_percpu(fib6_nh->rt6i_pcpu); 3668 } 3669 3670 fib_nh_common_release(&fib6_nh->nh_common); 3671 } 3672 3673 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg, 3674 gfp_t gfp_flags, 3675 struct netlink_ext_ack *extack) 3676 { 3677 struct net *net = cfg->fc_nlinfo.nl_net; 3678 struct fib6_info *rt = NULL; 3679 struct nexthop *nh = NULL; 3680 struct fib6_table *table; 3681 struct fib6_nh *fib6_nh; 3682 int err = -EINVAL; 3683 int addr_type; 3684 3685 /* RTF_PCPU is an internal flag; can not be set by userspace */ 3686 if (cfg->fc_flags & RTF_PCPU) { 3687 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU"); 3688 goto out; 3689 } 3690 3691 /* RTF_CACHE is an internal flag; can not be set by userspace */ 3692 if (cfg->fc_flags & RTF_CACHE) { 3693 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE"); 3694 goto out; 3695 } 3696 3697 if (cfg->fc_type > RTN_MAX) { 3698 NL_SET_ERR_MSG(extack, "Invalid route type"); 3699 goto out; 3700 } 3701 3702 if (cfg->fc_dst_len > 128) { 3703 NL_SET_ERR_MSG(extack, "Invalid prefix length"); 3704 goto out; 3705 } 3706 if (cfg->fc_src_len > 128) { 3707 NL_SET_ERR_MSG(extack, "Invalid source address length"); 3708 goto out; 3709 } 3710 #ifndef CONFIG_IPV6_SUBTREES 3711 if (cfg->fc_src_len) { 3712 NL_SET_ERR_MSG(extack, 3713 "Specifying source address requires IPV6_SUBTREES to be enabled"); 3714 goto out; 3715 } 3716 #endif 3717 if (cfg->fc_nh_id) { 3718 nh = nexthop_find_by_id(net, cfg->fc_nh_id); 3719 if (!nh) { 3720 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 3721 goto out; 3722 } 3723 err = fib6_check_nexthop(nh, cfg, extack); 3724 if (err) 3725 goto out; 3726 } 3727 3728 err = -ENOBUFS; 3729 if (cfg->fc_nlinfo.nlh && 3730 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) { 3731 table = fib6_get_table(net, cfg->fc_table); 3732 if (!table) { 3733 pr_warn("NLM_F_CREATE should be specified when creating new route\n"); 3734 table = fib6_new_table(net, cfg->fc_table); 3735 } 3736 } else { 3737 table = fib6_new_table(net, cfg->fc_table); 3738 } 3739 3740 if (!table) 3741 goto out; 3742 3743 err = -ENOMEM; 3744 rt = fib6_info_alloc(gfp_flags, !nh); 3745 if (!rt) 3746 goto out; 3747 3748 rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len, 3749 extack); 3750 if (IS_ERR(rt->fib6_metrics)) { 3751 err = PTR_ERR(rt->fib6_metrics); 3752 /* Do not leave garbage there. */ 3753 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics; 3754 goto out_free; 3755 } 3756 3757 if (cfg->fc_flags & RTF_ADDRCONF) 3758 rt->dst_nocount = true; 3759 3760 if (cfg->fc_flags & RTF_EXPIRES) 3761 fib6_set_expires(rt, jiffies + 3762 clock_t_to_jiffies(cfg->fc_expires)); 3763 else 3764 fib6_clean_expires(rt); 3765 3766 if (cfg->fc_protocol == RTPROT_UNSPEC) 3767 cfg->fc_protocol = RTPROT_BOOT; 3768 rt->fib6_protocol = cfg->fc_protocol; 3769 3770 rt->fib6_table = table; 3771 rt->fib6_metric = cfg->fc_metric; 3772 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST; 3773 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY; 3774 3775 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len); 3776 rt->fib6_dst.plen = cfg->fc_dst_len; 3777 3778 #ifdef CONFIG_IPV6_SUBTREES 3779 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len); 3780 rt->fib6_src.plen = cfg->fc_src_len; 3781 #endif 3782 if (nh) { 3783 if (rt->fib6_src.plen) { 3784 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing"); 3785 goto out_free; 3786 } 3787 if (!nexthop_get(nh)) { 3788 NL_SET_ERR_MSG(extack, "Nexthop has been deleted"); 3789 goto out_free; 3790 } 3791 rt->nh = nh; 3792 fib6_nh = nexthop_fib6_nh(rt->nh); 3793 } else { 3794 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack); 3795 if (err) 3796 goto out; 3797 3798 fib6_nh = rt->fib6_nh; 3799 3800 /* We cannot add true routes via loopback here, they would 3801 * result in kernel looping; promote them to reject routes 3802 */ 3803 addr_type = ipv6_addr_type(&cfg->fc_dst); 3804 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev, 3805 addr_type)) 3806 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP; 3807 } 3808 3809 if (!ipv6_addr_any(&cfg->fc_prefsrc)) { 3810 struct net_device *dev = fib6_nh->fib_nh_dev; 3811 3812 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) { 3813 NL_SET_ERR_MSG(extack, "Invalid source address"); 3814 err = -EINVAL; 3815 goto out; 3816 } 3817 rt->fib6_prefsrc.addr = cfg->fc_prefsrc; 3818 rt->fib6_prefsrc.plen = 128; 3819 } else 3820 rt->fib6_prefsrc.plen = 0; 3821 3822 return rt; 3823 out: 3824 fib6_info_release(rt); 3825 return ERR_PTR(err); 3826 out_free: 3827 ip_fib_metrics_put(rt->fib6_metrics); 3828 kfree(rt); 3829 return ERR_PTR(err); 3830 } 3831 3832 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags, 3833 struct netlink_ext_ack *extack) 3834 { 3835 struct fib6_info *rt; 3836 int err; 3837 3838 rt = ip6_route_info_create(cfg, gfp_flags, extack); 3839 if (IS_ERR(rt)) 3840 return PTR_ERR(rt); 3841 3842 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack); 3843 fib6_info_release(rt); 3844 3845 return err; 3846 } 3847 3848 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info) 3849 { 3850 struct net *net = info->nl_net; 3851 struct fib6_table *table; 3852 int err; 3853 3854 if (rt == net->ipv6.fib6_null_entry) { 3855 err = -ENOENT; 3856 goto out; 3857 } 3858 3859 table = rt->fib6_table; 3860 spin_lock_bh(&table->tb6_lock); 3861 err = fib6_del(rt, info); 3862 spin_unlock_bh(&table->tb6_lock); 3863 3864 out: 3865 fib6_info_release(rt); 3866 return err; 3867 } 3868 3869 int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify) 3870 { 3871 struct nl_info info = { 3872 .nl_net = net, 3873 .skip_notify = skip_notify 3874 }; 3875 3876 return __ip6_del_rt(rt, &info); 3877 } 3878 3879 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) 3880 { 3881 struct nl_info *info = &cfg->fc_nlinfo; 3882 struct net *net = info->nl_net; 3883 struct sk_buff *skb = NULL; 3884 struct fib6_table *table; 3885 int err = -ENOENT; 3886 3887 if (rt == net->ipv6.fib6_null_entry) 3888 goto out_put; 3889 table = rt->fib6_table; 3890 spin_lock_bh(&table->tb6_lock); 3891 3892 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { 3893 struct fib6_info *sibling, *next_sibling; 3894 struct fib6_node *fn; 3895 3896 /* prefer to send a single notification with all hops */ 3897 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 3898 if (skb) { 3899 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 3900 3901 if (rt6_fill_node(net, skb, rt, NULL, 3902 NULL, NULL, 0, RTM_DELROUTE, 3903 info->portid, seq, 0) < 0) { 3904 kfree_skb(skb); 3905 skb = NULL; 3906 } else 3907 info->skip_notify = 1; 3908 } 3909 3910 /* 'rt' points to the first sibling route. If it is not the 3911 * leaf, then we do not need to send a notification. Otherwise, 3912 * we need to check if the last sibling has a next route or not 3913 * and emit a replace or delete notification, respectively. 3914 */ 3915 info->skip_notify_kernel = 1; 3916 fn = rcu_dereference_protected(rt->fib6_node, 3917 lockdep_is_held(&table->tb6_lock)); 3918 if (rcu_access_pointer(fn->leaf) == rt) { 3919 struct fib6_info *last_sibling, *replace_rt; 3920 3921 last_sibling = list_last_entry(&rt->fib6_siblings, 3922 struct fib6_info, 3923 fib6_siblings); 3924 replace_rt = rcu_dereference_protected( 3925 last_sibling->fib6_next, 3926 lockdep_is_held(&table->tb6_lock)); 3927 if (replace_rt) 3928 call_fib6_entry_notifiers_replace(net, 3929 replace_rt); 3930 else 3931 call_fib6_multipath_entry_notifiers(net, 3932 FIB_EVENT_ENTRY_DEL, 3933 rt, rt->fib6_nsiblings, 3934 NULL); 3935 } 3936 list_for_each_entry_safe(sibling, next_sibling, 3937 &rt->fib6_siblings, 3938 fib6_siblings) { 3939 err = fib6_del(sibling, info); 3940 if (err) 3941 goto out_unlock; 3942 } 3943 } 3944 3945 err = fib6_del(rt, info); 3946 out_unlock: 3947 spin_unlock_bh(&table->tb6_lock); 3948 out_put: 3949 fib6_info_release(rt); 3950 3951 if (skb) { 3952 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 3953 info->nlh, gfp_any()); 3954 } 3955 return err; 3956 } 3957 3958 static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg) 3959 { 3960 int rc = -ESRCH; 3961 3962 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex) 3963 goto out; 3964 3965 if (cfg->fc_flags & RTF_GATEWAY && 3966 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway)) 3967 goto out; 3968 3969 rc = rt6_remove_exception_rt(rt); 3970 out: 3971 return rc; 3972 } 3973 3974 static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt, 3975 struct fib6_nh *nh) 3976 { 3977 struct fib6_result res = { 3978 .f6i = rt, 3979 .nh = nh, 3980 }; 3981 struct rt6_info *rt_cache; 3982 3983 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src); 3984 if (rt_cache) 3985 return __ip6_del_cached_rt(rt_cache, cfg); 3986 3987 return 0; 3988 } 3989 3990 struct fib6_nh_del_cached_rt_arg { 3991 struct fib6_config *cfg; 3992 struct fib6_info *f6i; 3993 }; 3994 3995 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg) 3996 { 3997 struct fib6_nh_del_cached_rt_arg *arg = _arg; 3998 int rc; 3999 4000 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh); 4001 return rc != -ESRCH ? rc : 0; 4002 } 4003 4004 static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i) 4005 { 4006 struct fib6_nh_del_cached_rt_arg arg = { 4007 .cfg = cfg, 4008 .f6i = f6i 4009 }; 4010 4011 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg); 4012 } 4013 4014 static int ip6_route_del(struct fib6_config *cfg, 4015 struct netlink_ext_ack *extack) 4016 { 4017 struct fib6_table *table; 4018 struct fib6_info *rt; 4019 struct fib6_node *fn; 4020 int err = -ESRCH; 4021 4022 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table); 4023 if (!table) { 4024 NL_SET_ERR_MSG(extack, "FIB table does not exist"); 4025 return err; 4026 } 4027 4028 rcu_read_lock(); 4029 4030 fn = fib6_locate(&table->tb6_root, 4031 &cfg->fc_dst, cfg->fc_dst_len, 4032 &cfg->fc_src, cfg->fc_src_len, 4033 !(cfg->fc_flags & RTF_CACHE)); 4034 4035 if (fn) { 4036 for_each_fib6_node_rt_rcu(fn) { 4037 struct fib6_nh *nh; 4038 4039 if (rt->nh && cfg->fc_nh_id && 4040 rt->nh->id != cfg->fc_nh_id) 4041 continue; 4042 4043 if (cfg->fc_flags & RTF_CACHE) { 4044 int rc = 0; 4045 4046 if (rt->nh) { 4047 rc = ip6_del_cached_rt_nh(cfg, rt); 4048 } else if (cfg->fc_nh_id) { 4049 continue; 4050 } else { 4051 nh = rt->fib6_nh; 4052 rc = ip6_del_cached_rt(cfg, rt, nh); 4053 } 4054 if (rc != -ESRCH) { 4055 rcu_read_unlock(); 4056 return rc; 4057 } 4058 continue; 4059 } 4060 4061 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric) 4062 continue; 4063 if (cfg->fc_protocol && 4064 cfg->fc_protocol != rt->fib6_protocol) 4065 continue; 4066 4067 if (rt->nh) { 4068 if (!fib6_info_hold_safe(rt)) 4069 continue; 4070 rcu_read_unlock(); 4071 4072 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4073 } 4074 if (cfg->fc_nh_id) 4075 continue; 4076 4077 nh = rt->fib6_nh; 4078 if (cfg->fc_ifindex && 4079 (!nh->fib_nh_dev || 4080 nh->fib_nh_dev->ifindex != cfg->fc_ifindex)) 4081 continue; 4082 if (cfg->fc_flags & RTF_GATEWAY && 4083 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6)) 4084 continue; 4085 if (!fib6_info_hold_safe(rt)) 4086 continue; 4087 rcu_read_unlock(); 4088 4089 /* if gateway was specified only delete the one hop */ 4090 if (cfg->fc_flags & RTF_GATEWAY) 4091 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4092 4093 return __ip6_del_rt_siblings(rt, cfg); 4094 } 4095 } 4096 rcu_read_unlock(); 4097 4098 return err; 4099 } 4100 4101 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 4102 { 4103 struct netevent_redirect netevent; 4104 struct rt6_info *rt, *nrt = NULL; 4105 struct fib6_result res = {}; 4106 struct ndisc_options ndopts; 4107 struct inet6_dev *in6_dev; 4108 struct neighbour *neigh; 4109 struct rd_msg *msg; 4110 int optlen, on_link; 4111 u8 *lladdr; 4112 4113 optlen = skb_tail_pointer(skb) - skb_transport_header(skb); 4114 optlen -= sizeof(*msg); 4115 4116 if (optlen < 0) { 4117 net_dbg_ratelimited("rt6_do_redirect: packet too short\n"); 4118 return; 4119 } 4120 4121 msg = (struct rd_msg *)icmp6_hdr(skb); 4122 4123 if (ipv6_addr_is_multicast(&msg->dest)) { 4124 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n"); 4125 return; 4126 } 4127 4128 on_link = 0; 4129 if (ipv6_addr_equal(&msg->dest, &msg->target)) { 4130 on_link = 1; 4131 } else if (ipv6_addr_type(&msg->target) != 4132 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) { 4133 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n"); 4134 return; 4135 } 4136 4137 in6_dev = __in6_dev_get(skb->dev); 4138 if (!in6_dev) 4139 return; 4140 if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects) 4141 return; 4142 4143 /* RFC2461 8.1: 4144 * The IP source address of the Redirect MUST be the same as the current 4145 * first-hop router for the specified ICMP Destination Address. 4146 */ 4147 4148 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) { 4149 net_dbg_ratelimited("rt6_redirect: invalid ND options\n"); 4150 return; 4151 } 4152 4153 lladdr = NULL; 4154 if (ndopts.nd_opts_tgt_lladdr) { 4155 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, 4156 skb->dev); 4157 if (!lladdr) { 4158 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n"); 4159 return; 4160 } 4161 } 4162 4163 rt = (struct rt6_info *) dst; 4164 if (rt->rt6i_flags & RTF_REJECT) { 4165 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n"); 4166 return; 4167 } 4168 4169 /* Redirect received -> path was valid. 4170 * Look, redirects are sent only in response to data packets, 4171 * so that this nexthop apparently is reachable. --ANK 4172 */ 4173 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr); 4174 4175 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1); 4176 if (!neigh) 4177 return; 4178 4179 /* 4180 * We have finally decided to accept it. 4181 */ 4182 4183 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE, 4184 NEIGH_UPDATE_F_WEAK_OVERRIDE| 4185 NEIGH_UPDATE_F_OVERRIDE| 4186 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER| 4187 NEIGH_UPDATE_F_ISROUTER)), 4188 NDISC_REDIRECT, &ndopts); 4189 4190 rcu_read_lock(); 4191 res.f6i = rcu_dereference(rt->from); 4192 if (!res.f6i) 4193 goto out; 4194 4195 if (res.f6i->nh) { 4196 struct fib6_nh_match_arg arg = { 4197 .dev = dst->dev, 4198 .gw = &rt->rt6i_gateway, 4199 }; 4200 4201 nexthop_for_each_fib6_nh(res.f6i->nh, 4202 fib6_nh_find_match, &arg); 4203 4204 /* fib6_info uses a nexthop that does not have fib6_nh 4205 * using the dst->dev. Should be impossible 4206 */ 4207 if (!arg.match) 4208 goto out; 4209 res.nh = arg.match; 4210 } else { 4211 res.nh = res.f6i->fib6_nh; 4212 } 4213 4214 res.fib6_flags = res.f6i->fib6_flags; 4215 res.fib6_type = res.f6i->fib6_type; 4216 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL); 4217 if (!nrt) 4218 goto out; 4219 4220 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE; 4221 if (on_link) 4222 nrt->rt6i_flags &= ~RTF_GATEWAY; 4223 4224 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key; 4225 4226 /* rt6_insert_exception() will take care of duplicated exceptions */ 4227 if (rt6_insert_exception(nrt, &res)) { 4228 dst_release_immediate(&nrt->dst); 4229 goto out; 4230 } 4231 4232 netevent.old = &rt->dst; 4233 netevent.new = &nrt->dst; 4234 netevent.daddr = &msg->dest; 4235 netevent.neigh = neigh; 4236 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent); 4237 4238 out: 4239 rcu_read_unlock(); 4240 neigh_release(neigh); 4241 } 4242 4243 #ifdef CONFIG_IPV6_ROUTE_INFO 4244 static struct fib6_info *rt6_get_route_info(struct net *net, 4245 const struct in6_addr *prefix, int prefixlen, 4246 const struct in6_addr *gwaddr, 4247 struct net_device *dev) 4248 { 4249 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4250 int ifindex = dev->ifindex; 4251 struct fib6_node *fn; 4252 struct fib6_info *rt = NULL; 4253 struct fib6_table *table; 4254 4255 table = fib6_get_table(net, tb_id); 4256 if (!table) 4257 return NULL; 4258 4259 rcu_read_lock(); 4260 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true); 4261 if (!fn) 4262 goto out; 4263 4264 for_each_fib6_node_rt_rcu(fn) { 4265 /* these routes do not use nexthops */ 4266 if (rt->nh) 4267 continue; 4268 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex) 4269 continue; 4270 if (!(rt->fib6_flags & RTF_ROUTEINFO) || 4271 !rt->fib6_nh->fib_nh_gw_family) 4272 continue; 4273 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr)) 4274 continue; 4275 if (!fib6_info_hold_safe(rt)) 4276 continue; 4277 break; 4278 } 4279 out: 4280 rcu_read_unlock(); 4281 return rt; 4282 } 4283 4284 static struct fib6_info *rt6_add_route_info(struct net *net, 4285 const struct in6_addr *prefix, int prefixlen, 4286 const struct in6_addr *gwaddr, 4287 struct net_device *dev, 4288 unsigned int pref) 4289 { 4290 struct fib6_config cfg = { 4291 .fc_metric = IP6_RT_PRIO_USER, 4292 .fc_ifindex = dev->ifindex, 4293 .fc_dst_len = prefixlen, 4294 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO | 4295 RTF_UP | RTF_PREF(pref), 4296 .fc_protocol = RTPROT_RA, 4297 .fc_type = RTN_UNICAST, 4298 .fc_nlinfo.portid = 0, 4299 .fc_nlinfo.nlh = NULL, 4300 .fc_nlinfo.nl_net = net, 4301 }; 4302 4303 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4304 cfg.fc_dst = *prefix; 4305 cfg.fc_gateway = *gwaddr; 4306 4307 /* We should treat it as a default route if prefix length is 0. */ 4308 if (!prefixlen) 4309 cfg.fc_flags |= RTF_DEFAULT; 4310 4311 ip6_route_add(&cfg, GFP_ATOMIC, NULL); 4312 4313 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev); 4314 } 4315 #endif 4316 4317 struct fib6_info *rt6_get_dflt_router(struct net *net, 4318 const struct in6_addr *addr, 4319 struct net_device *dev) 4320 { 4321 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; 4322 struct fib6_info *rt; 4323 struct fib6_table *table; 4324 4325 table = fib6_get_table(net, tb_id); 4326 if (!table) 4327 return NULL; 4328 4329 rcu_read_lock(); 4330 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4331 struct fib6_nh *nh; 4332 4333 /* RA routes do not use nexthops */ 4334 if (rt->nh) 4335 continue; 4336 4337 nh = rt->fib6_nh; 4338 if (dev == nh->fib_nh_dev && 4339 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) && 4340 ipv6_addr_equal(&nh->fib_nh_gw6, addr)) 4341 break; 4342 } 4343 if (rt && !fib6_info_hold_safe(rt)) 4344 rt = NULL; 4345 rcu_read_unlock(); 4346 return rt; 4347 } 4348 4349 struct fib6_info *rt6_add_dflt_router(struct net *net, 4350 const struct in6_addr *gwaddr, 4351 struct net_device *dev, 4352 unsigned int pref, 4353 u32 defrtr_usr_metric) 4354 { 4355 struct fib6_config cfg = { 4356 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, 4357 .fc_metric = defrtr_usr_metric, 4358 .fc_ifindex = dev->ifindex, 4359 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | 4360 RTF_UP | RTF_EXPIRES | RTF_PREF(pref), 4361 .fc_protocol = RTPROT_RA, 4362 .fc_type = RTN_UNICAST, 4363 .fc_nlinfo.portid = 0, 4364 .fc_nlinfo.nlh = NULL, 4365 .fc_nlinfo.nl_net = net, 4366 }; 4367 4368 cfg.fc_gateway = *gwaddr; 4369 4370 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) { 4371 struct fib6_table *table; 4372 4373 table = fib6_get_table(dev_net(dev), cfg.fc_table); 4374 if (table) 4375 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER; 4376 } 4377 4378 return rt6_get_dflt_router(net, gwaddr, dev); 4379 } 4380 4381 static void __rt6_purge_dflt_routers(struct net *net, 4382 struct fib6_table *table) 4383 { 4384 struct fib6_info *rt; 4385 4386 restart: 4387 rcu_read_lock(); 4388 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4389 struct net_device *dev = fib6_info_nh_dev(rt); 4390 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL; 4391 4392 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) && 4393 (!idev || idev->cnf.accept_ra != 2) && 4394 fib6_info_hold_safe(rt)) { 4395 rcu_read_unlock(); 4396 ip6_del_rt(net, rt, false); 4397 goto restart; 4398 } 4399 } 4400 rcu_read_unlock(); 4401 4402 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; 4403 } 4404 4405 void rt6_purge_dflt_routers(struct net *net) 4406 { 4407 struct fib6_table *table; 4408 struct hlist_head *head; 4409 unsigned int h; 4410 4411 rcu_read_lock(); 4412 4413 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { 4414 head = &net->ipv6.fib_table_hash[h]; 4415 hlist_for_each_entry_rcu(table, head, tb6_hlist) { 4416 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) 4417 __rt6_purge_dflt_routers(net, table); 4418 } 4419 } 4420 4421 rcu_read_unlock(); 4422 } 4423 4424 static void rtmsg_to_fib6_config(struct net *net, 4425 struct in6_rtmsg *rtmsg, 4426 struct fib6_config *cfg) 4427 { 4428 *cfg = (struct fib6_config){ 4429 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ? 4430 : RT6_TABLE_MAIN, 4431 .fc_ifindex = rtmsg->rtmsg_ifindex, 4432 .fc_metric = rtmsg->rtmsg_metric ? : IP6_RT_PRIO_USER, 4433 .fc_expires = rtmsg->rtmsg_info, 4434 .fc_dst_len = rtmsg->rtmsg_dst_len, 4435 .fc_src_len = rtmsg->rtmsg_src_len, 4436 .fc_flags = rtmsg->rtmsg_flags, 4437 .fc_type = rtmsg->rtmsg_type, 4438 4439 .fc_nlinfo.nl_net = net, 4440 4441 .fc_dst = rtmsg->rtmsg_dst, 4442 .fc_src = rtmsg->rtmsg_src, 4443 .fc_gateway = rtmsg->rtmsg_gateway, 4444 }; 4445 } 4446 4447 int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg) 4448 { 4449 struct fib6_config cfg; 4450 int err; 4451 4452 if (cmd != SIOCADDRT && cmd != SIOCDELRT) 4453 return -EINVAL; 4454 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 4455 return -EPERM; 4456 4457 rtmsg_to_fib6_config(net, rtmsg, &cfg); 4458 4459 rtnl_lock(); 4460 switch (cmd) { 4461 case SIOCADDRT: 4462 err = ip6_route_add(&cfg, GFP_KERNEL, NULL); 4463 break; 4464 case SIOCDELRT: 4465 err = ip6_route_del(&cfg, NULL); 4466 break; 4467 } 4468 rtnl_unlock(); 4469 return err; 4470 } 4471 4472 /* 4473 * Drop the packet on the floor 4474 */ 4475 4476 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes) 4477 { 4478 struct dst_entry *dst = skb_dst(skb); 4479 struct net *net = dev_net(dst->dev); 4480 struct inet6_dev *idev; 4481 int type; 4482 4483 if (netif_is_l3_master(skb->dev) && 4484 dst->dev == net->loopback_dev) 4485 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 4486 else 4487 idev = ip6_dst_idev(dst); 4488 4489 switch (ipstats_mib_noroutes) { 4490 case IPSTATS_MIB_INNOROUTES: 4491 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr); 4492 if (type == IPV6_ADDR_ANY) { 4493 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 4494 break; 4495 } 4496 fallthrough; 4497 case IPSTATS_MIB_OUTNOROUTES: 4498 IP6_INC_STATS(net, idev, ipstats_mib_noroutes); 4499 break; 4500 } 4501 4502 /* Start over by dropping the dst for l3mdev case */ 4503 if (netif_is_l3_master(skb->dev)) 4504 skb_dst_drop(skb); 4505 4506 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0); 4507 kfree_skb(skb); 4508 return 0; 4509 } 4510 4511 static int ip6_pkt_discard(struct sk_buff *skb) 4512 { 4513 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES); 4514 } 4515 4516 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4517 { 4518 skb->dev = skb_dst(skb)->dev; 4519 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES); 4520 } 4521 4522 static int ip6_pkt_prohibit(struct sk_buff *skb) 4523 { 4524 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES); 4525 } 4526 4527 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4528 { 4529 skb->dev = skb_dst(skb)->dev; 4530 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES); 4531 } 4532 4533 /* 4534 * Allocate a dst for local (unicast / anycast) address. 4535 */ 4536 4537 struct fib6_info *addrconf_f6i_alloc(struct net *net, 4538 struct inet6_dev *idev, 4539 const struct in6_addr *addr, 4540 bool anycast, gfp_t gfp_flags) 4541 { 4542 struct fib6_config cfg = { 4543 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL, 4544 .fc_ifindex = idev->dev->ifindex, 4545 .fc_flags = RTF_UP | RTF_NONEXTHOP, 4546 .fc_dst = *addr, 4547 .fc_dst_len = 128, 4548 .fc_protocol = RTPROT_KERNEL, 4549 .fc_nlinfo.nl_net = net, 4550 .fc_ignore_dev_down = true, 4551 }; 4552 struct fib6_info *f6i; 4553 4554 if (anycast) { 4555 cfg.fc_type = RTN_ANYCAST; 4556 cfg.fc_flags |= RTF_ANYCAST; 4557 } else { 4558 cfg.fc_type = RTN_LOCAL; 4559 cfg.fc_flags |= RTF_LOCAL; 4560 } 4561 4562 f6i = ip6_route_info_create(&cfg, gfp_flags, NULL); 4563 if (!IS_ERR(f6i)) 4564 f6i->dst_nocount = true; 4565 return f6i; 4566 } 4567 4568 /* remove deleted ip from prefsrc entries */ 4569 struct arg_dev_net_ip { 4570 struct net_device *dev; 4571 struct net *net; 4572 struct in6_addr *addr; 4573 }; 4574 4575 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg) 4576 { 4577 struct net_device *dev = ((struct arg_dev_net_ip *)arg)->dev; 4578 struct net *net = ((struct arg_dev_net_ip *)arg)->net; 4579 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr; 4580 4581 if (!rt->nh && 4582 ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) && 4583 rt != net->ipv6.fib6_null_entry && 4584 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr)) { 4585 spin_lock_bh(&rt6_exception_lock); 4586 /* remove prefsrc entry */ 4587 rt->fib6_prefsrc.plen = 0; 4588 spin_unlock_bh(&rt6_exception_lock); 4589 } 4590 return 0; 4591 } 4592 4593 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp) 4594 { 4595 struct net *net = dev_net(ifp->idev->dev); 4596 struct arg_dev_net_ip adni = { 4597 .dev = ifp->idev->dev, 4598 .net = net, 4599 .addr = &ifp->addr, 4600 }; 4601 fib6_clean_all(net, fib6_remove_prefsrc, &adni); 4602 } 4603 4604 #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT) 4605 4606 /* Remove routers and update dst entries when gateway turn into host. */ 4607 static int fib6_clean_tohost(struct fib6_info *rt, void *arg) 4608 { 4609 struct in6_addr *gateway = (struct in6_addr *)arg; 4610 struct fib6_nh *nh; 4611 4612 /* RA routes do not use nexthops */ 4613 if (rt->nh) 4614 return 0; 4615 4616 nh = rt->fib6_nh; 4617 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) && 4618 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6)) 4619 return -1; 4620 4621 /* Further clean up cached routes in exception table. 4622 * This is needed because cached route may have a different 4623 * gateway than its 'parent' in the case of an ip redirect. 4624 */ 4625 fib6_nh_exceptions_clean_tohost(nh, gateway); 4626 4627 return 0; 4628 } 4629 4630 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway) 4631 { 4632 fib6_clean_all(net, fib6_clean_tohost, gateway); 4633 } 4634 4635 struct arg_netdev_event { 4636 const struct net_device *dev; 4637 union { 4638 unsigned char nh_flags; 4639 unsigned long event; 4640 }; 4641 }; 4642 4643 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt) 4644 { 4645 struct fib6_info *iter; 4646 struct fib6_node *fn; 4647 4648 fn = rcu_dereference_protected(rt->fib6_node, 4649 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4650 iter = rcu_dereference_protected(fn->leaf, 4651 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4652 while (iter) { 4653 if (iter->fib6_metric == rt->fib6_metric && 4654 rt6_qualify_for_ecmp(iter)) 4655 return iter; 4656 iter = rcu_dereference_protected(iter->fib6_next, 4657 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4658 } 4659 4660 return NULL; 4661 } 4662 4663 /* only called for fib entries with builtin fib6_nh */ 4664 static bool rt6_is_dead(const struct fib6_info *rt) 4665 { 4666 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD || 4667 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN && 4668 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev))) 4669 return true; 4670 4671 return false; 4672 } 4673 4674 static int rt6_multipath_total_weight(const struct fib6_info *rt) 4675 { 4676 struct fib6_info *iter; 4677 int total = 0; 4678 4679 if (!rt6_is_dead(rt)) 4680 total += rt->fib6_nh->fib_nh_weight; 4681 4682 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 4683 if (!rt6_is_dead(iter)) 4684 total += iter->fib6_nh->fib_nh_weight; 4685 } 4686 4687 return total; 4688 } 4689 4690 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) 4691 { 4692 int upper_bound = -1; 4693 4694 if (!rt6_is_dead(rt)) { 4695 *weight += rt->fib6_nh->fib_nh_weight; 4696 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, 4697 total) - 1; 4698 } 4699 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound); 4700 } 4701 4702 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total) 4703 { 4704 struct fib6_info *iter; 4705 int weight = 0; 4706 4707 rt6_upper_bound_set(rt, &weight, total); 4708 4709 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4710 rt6_upper_bound_set(iter, &weight, total); 4711 } 4712 4713 void rt6_multipath_rebalance(struct fib6_info *rt) 4714 { 4715 struct fib6_info *first; 4716 int total; 4717 4718 /* In case the entire multipath route was marked for flushing, 4719 * then there is no need to rebalance upon the removal of every 4720 * sibling route. 4721 */ 4722 if (!rt->fib6_nsiblings || rt->should_flush) 4723 return; 4724 4725 /* During lookup routes are evaluated in order, so we need to 4726 * make sure upper bounds are assigned from the first sibling 4727 * onwards. 4728 */ 4729 first = rt6_multipath_first_sibling(rt); 4730 if (WARN_ON_ONCE(!first)) 4731 return; 4732 4733 total = rt6_multipath_total_weight(first); 4734 rt6_multipath_upper_bound_set(first, total); 4735 } 4736 4737 static int fib6_ifup(struct fib6_info *rt, void *p_arg) 4738 { 4739 const struct arg_netdev_event *arg = p_arg; 4740 struct net *net = dev_net(arg->dev); 4741 4742 if (rt != net->ipv6.fib6_null_entry && !rt->nh && 4743 rt->fib6_nh->fib_nh_dev == arg->dev) { 4744 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags; 4745 fib6_update_sernum_upto_root(net, rt); 4746 rt6_multipath_rebalance(rt); 4747 } 4748 4749 return 0; 4750 } 4751 4752 void rt6_sync_up(struct net_device *dev, unsigned char nh_flags) 4753 { 4754 struct arg_netdev_event arg = { 4755 .dev = dev, 4756 { 4757 .nh_flags = nh_flags, 4758 }, 4759 }; 4760 4761 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev)) 4762 arg.nh_flags |= RTNH_F_LINKDOWN; 4763 4764 fib6_clean_all(dev_net(dev), fib6_ifup, &arg); 4765 } 4766 4767 /* only called for fib entries with inline fib6_nh */ 4768 static bool rt6_multipath_uses_dev(const struct fib6_info *rt, 4769 const struct net_device *dev) 4770 { 4771 struct fib6_info *iter; 4772 4773 if (rt->fib6_nh->fib_nh_dev == dev) 4774 return true; 4775 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4776 if (iter->fib6_nh->fib_nh_dev == dev) 4777 return true; 4778 4779 return false; 4780 } 4781 4782 static void rt6_multipath_flush(struct fib6_info *rt) 4783 { 4784 struct fib6_info *iter; 4785 4786 rt->should_flush = 1; 4787 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4788 iter->should_flush = 1; 4789 } 4790 4791 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt, 4792 const struct net_device *down_dev) 4793 { 4794 struct fib6_info *iter; 4795 unsigned int dead = 0; 4796 4797 if (rt->fib6_nh->fib_nh_dev == down_dev || 4798 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4799 dead++; 4800 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4801 if (iter->fib6_nh->fib_nh_dev == down_dev || 4802 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4803 dead++; 4804 4805 return dead; 4806 } 4807 4808 static void rt6_multipath_nh_flags_set(struct fib6_info *rt, 4809 const struct net_device *dev, 4810 unsigned char nh_flags) 4811 { 4812 struct fib6_info *iter; 4813 4814 if (rt->fib6_nh->fib_nh_dev == dev) 4815 rt->fib6_nh->fib_nh_flags |= nh_flags; 4816 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4817 if (iter->fib6_nh->fib_nh_dev == dev) 4818 iter->fib6_nh->fib_nh_flags |= nh_flags; 4819 } 4820 4821 /* called with write lock held for table with rt */ 4822 static int fib6_ifdown(struct fib6_info *rt, void *p_arg) 4823 { 4824 const struct arg_netdev_event *arg = p_arg; 4825 const struct net_device *dev = arg->dev; 4826 struct net *net = dev_net(dev); 4827 4828 if (rt == net->ipv6.fib6_null_entry || rt->nh) 4829 return 0; 4830 4831 switch (arg->event) { 4832 case NETDEV_UNREGISTER: 4833 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4834 case NETDEV_DOWN: 4835 if (rt->should_flush) 4836 return -1; 4837 if (!rt->fib6_nsiblings) 4838 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4839 if (rt6_multipath_uses_dev(rt, dev)) { 4840 unsigned int count; 4841 4842 count = rt6_multipath_dead_count(rt, dev); 4843 if (rt->fib6_nsiblings + 1 == count) { 4844 rt6_multipath_flush(rt); 4845 return -1; 4846 } 4847 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD | 4848 RTNH_F_LINKDOWN); 4849 fib6_update_sernum(net, rt); 4850 rt6_multipath_rebalance(rt); 4851 } 4852 return -2; 4853 case NETDEV_CHANGE: 4854 if (rt->fib6_nh->fib_nh_dev != dev || 4855 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 4856 break; 4857 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 4858 rt6_multipath_rebalance(rt); 4859 break; 4860 } 4861 4862 return 0; 4863 } 4864 4865 void rt6_sync_down_dev(struct net_device *dev, unsigned long event) 4866 { 4867 struct arg_netdev_event arg = { 4868 .dev = dev, 4869 { 4870 .event = event, 4871 }, 4872 }; 4873 struct net *net = dev_net(dev); 4874 4875 if (net->ipv6.sysctl.skip_notify_on_dev_down) 4876 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); 4877 else 4878 fib6_clean_all(net, fib6_ifdown, &arg); 4879 } 4880 4881 void rt6_disable_ip(struct net_device *dev, unsigned long event) 4882 { 4883 rt6_sync_down_dev(dev, event); 4884 rt6_uncached_list_flush_dev(dev_net(dev), dev); 4885 neigh_ifdown(&nd_tbl, dev); 4886 } 4887 4888 struct rt6_mtu_change_arg { 4889 struct net_device *dev; 4890 unsigned int mtu; 4891 struct fib6_info *f6i; 4892 }; 4893 4894 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) 4895 { 4896 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg; 4897 struct fib6_info *f6i = arg->f6i; 4898 4899 /* For administrative MTU increase, there is no way to discover 4900 * IPv6 PMTU increase, so PMTU increase should be updated here. 4901 * Since RFC 1981 doesn't include administrative MTU increase 4902 * update PMTU increase is a MUST. (i.e. jumbo frame) 4903 */ 4904 if (nh->fib_nh_dev == arg->dev) { 4905 struct inet6_dev *idev = __in6_dev_get(arg->dev); 4906 u32 mtu = f6i->fib6_pmtu; 4907 4908 if (mtu >= arg->mtu || 4909 (mtu < arg->mtu && mtu == idev->cnf.mtu6)) 4910 fib6_metric_set(f6i, RTAX_MTU, arg->mtu); 4911 4912 spin_lock_bh(&rt6_exception_lock); 4913 rt6_exceptions_update_pmtu(idev, nh, arg->mtu); 4914 spin_unlock_bh(&rt6_exception_lock); 4915 } 4916 4917 return 0; 4918 } 4919 4920 static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg) 4921 { 4922 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg; 4923 struct inet6_dev *idev; 4924 4925 /* In IPv6 pmtu discovery is not optional, 4926 so that RTAX_MTU lock cannot disable it. 4927 We still use this lock to block changes 4928 caused by addrconf/ndisc. 4929 */ 4930 4931 idev = __in6_dev_get(arg->dev); 4932 if (!idev) 4933 return 0; 4934 4935 if (fib6_metric_locked(f6i, RTAX_MTU)) 4936 return 0; 4937 4938 arg->f6i = f6i; 4939 if (f6i->nh) { 4940 /* fib6_nh_mtu_change only returns 0, so this is safe */ 4941 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change, 4942 arg); 4943 } 4944 4945 return fib6_nh_mtu_change(f6i->fib6_nh, arg); 4946 } 4947 4948 void rt6_mtu_change(struct net_device *dev, unsigned int mtu) 4949 { 4950 struct rt6_mtu_change_arg arg = { 4951 .dev = dev, 4952 .mtu = mtu, 4953 }; 4954 4955 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg); 4956 } 4957 4958 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = { 4959 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 }, 4960 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) }, 4961 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) }, 4962 [RTA_OIF] = { .type = NLA_U32 }, 4963 [RTA_IIF] = { .type = NLA_U32 }, 4964 [RTA_PRIORITY] = { .type = NLA_U32 }, 4965 [RTA_METRICS] = { .type = NLA_NESTED }, 4966 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, 4967 [RTA_PREF] = { .type = NLA_U8 }, 4968 [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, 4969 [RTA_ENCAP] = { .type = NLA_NESTED }, 4970 [RTA_EXPIRES] = { .type = NLA_U32 }, 4971 [RTA_UID] = { .type = NLA_U32 }, 4972 [RTA_MARK] = { .type = NLA_U32 }, 4973 [RTA_TABLE] = { .type = NLA_U32 }, 4974 [RTA_IP_PROTO] = { .type = NLA_U8 }, 4975 [RTA_SPORT] = { .type = NLA_U16 }, 4976 [RTA_DPORT] = { .type = NLA_U16 }, 4977 [RTA_NH_ID] = { .type = NLA_U32 }, 4978 }; 4979 4980 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, 4981 struct fib6_config *cfg, 4982 struct netlink_ext_ack *extack) 4983 { 4984 struct rtmsg *rtm; 4985 struct nlattr *tb[RTA_MAX+1]; 4986 unsigned int pref; 4987 int err; 4988 4989 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 4990 rtm_ipv6_policy, extack); 4991 if (err < 0) 4992 goto errout; 4993 4994 err = -EINVAL; 4995 rtm = nlmsg_data(nlh); 4996 4997 *cfg = (struct fib6_config){ 4998 .fc_table = rtm->rtm_table, 4999 .fc_dst_len = rtm->rtm_dst_len, 5000 .fc_src_len = rtm->rtm_src_len, 5001 .fc_flags = RTF_UP, 5002 .fc_protocol = rtm->rtm_protocol, 5003 .fc_type = rtm->rtm_type, 5004 5005 .fc_nlinfo.portid = NETLINK_CB(skb).portid, 5006 .fc_nlinfo.nlh = nlh, 5007 .fc_nlinfo.nl_net = sock_net(skb->sk), 5008 }; 5009 5010 if (rtm->rtm_type == RTN_UNREACHABLE || 5011 rtm->rtm_type == RTN_BLACKHOLE || 5012 rtm->rtm_type == RTN_PROHIBIT || 5013 rtm->rtm_type == RTN_THROW) 5014 cfg->fc_flags |= RTF_REJECT; 5015 5016 if (rtm->rtm_type == RTN_LOCAL) 5017 cfg->fc_flags |= RTF_LOCAL; 5018 5019 if (rtm->rtm_flags & RTM_F_CLONED) 5020 cfg->fc_flags |= RTF_CACHE; 5021 5022 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK); 5023 5024 if (tb[RTA_NH_ID]) { 5025 if (tb[RTA_GATEWAY] || tb[RTA_OIF] || 5026 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) { 5027 NL_SET_ERR_MSG(extack, 5028 "Nexthop specification and nexthop id are mutually exclusive"); 5029 goto errout; 5030 } 5031 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]); 5032 } 5033 5034 if (tb[RTA_GATEWAY]) { 5035 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]); 5036 cfg->fc_flags |= RTF_GATEWAY; 5037 } 5038 if (tb[RTA_VIA]) { 5039 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute"); 5040 goto errout; 5041 } 5042 5043 if (tb[RTA_DST]) { 5044 int plen = (rtm->rtm_dst_len + 7) >> 3; 5045 5046 if (nla_len(tb[RTA_DST]) < plen) 5047 goto errout; 5048 5049 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen); 5050 } 5051 5052 if (tb[RTA_SRC]) { 5053 int plen = (rtm->rtm_src_len + 7) >> 3; 5054 5055 if (nla_len(tb[RTA_SRC]) < plen) 5056 goto errout; 5057 5058 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen); 5059 } 5060 5061 if (tb[RTA_PREFSRC]) 5062 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]); 5063 5064 if (tb[RTA_OIF]) 5065 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]); 5066 5067 if (tb[RTA_PRIORITY]) 5068 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]); 5069 5070 if (tb[RTA_METRICS]) { 5071 cfg->fc_mx = nla_data(tb[RTA_METRICS]); 5072 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]); 5073 } 5074 5075 if (tb[RTA_TABLE]) 5076 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]); 5077 5078 if (tb[RTA_MULTIPATH]) { 5079 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]); 5080 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]); 5081 5082 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp, 5083 cfg->fc_mp_len, extack); 5084 if (err < 0) 5085 goto errout; 5086 } 5087 5088 if (tb[RTA_PREF]) { 5089 pref = nla_get_u8(tb[RTA_PREF]); 5090 if (pref != ICMPV6_ROUTER_PREF_LOW && 5091 pref != ICMPV6_ROUTER_PREF_HIGH) 5092 pref = ICMPV6_ROUTER_PREF_MEDIUM; 5093 cfg->fc_flags |= RTF_PREF(pref); 5094 } 5095 5096 if (tb[RTA_ENCAP]) 5097 cfg->fc_encap = tb[RTA_ENCAP]; 5098 5099 if (tb[RTA_ENCAP_TYPE]) { 5100 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]); 5101 5102 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack); 5103 if (err < 0) 5104 goto errout; 5105 } 5106 5107 if (tb[RTA_EXPIRES]) { 5108 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ); 5109 5110 if (addrconf_finite_timeout(timeout)) { 5111 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ); 5112 cfg->fc_flags |= RTF_EXPIRES; 5113 } 5114 } 5115 5116 err = 0; 5117 errout: 5118 return err; 5119 } 5120 5121 struct rt6_nh { 5122 struct fib6_info *fib6_info; 5123 struct fib6_config r_cfg; 5124 struct list_head next; 5125 }; 5126 5127 static int ip6_route_info_append(struct net *net, 5128 struct list_head *rt6_nh_list, 5129 struct fib6_info *rt, 5130 struct fib6_config *r_cfg) 5131 { 5132 struct rt6_nh *nh; 5133 int err = -EEXIST; 5134 5135 list_for_each_entry(nh, rt6_nh_list, next) { 5136 /* check if fib6_info already exists */ 5137 if (rt6_duplicate_nexthop(nh->fib6_info, rt)) 5138 return err; 5139 } 5140 5141 nh = kzalloc(sizeof(*nh), GFP_KERNEL); 5142 if (!nh) 5143 return -ENOMEM; 5144 nh->fib6_info = rt; 5145 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg)); 5146 list_add_tail(&nh->next, rt6_nh_list); 5147 5148 return 0; 5149 } 5150 5151 static void ip6_route_mpath_notify(struct fib6_info *rt, 5152 struct fib6_info *rt_last, 5153 struct nl_info *info, 5154 __u16 nlflags) 5155 { 5156 /* if this is an APPEND route, then rt points to the first route 5157 * inserted and rt_last points to last route inserted. Userspace 5158 * wants a consistent dump of the route which starts at the first 5159 * nexthop. Since sibling routes are always added at the end of 5160 * the list, find the first sibling of the last route appended 5161 */ 5162 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) { 5163 rt = list_first_entry(&rt_last->fib6_siblings, 5164 struct fib6_info, 5165 fib6_siblings); 5166 } 5167 5168 if (rt) 5169 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags); 5170 } 5171 5172 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt) 5173 { 5174 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt); 5175 bool should_notify = false; 5176 struct fib6_info *leaf; 5177 struct fib6_node *fn; 5178 5179 rcu_read_lock(); 5180 fn = rcu_dereference(rt->fib6_node); 5181 if (!fn) 5182 goto out; 5183 5184 leaf = rcu_dereference(fn->leaf); 5185 if (!leaf) 5186 goto out; 5187 5188 if (rt == leaf || 5189 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric && 5190 rt6_qualify_for_ecmp(leaf))) 5191 should_notify = true; 5192 out: 5193 rcu_read_unlock(); 5194 5195 return should_notify; 5196 } 5197 5198 static int ip6_route_multipath_add(struct fib6_config *cfg, 5199 struct netlink_ext_ack *extack) 5200 { 5201 struct fib6_info *rt_notif = NULL, *rt_last = NULL; 5202 struct nl_info *info = &cfg->fc_nlinfo; 5203 struct fib6_config r_cfg; 5204 struct rtnexthop *rtnh; 5205 struct fib6_info *rt; 5206 struct rt6_nh *err_nh; 5207 struct rt6_nh *nh, *nh_safe; 5208 __u16 nlflags; 5209 int remaining; 5210 int attrlen; 5211 int err = 1; 5212 int nhn = 0; 5213 int replace = (cfg->fc_nlinfo.nlh && 5214 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE)); 5215 LIST_HEAD(rt6_nh_list); 5216 5217 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE; 5218 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND) 5219 nlflags |= NLM_F_APPEND; 5220 5221 remaining = cfg->fc_mp_len; 5222 rtnh = (struct rtnexthop *)cfg->fc_mp; 5223 5224 /* Parse a Multipath Entry and build a list (rt6_nh_list) of 5225 * fib6_info structs per nexthop 5226 */ 5227 while (rtnh_ok(rtnh, remaining)) { 5228 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5229 if (rtnh->rtnh_ifindex) 5230 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5231 5232 attrlen = rtnh_attrlen(rtnh); 5233 if (attrlen > 0) { 5234 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5235 5236 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5237 if (nla) { 5238 r_cfg.fc_gateway = nla_get_in6_addr(nla); 5239 r_cfg.fc_flags |= RTF_GATEWAY; 5240 } 5241 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP); 5242 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); 5243 if (nla) 5244 r_cfg.fc_encap_type = nla_get_u16(nla); 5245 } 5246 5247 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK); 5248 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack); 5249 if (IS_ERR(rt)) { 5250 err = PTR_ERR(rt); 5251 rt = NULL; 5252 goto cleanup; 5253 } 5254 if (!rt6_qualify_for_ecmp(rt)) { 5255 err = -EINVAL; 5256 NL_SET_ERR_MSG(extack, 5257 "Device only routes can not be added for IPv6 using the multipath API."); 5258 fib6_info_release(rt); 5259 goto cleanup; 5260 } 5261 5262 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1; 5263 5264 err = ip6_route_info_append(info->nl_net, &rt6_nh_list, 5265 rt, &r_cfg); 5266 if (err) { 5267 fib6_info_release(rt); 5268 goto cleanup; 5269 } 5270 5271 rtnh = rtnh_next(rtnh, &remaining); 5272 } 5273 5274 if (list_empty(&rt6_nh_list)) { 5275 NL_SET_ERR_MSG(extack, 5276 "Invalid nexthop configuration - no valid nexthops"); 5277 return -EINVAL; 5278 } 5279 5280 /* for add and replace send one notification with all nexthops. 5281 * Skip the notification in fib6_add_rt2node and send one with 5282 * the full route when done 5283 */ 5284 info->skip_notify = 1; 5285 5286 /* For add and replace, send one notification with all nexthops. For 5287 * append, send one notification with all appended nexthops. 5288 */ 5289 info->skip_notify_kernel = 1; 5290 5291 err_nh = NULL; 5292 list_for_each_entry(nh, &rt6_nh_list, next) { 5293 err = __ip6_ins_rt(nh->fib6_info, info, extack); 5294 fib6_info_release(nh->fib6_info); 5295 5296 if (!err) { 5297 /* save reference to last route successfully inserted */ 5298 rt_last = nh->fib6_info; 5299 5300 /* save reference to first route for notification */ 5301 if (!rt_notif) 5302 rt_notif = nh->fib6_info; 5303 } 5304 5305 /* nh->fib6_info is used or freed at this point, reset to NULL*/ 5306 nh->fib6_info = NULL; 5307 if (err) { 5308 if (replace && nhn) 5309 NL_SET_ERR_MSG_MOD(extack, 5310 "multipath route replace failed (check consistency of installed routes)"); 5311 err_nh = nh; 5312 goto add_errout; 5313 } 5314 5315 /* Because each route is added like a single route we remove 5316 * these flags after the first nexthop: if there is a collision, 5317 * we have already failed to add the first nexthop: 5318 * fib6_add_rt2node() has rejected it; when replacing, old 5319 * nexthops have been replaced by first new, the rest should 5320 * be added to it. 5321 */ 5322 if (cfg->fc_nlinfo.nlh) { 5323 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL | 5324 NLM_F_REPLACE); 5325 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE; 5326 } 5327 nhn++; 5328 } 5329 5330 /* An in-kernel notification should only be sent in case the new 5331 * multipath route is added as the first route in the node, or if 5332 * it was appended to it. We pass 'rt_notif' since it is the first 5333 * sibling and might allow us to skip some checks in the replace case. 5334 */ 5335 if (ip6_route_mpath_should_notify(rt_notif)) { 5336 enum fib_event_type fib_event; 5337 5338 if (rt_notif->fib6_nsiblings != nhn - 1) 5339 fib_event = FIB_EVENT_ENTRY_APPEND; 5340 else 5341 fib_event = FIB_EVENT_ENTRY_REPLACE; 5342 5343 err = call_fib6_multipath_entry_notifiers(info->nl_net, 5344 fib_event, rt_notif, 5345 nhn - 1, extack); 5346 if (err) { 5347 /* Delete all the siblings that were just added */ 5348 err_nh = NULL; 5349 goto add_errout; 5350 } 5351 } 5352 5353 /* success ... tell user about new route */ 5354 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5355 goto cleanup; 5356 5357 add_errout: 5358 /* send notification for routes that were added so that 5359 * the delete notifications sent by ip6_route_del are 5360 * coherent 5361 */ 5362 if (rt_notif) 5363 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5364 5365 /* Delete routes that were already added */ 5366 list_for_each_entry(nh, &rt6_nh_list, next) { 5367 if (err_nh == nh) 5368 break; 5369 ip6_route_del(&nh->r_cfg, extack); 5370 } 5371 5372 cleanup: 5373 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) { 5374 if (nh->fib6_info) 5375 fib6_info_release(nh->fib6_info); 5376 list_del(&nh->next); 5377 kfree(nh); 5378 } 5379 5380 return err; 5381 } 5382 5383 static int ip6_route_multipath_del(struct fib6_config *cfg, 5384 struct netlink_ext_ack *extack) 5385 { 5386 struct fib6_config r_cfg; 5387 struct rtnexthop *rtnh; 5388 int last_err = 0; 5389 int remaining; 5390 int attrlen; 5391 int err; 5392 5393 remaining = cfg->fc_mp_len; 5394 rtnh = (struct rtnexthop *)cfg->fc_mp; 5395 5396 /* Parse a Multipath Entry */ 5397 while (rtnh_ok(rtnh, remaining)) { 5398 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5399 if (rtnh->rtnh_ifindex) 5400 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5401 5402 attrlen = rtnh_attrlen(rtnh); 5403 if (attrlen > 0) { 5404 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5405 5406 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5407 if (nla) { 5408 nla_memcpy(&r_cfg.fc_gateway, nla, 16); 5409 r_cfg.fc_flags |= RTF_GATEWAY; 5410 } 5411 } 5412 err = ip6_route_del(&r_cfg, extack); 5413 if (err) 5414 last_err = err; 5415 5416 rtnh = rtnh_next(rtnh, &remaining); 5417 } 5418 5419 return last_err; 5420 } 5421 5422 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5423 struct netlink_ext_ack *extack) 5424 { 5425 struct fib6_config cfg; 5426 int err; 5427 5428 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5429 if (err < 0) 5430 return err; 5431 5432 if (cfg.fc_nh_id && 5433 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) { 5434 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 5435 return -EINVAL; 5436 } 5437 5438 if (cfg.fc_mp) 5439 return ip6_route_multipath_del(&cfg, extack); 5440 else { 5441 cfg.fc_delete_all_nh = 1; 5442 return ip6_route_del(&cfg, extack); 5443 } 5444 } 5445 5446 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5447 struct netlink_ext_ack *extack) 5448 { 5449 struct fib6_config cfg; 5450 int err; 5451 5452 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5453 if (err < 0) 5454 return err; 5455 5456 if (cfg.fc_metric == 0) 5457 cfg.fc_metric = IP6_RT_PRIO_USER; 5458 5459 if (cfg.fc_mp) 5460 return ip6_route_multipath_add(&cfg, extack); 5461 else 5462 return ip6_route_add(&cfg, GFP_KERNEL, extack); 5463 } 5464 5465 /* add the overhead of this fib6_nh to nexthop_len */ 5466 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg) 5467 { 5468 int *nexthop_len = arg; 5469 5470 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */ 5471 + NLA_ALIGN(sizeof(struct rtnexthop)) 5472 + nla_total_size(16); /* RTA_GATEWAY */ 5473 5474 if (nh->fib_nh_lws) { 5475 /* RTA_ENCAP_TYPE */ 5476 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5477 /* RTA_ENCAP */ 5478 *nexthop_len += nla_total_size(2); 5479 } 5480 5481 return 0; 5482 } 5483 5484 static size_t rt6_nlmsg_size(struct fib6_info *f6i) 5485 { 5486 int nexthop_len; 5487 5488 if (f6i->nh) { 5489 nexthop_len = nla_total_size(4); /* RTA_NH_ID */ 5490 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size, 5491 &nexthop_len); 5492 } else { 5493 struct fib6_nh *nh = f6i->fib6_nh; 5494 5495 nexthop_len = 0; 5496 if (f6i->fib6_nsiblings) { 5497 nexthop_len = nla_total_size(0) /* RTA_MULTIPATH */ 5498 + NLA_ALIGN(sizeof(struct rtnexthop)) 5499 + nla_total_size(16) /* RTA_GATEWAY */ 5500 + lwtunnel_get_encap_size(nh->fib_nh_lws); 5501 5502 nexthop_len *= f6i->fib6_nsiblings; 5503 } 5504 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5505 } 5506 5507 return NLMSG_ALIGN(sizeof(struct rtmsg)) 5508 + nla_total_size(16) /* RTA_SRC */ 5509 + nla_total_size(16) /* RTA_DST */ 5510 + nla_total_size(16) /* RTA_GATEWAY */ 5511 + nla_total_size(16) /* RTA_PREFSRC */ 5512 + nla_total_size(4) /* RTA_TABLE */ 5513 + nla_total_size(4) /* RTA_IIF */ 5514 + nla_total_size(4) /* RTA_OIF */ 5515 + nla_total_size(4) /* RTA_PRIORITY */ 5516 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */ 5517 + nla_total_size(sizeof(struct rta_cacheinfo)) 5518 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */ 5519 + nla_total_size(1) /* RTA_PREF */ 5520 + nexthop_len; 5521 } 5522 5523 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh, 5524 unsigned char *flags) 5525 { 5526 if (nexthop_is_multipath(nh)) { 5527 struct nlattr *mp; 5528 5529 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5530 if (!mp) 5531 goto nla_put_failure; 5532 5533 if (nexthop_mpath_fill_node(skb, nh, AF_INET6)) 5534 goto nla_put_failure; 5535 5536 nla_nest_end(skb, mp); 5537 } else { 5538 struct fib6_nh *fib6_nh; 5539 5540 fib6_nh = nexthop_fib6_nh(nh); 5541 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6, 5542 flags, false) < 0) 5543 goto nla_put_failure; 5544 } 5545 5546 return 0; 5547 5548 nla_put_failure: 5549 return -EMSGSIZE; 5550 } 5551 5552 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 5553 struct fib6_info *rt, struct dst_entry *dst, 5554 struct in6_addr *dest, struct in6_addr *src, 5555 int iif, int type, u32 portid, u32 seq, 5556 unsigned int flags) 5557 { 5558 struct rt6_info *rt6 = (struct rt6_info *)dst; 5559 struct rt6key *rt6_dst, *rt6_src; 5560 u32 *pmetrics, table, rt6_flags; 5561 unsigned char nh_flags = 0; 5562 struct nlmsghdr *nlh; 5563 struct rtmsg *rtm; 5564 long expires = 0; 5565 5566 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags); 5567 if (!nlh) 5568 return -EMSGSIZE; 5569 5570 if (rt6) { 5571 rt6_dst = &rt6->rt6i_dst; 5572 rt6_src = &rt6->rt6i_src; 5573 rt6_flags = rt6->rt6i_flags; 5574 } else { 5575 rt6_dst = &rt->fib6_dst; 5576 rt6_src = &rt->fib6_src; 5577 rt6_flags = rt->fib6_flags; 5578 } 5579 5580 rtm = nlmsg_data(nlh); 5581 rtm->rtm_family = AF_INET6; 5582 rtm->rtm_dst_len = rt6_dst->plen; 5583 rtm->rtm_src_len = rt6_src->plen; 5584 rtm->rtm_tos = 0; 5585 if (rt->fib6_table) 5586 table = rt->fib6_table->tb6_id; 5587 else 5588 table = RT6_TABLE_UNSPEC; 5589 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT; 5590 if (nla_put_u32(skb, RTA_TABLE, table)) 5591 goto nla_put_failure; 5592 5593 rtm->rtm_type = rt->fib6_type; 5594 rtm->rtm_flags = 0; 5595 rtm->rtm_scope = RT_SCOPE_UNIVERSE; 5596 rtm->rtm_protocol = rt->fib6_protocol; 5597 5598 if (rt6_flags & RTF_CACHE) 5599 rtm->rtm_flags |= RTM_F_CLONED; 5600 5601 if (dest) { 5602 if (nla_put_in6_addr(skb, RTA_DST, dest)) 5603 goto nla_put_failure; 5604 rtm->rtm_dst_len = 128; 5605 } else if (rtm->rtm_dst_len) 5606 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr)) 5607 goto nla_put_failure; 5608 #ifdef CONFIG_IPV6_SUBTREES 5609 if (src) { 5610 if (nla_put_in6_addr(skb, RTA_SRC, src)) 5611 goto nla_put_failure; 5612 rtm->rtm_src_len = 128; 5613 } else if (rtm->rtm_src_len && 5614 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr)) 5615 goto nla_put_failure; 5616 #endif 5617 if (iif) { 5618 #ifdef CONFIG_IPV6_MROUTE 5619 if (ipv6_addr_is_multicast(&rt6_dst->addr)) { 5620 int err = ip6mr_get_route(net, skb, rtm, portid); 5621 5622 if (err == 0) 5623 return 0; 5624 if (err < 0) 5625 goto nla_put_failure; 5626 } else 5627 #endif 5628 if (nla_put_u32(skb, RTA_IIF, iif)) 5629 goto nla_put_failure; 5630 } else if (dest) { 5631 struct in6_addr saddr_buf; 5632 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 && 5633 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5634 goto nla_put_failure; 5635 } 5636 5637 if (rt->fib6_prefsrc.plen) { 5638 struct in6_addr saddr_buf; 5639 saddr_buf = rt->fib6_prefsrc.addr; 5640 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5641 goto nla_put_failure; 5642 } 5643 5644 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics; 5645 if (rtnetlink_put_metrics(skb, pmetrics) < 0) 5646 goto nla_put_failure; 5647 5648 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric)) 5649 goto nla_put_failure; 5650 5651 /* For multipath routes, walk the siblings list and add 5652 * each as a nexthop within RTA_MULTIPATH. 5653 */ 5654 if (rt6) { 5655 if (rt6_flags & RTF_GATEWAY && 5656 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway)) 5657 goto nla_put_failure; 5658 5659 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex)) 5660 goto nla_put_failure; 5661 5662 if (dst->lwtstate && 5663 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 5664 goto nla_put_failure; 5665 } else if (rt->fib6_nsiblings) { 5666 struct fib6_info *sibling, *next_sibling; 5667 struct nlattr *mp; 5668 5669 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5670 if (!mp) 5671 goto nla_put_failure; 5672 5673 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common, 5674 rt->fib6_nh->fib_nh_weight, AF_INET6) < 0) 5675 goto nla_put_failure; 5676 5677 list_for_each_entry_safe(sibling, next_sibling, 5678 &rt->fib6_siblings, fib6_siblings) { 5679 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common, 5680 sibling->fib6_nh->fib_nh_weight, 5681 AF_INET6) < 0) 5682 goto nla_put_failure; 5683 } 5684 5685 nla_nest_end(skb, mp); 5686 } else if (rt->nh) { 5687 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id)) 5688 goto nla_put_failure; 5689 5690 if (nexthop_is_blackhole(rt->nh)) 5691 rtm->rtm_type = RTN_BLACKHOLE; 5692 5693 if (net->ipv4.sysctl_nexthop_compat_mode && 5694 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0) 5695 goto nla_put_failure; 5696 5697 rtm->rtm_flags |= nh_flags; 5698 } else { 5699 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6, 5700 &nh_flags, false) < 0) 5701 goto nla_put_failure; 5702 5703 rtm->rtm_flags |= nh_flags; 5704 } 5705 5706 if (rt6_flags & RTF_EXPIRES) { 5707 expires = dst ? dst->expires : rt->expires; 5708 expires -= jiffies; 5709 } 5710 5711 if (!dst) { 5712 if (rt->offload) 5713 rtm->rtm_flags |= RTM_F_OFFLOAD; 5714 if (rt->trap) 5715 rtm->rtm_flags |= RTM_F_TRAP; 5716 if (rt->offload_failed) 5717 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED; 5718 } 5719 5720 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0) 5721 goto nla_put_failure; 5722 5723 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags))) 5724 goto nla_put_failure; 5725 5726 5727 nlmsg_end(skb, nlh); 5728 return 0; 5729 5730 nla_put_failure: 5731 nlmsg_cancel(skb, nlh); 5732 return -EMSGSIZE; 5733 } 5734 5735 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg) 5736 { 5737 const struct net_device *dev = arg; 5738 5739 if (nh->fib_nh_dev == dev) 5740 return 1; 5741 5742 return 0; 5743 } 5744 5745 static bool fib6_info_uses_dev(const struct fib6_info *f6i, 5746 const struct net_device *dev) 5747 { 5748 if (f6i->nh) { 5749 struct net_device *_dev = (struct net_device *)dev; 5750 5751 return !!nexthop_for_each_fib6_nh(f6i->nh, 5752 fib6_info_nh_uses_dev, 5753 _dev); 5754 } 5755 5756 if (f6i->fib6_nh->fib_nh_dev == dev) 5757 return true; 5758 5759 if (f6i->fib6_nsiblings) { 5760 struct fib6_info *sibling, *next_sibling; 5761 5762 list_for_each_entry_safe(sibling, next_sibling, 5763 &f6i->fib6_siblings, fib6_siblings) { 5764 if (sibling->fib6_nh->fib_nh_dev == dev) 5765 return true; 5766 } 5767 } 5768 5769 return false; 5770 } 5771 5772 struct fib6_nh_exception_dump_walker { 5773 struct rt6_rtnl_dump_arg *dump; 5774 struct fib6_info *rt; 5775 unsigned int flags; 5776 unsigned int skip; 5777 unsigned int count; 5778 }; 5779 5780 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) 5781 { 5782 struct fib6_nh_exception_dump_walker *w = arg; 5783 struct rt6_rtnl_dump_arg *dump = w->dump; 5784 struct rt6_exception_bucket *bucket; 5785 struct rt6_exception *rt6_ex; 5786 int i, err; 5787 5788 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 5789 if (!bucket) 5790 return 0; 5791 5792 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 5793 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 5794 if (w->skip) { 5795 w->skip--; 5796 continue; 5797 } 5798 5799 /* Expiration of entries doesn't bump sernum, insertion 5800 * does. Removal is triggered by insertion, so we can 5801 * rely on the fact that if entries change between two 5802 * partial dumps, this node is scanned again completely, 5803 * see rt6_insert_exception() and fib6_dump_table(). 5804 * 5805 * Count expired entries we go through as handled 5806 * entries that we'll skip next time, in case of partial 5807 * node dump. Otherwise, if entries expire meanwhile, 5808 * we'll skip the wrong amount. 5809 */ 5810 if (rt6_check_expired(rt6_ex->rt6i)) { 5811 w->count++; 5812 continue; 5813 } 5814 5815 err = rt6_fill_node(dump->net, dump->skb, w->rt, 5816 &rt6_ex->rt6i->dst, NULL, NULL, 0, 5817 RTM_NEWROUTE, 5818 NETLINK_CB(dump->cb->skb).portid, 5819 dump->cb->nlh->nlmsg_seq, w->flags); 5820 if (err) 5821 return err; 5822 5823 w->count++; 5824 } 5825 bucket++; 5826 } 5827 5828 return 0; 5829 } 5830 5831 /* Return -1 if done with node, number of handled routes on partial dump */ 5832 int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip) 5833 { 5834 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg; 5835 struct fib_dump_filter *filter = &arg->filter; 5836 unsigned int flags = NLM_F_MULTI; 5837 struct net *net = arg->net; 5838 int count = 0; 5839 5840 if (rt == net->ipv6.fib6_null_entry) 5841 return -1; 5842 5843 if ((filter->flags & RTM_F_PREFIX) && 5844 !(rt->fib6_flags & RTF_PREFIX_RT)) { 5845 /* success since this is not a prefix route */ 5846 return -1; 5847 } 5848 if (filter->filter_set && 5849 ((filter->rt_type && rt->fib6_type != filter->rt_type) || 5850 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) || 5851 (filter->protocol && rt->fib6_protocol != filter->protocol))) { 5852 return -1; 5853 } 5854 5855 if (filter->filter_set || 5856 !filter->dump_routes || !filter->dump_exceptions) { 5857 flags |= NLM_F_DUMP_FILTERED; 5858 } 5859 5860 if (filter->dump_routes) { 5861 if (skip) { 5862 skip--; 5863 } else { 5864 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 5865 0, RTM_NEWROUTE, 5866 NETLINK_CB(arg->cb->skb).portid, 5867 arg->cb->nlh->nlmsg_seq, flags)) { 5868 return 0; 5869 } 5870 count++; 5871 } 5872 } 5873 5874 if (filter->dump_exceptions) { 5875 struct fib6_nh_exception_dump_walker w = { .dump = arg, 5876 .rt = rt, 5877 .flags = flags, 5878 .skip = skip, 5879 .count = 0 }; 5880 int err; 5881 5882 rcu_read_lock(); 5883 if (rt->nh) { 5884 err = nexthop_for_each_fib6_nh(rt->nh, 5885 rt6_nh_dump_exceptions, 5886 &w); 5887 } else { 5888 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w); 5889 } 5890 rcu_read_unlock(); 5891 5892 if (err) 5893 return count += w.count; 5894 } 5895 5896 return -1; 5897 } 5898 5899 static int inet6_rtm_valid_getroute_req(struct sk_buff *skb, 5900 const struct nlmsghdr *nlh, 5901 struct nlattr **tb, 5902 struct netlink_ext_ack *extack) 5903 { 5904 struct rtmsg *rtm; 5905 int i, err; 5906 5907 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 5908 NL_SET_ERR_MSG_MOD(extack, 5909 "Invalid header for get route request"); 5910 return -EINVAL; 5911 } 5912 5913 if (!netlink_strict_get_check(skb)) 5914 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5915 rtm_ipv6_policy, extack); 5916 5917 rtm = nlmsg_data(nlh); 5918 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) || 5919 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) || 5920 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope || 5921 rtm->rtm_type) { 5922 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request"); 5923 return -EINVAL; 5924 } 5925 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) { 5926 NL_SET_ERR_MSG_MOD(extack, 5927 "Invalid flags for get route request"); 5928 return -EINVAL; 5929 } 5930 5931 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 5932 rtm_ipv6_policy, extack); 5933 if (err) 5934 return err; 5935 5936 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 5937 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 5938 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6"); 5939 return -EINVAL; 5940 } 5941 5942 for (i = 0; i <= RTA_MAX; i++) { 5943 if (!tb[i]) 5944 continue; 5945 5946 switch (i) { 5947 case RTA_SRC: 5948 case RTA_DST: 5949 case RTA_IIF: 5950 case RTA_OIF: 5951 case RTA_MARK: 5952 case RTA_UID: 5953 case RTA_SPORT: 5954 case RTA_DPORT: 5955 case RTA_IP_PROTO: 5956 break; 5957 default: 5958 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request"); 5959 return -EINVAL; 5960 } 5961 } 5962 5963 return 0; 5964 } 5965 5966 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 5967 struct netlink_ext_ack *extack) 5968 { 5969 struct net *net = sock_net(in_skb->sk); 5970 struct nlattr *tb[RTA_MAX+1]; 5971 int err, iif = 0, oif = 0; 5972 struct fib6_info *from; 5973 struct dst_entry *dst; 5974 struct rt6_info *rt; 5975 struct sk_buff *skb; 5976 struct rtmsg *rtm; 5977 struct flowi6 fl6 = {}; 5978 bool fibmatch; 5979 5980 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 5981 if (err < 0) 5982 goto errout; 5983 5984 err = -EINVAL; 5985 rtm = nlmsg_data(nlh); 5986 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0); 5987 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH); 5988 5989 if (tb[RTA_SRC]) { 5990 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr)) 5991 goto errout; 5992 5993 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]); 5994 } 5995 5996 if (tb[RTA_DST]) { 5997 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr)) 5998 goto errout; 5999 6000 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]); 6001 } 6002 6003 if (tb[RTA_IIF]) 6004 iif = nla_get_u32(tb[RTA_IIF]); 6005 6006 if (tb[RTA_OIF]) 6007 oif = nla_get_u32(tb[RTA_OIF]); 6008 6009 if (tb[RTA_MARK]) 6010 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]); 6011 6012 if (tb[RTA_UID]) 6013 fl6.flowi6_uid = make_kuid(current_user_ns(), 6014 nla_get_u32(tb[RTA_UID])); 6015 else 6016 fl6.flowi6_uid = iif ? INVALID_UID : current_uid(); 6017 6018 if (tb[RTA_SPORT]) 6019 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]); 6020 6021 if (tb[RTA_DPORT]) 6022 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]); 6023 6024 if (tb[RTA_IP_PROTO]) { 6025 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 6026 &fl6.flowi6_proto, AF_INET6, 6027 extack); 6028 if (err) 6029 goto errout; 6030 } 6031 6032 if (iif) { 6033 struct net_device *dev; 6034 int flags = 0; 6035 6036 rcu_read_lock(); 6037 6038 dev = dev_get_by_index_rcu(net, iif); 6039 if (!dev) { 6040 rcu_read_unlock(); 6041 err = -ENODEV; 6042 goto errout; 6043 } 6044 6045 fl6.flowi6_iif = iif; 6046 6047 if (!ipv6_addr_any(&fl6.saddr)) 6048 flags |= RT6_LOOKUP_F_HAS_SADDR; 6049 6050 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags); 6051 6052 rcu_read_unlock(); 6053 } else { 6054 fl6.flowi6_oif = oif; 6055 6056 dst = ip6_route_output(net, NULL, &fl6); 6057 } 6058 6059 6060 rt = container_of(dst, struct rt6_info, dst); 6061 if (rt->dst.error) { 6062 err = rt->dst.error; 6063 ip6_rt_put(rt); 6064 goto errout; 6065 } 6066 6067 if (rt == net->ipv6.ip6_null_entry) { 6068 err = rt->dst.error; 6069 ip6_rt_put(rt); 6070 goto errout; 6071 } 6072 6073 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 6074 if (!skb) { 6075 ip6_rt_put(rt); 6076 err = -ENOBUFS; 6077 goto errout; 6078 } 6079 6080 skb_dst_set(skb, &rt->dst); 6081 6082 rcu_read_lock(); 6083 from = rcu_dereference(rt->from); 6084 if (from) { 6085 if (fibmatch) 6086 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, 6087 iif, RTM_NEWROUTE, 6088 NETLINK_CB(in_skb).portid, 6089 nlh->nlmsg_seq, 0); 6090 else 6091 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr, 6092 &fl6.saddr, iif, RTM_NEWROUTE, 6093 NETLINK_CB(in_skb).portid, 6094 nlh->nlmsg_seq, 0); 6095 } else { 6096 err = -ENETUNREACH; 6097 } 6098 rcu_read_unlock(); 6099 6100 if (err < 0) { 6101 kfree_skb(skb); 6102 goto errout; 6103 } 6104 6105 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 6106 errout: 6107 return err; 6108 } 6109 6110 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, 6111 unsigned int nlm_flags) 6112 { 6113 struct sk_buff *skb; 6114 struct net *net = info->nl_net; 6115 u32 seq; 6116 int err; 6117 6118 err = -ENOBUFS; 6119 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6120 6121 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6122 if (!skb) 6123 goto errout; 6124 6125 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6126 event, info->portid, seq, nlm_flags); 6127 if (err < 0) { 6128 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6129 WARN_ON(err == -EMSGSIZE); 6130 kfree_skb(skb); 6131 goto errout; 6132 } 6133 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6134 info->nlh, gfp_any()); 6135 return; 6136 errout: 6137 if (err < 0) 6138 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6139 } 6140 6141 void fib6_rt_update(struct net *net, struct fib6_info *rt, 6142 struct nl_info *info) 6143 { 6144 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6145 struct sk_buff *skb; 6146 int err = -ENOBUFS; 6147 6148 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6149 if (!skb) 6150 goto errout; 6151 6152 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6153 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE); 6154 if (err < 0) { 6155 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6156 WARN_ON(err == -EMSGSIZE); 6157 kfree_skb(skb); 6158 goto errout; 6159 } 6160 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6161 info->nlh, gfp_any()); 6162 return; 6163 errout: 6164 if (err < 0) 6165 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6166 } 6167 6168 void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, 6169 bool offload, bool trap, bool offload_failed) 6170 { 6171 struct sk_buff *skb; 6172 int err; 6173 6174 if (f6i->offload == offload && f6i->trap == trap && 6175 f6i->offload_failed == offload_failed) 6176 return; 6177 6178 f6i->offload = offload; 6179 f6i->trap = trap; 6180 6181 /* 2 means send notifications only if offload_failed was changed. */ 6182 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && 6183 f6i->offload_failed == offload_failed) 6184 return; 6185 6186 f6i->offload_failed = offload_failed; 6187 6188 if (!rcu_access_pointer(f6i->fib6_node)) 6189 /* The route was removed from the tree, do not send 6190 * notification. 6191 */ 6192 return; 6193 6194 if (!net->ipv6.sysctl.fib_notify_on_flag_change) 6195 return; 6196 6197 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); 6198 if (!skb) { 6199 err = -ENOBUFS; 6200 goto errout; 6201 } 6202 6203 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0, 6204 0, 0); 6205 if (err < 0) { 6206 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6207 WARN_ON(err == -EMSGSIZE); 6208 kfree_skb(skb); 6209 goto errout; 6210 } 6211 6212 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL); 6213 return; 6214 6215 errout: 6216 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6217 } 6218 EXPORT_SYMBOL(fib6_info_hw_flags_set); 6219 6220 static int ip6_route_dev_notify(struct notifier_block *this, 6221 unsigned long event, void *ptr) 6222 { 6223 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6224 struct net *net = dev_net(dev); 6225 6226 if (!(dev->flags & IFF_LOOPBACK)) 6227 return NOTIFY_OK; 6228 6229 if (event == NETDEV_REGISTER) { 6230 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev; 6231 net->ipv6.ip6_null_entry->dst.dev = dev; 6232 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev); 6233 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6234 net->ipv6.ip6_prohibit_entry->dst.dev = dev; 6235 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev); 6236 net->ipv6.ip6_blk_hole_entry->dst.dev = dev; 6237 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev); 6238 #endif 6239 } else if (event == NETDEV_UNREGISTER && 6240 dev->reg_state != NETREG_UNREGISTERED) { 6241 /* NETDEV_UNREGISTER could be fired for multiple times by 6242 * netdev_wait_allrefs(). Make sure we only call this once. 6243 */ 6244 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev); 6245 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6246 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev); 6247 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev); 6248 #endif 6249 } 6250 6251 return NOTIFY_OK; 6252 } 6253 6254 /* 6255 * /proc 6256 */ 6257 6258 #ifdef CONFIG_PROC_FS 6259 static int rt6_stats_seq_show(struct seq_file *seq, void *v) 6260 { 6261 struct net *net = (struct net *)seq->private; 6262 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n", 6263 net->ipv6.rt6_stats->fib_nodes, 6264 net->ipv6.rt6_stats->fib_route_nodes, 6265 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc), 6266 net->ipv6.rt6_stats->fib_rt_entries, 6267 net->ipv6.rt6_stats->fib_rt_cache, 6268 dst_entries_get_slow(&net->ipv6.ip6_dst_ops), 6269 net->ipv6.rt6_stats->fib_discarded_routes); 6270 6271 return 0; 6272 } 6273 #endif /* CONFIG_PROC_FS */ 6274 6275 #ifdef CONFIG_SYSCTL 6276 6277 static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write, 6278 void *buffer, size_t *lenp, loff_t *ppos) 6279 { 6280 struct net *net; 6281 int delay; 6282 int ret; 6283 if (!write) 6284 return -EINVAL; 6285 6286 net = (struct net *)ctl->extra1; 6287 delay = net->ipv6.sysctl.flush_delay; 6288 ret = proc_dointvec(ctl, write, buffer, lenp, ppos); 6289 if (ret) 6290 return ret; 6291 6292 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); 6293 return 0; 6294 } 6295 6296 static struct ctl_table ipv6_route_table_template[] = { 6297 { 6298 .procname = "flush", 6299 .data = &init_net.ipv6.sysctl.flush_delay, 6300 .maxlen = sizeof(int), 6301 .mode = 0200, 6302 .proc_handler = ipv6_sysctl_rtcache_flush 6303 }, 6304 { 6305 .procname = "gc_thresh", 6306 .data = &ip6_dst_ops_template.gc_thresh, 6307 .maxlen = sizeof(int), 6308 .mode = 0644, 6309 .proc_handler = proc_dointvec, 6310 }, 6311 { 6312 .procname = "max_size", 6313 .data = &init_net.ipv6.sysctl.ip6_rt_max_size, 6314 .maxlen = sizeof(int), 6315 .mode = 0644, 6316 .proc_handler = proc_dointvec, 6317 }, 6318 { 6319 .procname = "gc_min_interval", 6320 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6321 .maxlen = sizeof(int), 6322 .mode = 0644, 6323 .proc_handler = proc_dointvec_jiffies, 6324 }, 6325 { 6326 .procname = "gc_timeout", 6327 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout, 6328 .maxlen = sizeof(int), 6329 .mode = 0644, 6330 .proc_handler = proc_dointvec_jiffies, 6331 }, 6332 { 6333 .procname = "gc_interval", 6334 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval, 6335 .maxlen = sizeof(int), 6336 .mode = 0644, 6337 .proc_handler = proc_dointvec_jiffies, 6338 }, 6339 { 6340 .procname = "gc_elasticity", 6341 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity, 6342 .maxlen = sizeof(int), 6343 .mode = 0644, 6344 .proc_handler = proc_dointvec, 6345 }, 6346 { 6347 .procname = "mtu_expires", 6348 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires, 6349 .maxlen = sizeof(int), 6350 .mode = 0644, 6351 .proc_handler = proc_dointvec_jiffies, 6352 }, 6353 { 6354 .procname = "min_adv_mss", 6355 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss, 6356 .maxlen = sizeof(int), 6357 .mode = 0644, 6358 .proc_handler = proc_dointvec, 6359 }, 6360 { 6361 .procname = "gc_min_interval_ms", 6362 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6363 .maxlen = sizeof(int), 6364 .mode = 0644, 6365 .proc_handler = proc_dointvec_ms_jiffies, 6366 }, 6367 { 6368 .procname = "skip_notify_on_dev_down", 6369 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down, 6370 .maxlen = sizeof(int), 6371 .mode = 0644, 6372 .proc_handler = proc_dointvec_minmax, 6373 .extra1 = SYSCTL_ZERO, 6374 .extra2 = SYSCTL_ONE, 6375 }, 6376 { } 6377 }; 6378 6379 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net) 6380 { 6381 struct ctl_table *table; 6382 6383 table = kmemdup(ipv6_route_table_template, 6384 sizeof(ipv6_route_table_template), 6385 GFP_KERNEL); 6386 6387 if (table) { 6388 table[0].data = &net->ipv6.sysctl.flush_delay; 6389 table[0].extra1 = net; 6390 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh; 6391 table[2].data = &net->ipv6.sysctl.ip6_rt_max_size; 6392 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6393 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout; 6394 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval; 6395 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity; 6396 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires; 6397 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss; 6398 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6399 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down; 6400 6401 /* Don't export sysctls to unprivileged users */ 6402 if (net->user_ns != &init_user_ns) 6403 table[0].procname = NULL; 6404 } 6405 6406 return table; 6407 } 6408 #endif 6409 6410 static int __net_init ip6_route_net_init(struct net *net) 6411 { 6412 int ret = -ENOMEM; 6413 6414 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template, 6415 sizeof(net->ipv6.ip6_dst_ops)); 6416 6417 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0) 6418 goto out_ip6_dst_ops; 6419 6420 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true); 6421 if (!net->ipv6.fib6_null_entry) 6422 goto out_ip6_dst_entries; 6423 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template, 6424 sizeof(*net->ipv6.fib6_null_entry)); 6425 6426 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template, 6427 sizeof(*net->ipv6.ip6_null_entry), 6428 GFP_KERNEL); 6429 if (!net->ipv6.ip6_null_entry) 6430 goto out_fib6_null_entry; 6431 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6432 dst_init_metrics(&net->ipv6.ip6_null_entry->dst, 6433 ip6_template_metrics, true); 6434 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->rt6i_uncached); 6435 6436 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6437 net->ipv6.fib6_has_custom_rules = false; 6438 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template, 6439 sizeof(*net->ipv6.ip6_prohibit_entry), 6440 GFP_KERNEL); 6441 if (!net->ipv6.ip6_prohibit_entry) 6442 goto out_ip6_null_entry; 6443 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6444 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst, 6445 ip6_template_metrics, true); 6446 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->rt6i_uncached); 6447 6448 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template, 6449 sizeof(*net->ipv6.ip6_blk_hole_entry), 6450 GFP_KERNEL); 6451 if (!net->ipv6.ip6_blk_hole_entry) 6452 goto out_ip6_prohibit_entry; 6453 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6454 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst, 6455 ip6_template_metrics, true); 6456 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->rt6i_uncached); 6457 #ifdef CONFIG_IPV6_SUBTREES 6458 net->ipv6.fib6_routes_require_src = 0; 6459 #endif 6460 #endif 6461 6462 net->ipv6.sysctl.flush_delay = 0; 6463 net->ipv6.sysctl.ip6_rt_max_size = 4096; 6464 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2; 6465 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ; 6466 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ; 6467 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9; 6468 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ; 6469 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40; 6470 net->ipv6.sysctl.skip_notify_on_dev_down = 0; 6471 6472 net->ipv6.ip6_rt_gc_expire = 30*HZ; 6473 6474 ret = 0; 6475 out: 6476 return ret; 6477 6478 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6479 out_ip6_prohibit_entry: 6480 kfree(net->ipv6.ip6_prohibit_entry); 6481 out_ip6_null_entry: 6482 kfree(net->ipv6.ip6_null_entry); 6483 #endif 6484 out_fib6_null_entry: 6485 kfree(net->ipv6.fib6_null_entry); 6486 out_ip6_dst_entries: 6487 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6488 out_ip6_dst_ops: 6489 goto out; 6490 } 6491 6492 static void __net_exit ip6_route_net_exit(struct net *net) 6493 { 6494 kfree(net->ipv6.fib6_null_entry); 6495 kfree(net->ipv6.ip6_null_entry); 6496 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6497 kfree(net->ipv6.ip6_prohibit_entry); 6498 kfree(net->ipv6.ip6_blk_hole_entry); 6499 #endif 6500 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6501 } 6502 6503 static int __net_init ip6_route_net_init_late(struct net *net) 6504 { 6505 #ifdef CONFIG_PROC_FS 6506 proc_create_net("ipv6_route", 0, net->proc_net, &ipv6_route_seq_ops, 6507 sizeof(struct ipv6_route_iter)); 6508 proc_create_net_single("rt6_stats", 0444, net->proc_net, 6509 rt6_stats_seq_show, NULL); 6510 #endif 6511 return 0; 6512 } 6513 6514 static void __net_exit ip6_route_net_exit_late(struct net *net) 6515 { 6516 #ifdef CONFIG_PROC_FS 6517 remove_proc_entry("ipv6_route", net->proc_net); 6518 remove_proc_entry("rt6_stats", net->proc_net); 6519 #endif 6520 } 6521 6522 static struct pernet_operations ip6_route_net_ops = { 6523 .init = ip6_route_net_init, 6524 .exit = ip6_route_net_exit, 6525 }; 6526 6527 static int __net_init ipv6_inetpeer_init(struct net *net) 6528 { 6529 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 6530 6531 if (!bp) 6532 return -ENOMEM; 6533 inet_peer_base_init(bp); 6534 net->ipv6.peers = bp; 6535 return 0; 6536 } 6537 6538 static void __net_exit ipv6_inetpeer_exit(struct net *net) 6539 { 6540 struct inet_peer_base *bp = net->ipv6.peers; 6541 6542 net->ipv6.peers = NULL; 6543 inetpeer_invalidate_tree(bp); 6544 kfree(bp); 6545 } 6546 6547 static struct pernet_operations ipv6_inetpeer_ops = { 6548 .init = ipv6_inetpeer_init, 6549 .exit = ipv6_inetpeer_exit, 6550 }; 6551 6552 static struct pernet_operations ip6_route_net_late_ops = { 6553 .init = ip6_route_net_init_late, 6554 .exit = ip6_route_net_exit_late, 6555 }; 6556 6557 static struct notifier_block ip6_route_dev_notifier = { 6558 .notifier_call = ip6_route_dev_notify, 6559 .priority = ADDRCONF_NOTIFY_PRIORITY - 10, 6560 }; 6561 6562 void __init ip6_route_init_special_entries(void) 6563 { 6564 /* Registering of the loopback is done before this portion of code, 6565 * the loopback reference in rt6_info will not be taken, do it 6566 * manually for init_net */ 6567 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev; 6568 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev; 6569 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6570 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6571 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev; 6572 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6573 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev; 6574 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6575 #endif 6576 } 6577 6578 #if IS_BUILTIN(CONFIG_IPV6) 6579 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6580 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt) 6581 6582 BTF_ID_LIST(btf_fib6_info_id) 6583 BTF_ID(struct, fib6_info) 6584 6585 static const struct bpf_iter_seq_info ipv6_route_seq_info = { 6586 .seq_ops = &ipv6_route_seq_ops, 6587 .init_seq_private = bpf_iter_init_seq_net, 6588 .fini_seq_private = bpf_iter_fini_seq_net, 6589 .seq_priv_size = sizeof(struct ipv6_route_iter), 6590 }; 6591 6592 static struct bpf_iter_reg ipv6_route_reg_info = { 6593 .target = "ipv6_route", 6594 .ctx_arg_info_size = 1, 6595 .ctx_arg_info = { 6596 { offsetof(struct bpf_iter__ipv6_route, rt), 6597 PTR_TO_BTF_ID_OR_NULL }, 6598 }, 6599 .seq_info = &ipv6_route_seq_info, 6600 }; 6601 6602 static int __init bpf_iter_register(void) 6603 { 6604 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id; 6605 return bpf_iter_reg_target(&ipv6_route_reg_info); 6606 } 6607 6608 static void bpf_iter_unregister(void) 6609 { 6610 bpf_iter_unreg_target(&ipv6_route_reg_info); 6611 } 6612 #endif 6613 #endif 6614 6615 int __init ip6_route_init(void) 6616 { 6617 int ret; 6618 int cpu; 6619 6620 ret = -ENOMEM; 6621 ip6_dst_ops_template.kmem_cachep = 6622 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0, 6623 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL); 6624 if (!ip6_dst_ops_template.kmem_cachep) 6625 goto out; 6626 6627 ret = dst_entries_init(&ip6_dst_blackhole_ops); 6628 if (ret) 6629 goto out_kmem_cache; 6630 6631 ret = register_pernet_subsys(&ipv6_inetpeer_ops); 6632 if (ret) 6633 goto out_dst_entries; 6634 6635 ret = register_pernet_subsys(&ip6_route_net_ops); 6636 if (ret) 6637 goto out_register_inetpeer; 6638 6639 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep; 6640 6641 ret = fib6_init(); 6642 if (ret) 6643 goto out_register_subsys; 6644 6645 ret = xfrm6_init(); 6646 if (ret) 6647 goto out_fib6_init; 6648 6649 ret = fib6_rules_init(); 6650 if (ret) 6651 goto xfrm6_init; 6652 6653 ret = register_pernet_subsys(&ip6_route_net_late_ops); 6654 if (ret) 6655 goto fib6_rules_init; 6656 6657 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE, 6658 inet6_rtm_newroute, NULL, 0); 6659 if (ret < 0) 6660 goto out_register_late_subsys; 6661 6662 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE, 6663 inet6_rtm_delroute, NULL, 0); 6664 if (ret < 0) 6665 goto out_register_late_subsys; 6666 6667 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE, 6668 inet6_rtm_getroute, NULL, 6669 RTNL_FLAG_DOIT_UNLOCKED); 6670 if (ret < 0) 6671 goto out_register_late_subsys; 6672 6673 ret = register_netdevice_notifier(&ip6_route_dev_notifier); 6674 if (ret) 6675 goto out_register_late_subsys; 6676 6677 #if IS_BUILTIN(CONFIG_IPV6) 6678 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6679 ret = bpf_iter_register(); 6680 if (ret) 6681 goto out_register_late_subsys; 6682 #endif 6683 #endif 6684 6685 for_each_possible_cpu(cpu) { 6686 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 6687 6688 INIT_LIST_HEAD(&ul->head); 6689 spin_lock_init(&ul->lock); 6690 } 6691 6692 out: 6693 return ret; 6694 6695 out_register_late_subsys: 6696 rtnl_unregister_all(PF_INET6); 6697 unregister_pernet_subsys(&ip6_route_net_late_ops); 6698 fib6_rules_init: 6699 fib6_rules_cleanup(); 6700 xfrm6_init: 6701 xfrm6_fini(); 6702 out_fib6_init: 6703 fib6_gc_cleanup(); 6704 out_register_subsys: 6705 unregister_pernet_subsys(&ip6_route_net_ops); 6706 out_register_inetpeer: 6707 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6708 out_dst_entries: 6709 dst_entries_destroy(&ip6_dst_blackhole_ops); 6710 out_kmem_cache: 6711 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6712 goto out; 6713 } 6714 6715 void ip6_route_cleanup(void) 6716 { 6717 #if IS_BUILTIN(CONFIG_IPV6) 6718 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6719 bpf_iter_unregister(); 6720 #endif 6721 #endif 6722 unregister_netdevice_notifier(&ip6_route_dev_notifier); 6723 unregister_pernet_subsys(&ip6_route_net_late_ops); 6724 fib6_rules_cleanup(); 6725 xfrm6_fini(); 6726 fib6_gc_cleanup(); 6727 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6728 unregister_pernet_subsys(&ip6_route_net_ops); 6729 dst_entries_destroy(&ip6_dst_blackhole_ops); 6730 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6731 } 6732