1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux INET6 implementation 4 * FIB front-end. 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 */ 9 10 /* Changes: 11 * 12 * YOSHIFUJI Hideaki @USAGI 13 * reworked default router selection. 14 * - respect outgoing interface 15 * - select from (probably) reachable routers (i.e. 16 * routers in REACHABLE, STALE, DELAY or PROBE states). 17 * - always select the same router if it is (probably) 18 * reachable. otherwise, round-robin the list. 19 * Ville Nuorvala 20 * Fixed routing subtrees. 21 */ 22 23 #define pr_fmt(fmt) "IPv6: " fmt 24 25 #include <linux/capability.h> 26 #include <linux/errno.h> 27 #include <linux/export.h> 28 #include <linux/types.h> 29 #include <linux/times.h> 30 #include <linux/socket.h> 31 #include <linux/sockios.h> 32 #include <linux/net.h> 33 #include <linux/route.h> 34 #include <linux/netdevice.h> 35 #include <linux/in6.h> 36 #include <linux/mroute6.h> 37 #include <linux/init.h> 38 #include <linux/if_arp.h> 39 #include <linux/proc_fs.h> 40 #include <linux/seq_file.h> 41 #include <linux/nsproxy.h> 42 #include <linux/slab.h> 43 #include <linux/jhash.h> 44 #include <linux/siphash.h> 45 #include <net/net_namespace.h> 46 #include <net/snmp.h> 47 #include <net/ipv6.h> 48 #include <net/ip6_fib.h> 49 #include <net/ip6_route.h> 50 #include <net/ndisc.h> 51 #include <net/addrconf.h> 52 #include <net/tcp.h> 53 #include <linux/rtnetlink.h> 54 #include <net/dst.h> 55 #include <net/dst_metadata.h> 56 #include <net/xfrm.h> 57 #include <net/netevent.h> 58 #include <net/netlink.h> 59 #include <net/rtnh.h> 60 #include <net/lwtunnel.h> 61 #include <net/ip_tunnels.h> 62 #include <net/l3mdev.h> 63 #include <net/ip.h> 64 #include <linux/uaccess.h> 65 #include <linux/btf_ids.h> 66 67 #ifdef CONFIG_SYSCTL 68 #include <linux/sysctl.h> 69 #endif 70 71 static int ip6_rt_type_to_error(u8 fib6_type); 72 73 #define CREATE_TRACE_POINTS 74 #include <trace/events/fib6.h> 75 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup); 76 #undef CREATE_TRACE_POINTS 77 78 enum rt6_nud_state { 79 RT6_NUD_FAIL_HARD = -3, 80 RT6_NUD_FAIL_PROBE = -2, 81 RT6_NUD_FAIL_DO_RR = -1, 82 RT6_NUD_SUCCEED = 1 83 }; 84 85 INDIRECT_CALLABLE_SCOPE 86 struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie); 87 static unsigned int ip6_default_advmss(const struct dst_entry *dst); 88 INDIRECT_CALLABLE_SCOPE 89 unsigned int ip6_mtu(const struct dst_entry *dst); 90 static void ip6_negative_advice(struct sock *sk, 91 struct dst_entry *dst); 92 static void ip6_dst_destroy(struct dst_entry *); 93 static void ip6_dst_ifdown(struct dst_entry *, 94 struct net_device *dev); 95 static void ip6_dst_gc(struct dst_ops *ops); 96 97 static int ip6_pkt_discard(struct sk_buff *skb); 98 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb); 99 static int ip6_pkt_prohibit(struct sk_buff *skb); 100 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb); 101 static void ip6_link_failure(struct sk_buff *skb); 102 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 103 struct sk_buff *skb, u32 mtu, 104 bool confirm_neigh); 105 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, 106 struct sk_buff *skb); 107 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 108 int strict); 109 static size_t rt6_nlmsg_size(struct fib6_info *f6i); 110 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 111 struct fib6_info *rt, struct dst_entry *dst, 112 struct in6_addr *dest, struct in6_addr *src, 113 int iif, int type, u32 portid, u32 seq, 114 unsigned int flags); 115 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 116 const struct in6_addr *daddr, 117 const struct in6_addr *saddr); 118 119 #ifdef CONFIG_IPV6_ROUTE_INFO 120 static struct fib6_info *rt6_add_route_info(struct net *net, 121 const struct in6_addr *prefix, int prefixlen, 122 const struct in6_addr *gwaddr, 123 struct net_device *dev, 124 unsigned int pref); 125 static struct fib6_info *rt6_get_route_info(struct net *net, 126 const struct in6_addr *prefix, int prefixlen, 127 const struct in6_addr *gwaddr, 128 struct net_device *dev); 129 #endif 130 131 struct uncached_list { 132 spinlock_t lock; 133 struct list_head head; 134 struct list_head quarantine; 135 }; 136 137 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); 138 139 void rt6_uncached_list_add(struct rt6_info *rt) 140 { 141 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); 142 143 rt->dst.rt_uncached_list = ul; 144 145 spin_lock_bh(&ul->lock); 146 list_add_tail(&rt->dst.rt_uncached, &ul->head); 147 spin_unlock_bh(&ul->lock); 148 } 149 150 void rt6_uncached_list_del(struct rt6_info *rt) 151 { 152 if (!list_empty(&rt->dst.rt_uncached)) { 153 struct uncached_list *ul = rt->dst.rt_uncached_list; 154 155 spin_lock_bh(&ul->lock); 156 list_del_init(&rt->dst.rt_uncached); 157 spin_unlock_bh(&ul->lock); 158 } 159 } 160 161 static void rt6_uncached_list_flush_dev(struct net_device *dev) 162 { 163 int cpu; 164 165 for_each_possible_cpu(cpu) { 166 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 167 struct rt6_info *rt, *safe; 168 169 if (list_empty(&ul->head)) 170 continue; 171 172 spin_lock_bh(&ul->lock); 173 list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { 174 struct inet6_dev *rt_idev = rt->rt6i_idev; 175 struct net_device *rt_dev = rt->dst.dev; 176 bool handled = false; 177 178 if (rt_idev->dev == dev) { 179 rt->rt6i_idev = in6_dev_get(blackhole_netdev); 180 in6_dev_put(rt_idev); 181 handled = true; 182 } 183 184 if (rt_dev == dev) { 185 rt->dst.dev = blackhole_netdev; 186 netdev_ref_replace(rt_dev, blackhole_netdev, 187 &rt->dst.dev_tracker, 188 GFP_ATOMIC); 189 handled = true; 190 } 191 if (handled) 192 list_move(&rt->dst.rt_uncached, 193 &ul->quarantine); 194 } 195 spin_unlock_bh(&ul->lock); 196 } 197 } 198 199 static inline const void *choose_neigh_daddr(const struct in6_addr *p, 200 struct sk_buff *skb, 201 const void *daddr) 202 { 203 if (!ipv6_addr_any(p)) 204 return (const void *) p; 205 else if (skb) 206 return &ipv6_hdr(skb)->daddr; 207 return daddr; 208 } 209 210 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw, 211 struct net_device *dev, 212 struct sk_buff *skb, 213 const void *daddr) 214 { 215 struct neighbour *n; 216 217 daddr = choose_neigh_daddr(gw, skb, daddr); 218 n = __ipv6_neigh_lookup(dev, daddr); 219 if (n) 220 return n; 221 222 n = neigh_create(&nd_tbl, daddr, dev); 223 return IS_ERR(n) ? NULL : n; 224 } 225 226 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst, 227 struct sk_buff *skb, 228 const void *daddr) 229 { 230 const struct rt6_info *rt = dst_rt6_info(dst); 231 232 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any), 233 dst->dev, skb, daddr); 234 } 235 236 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr) 237 { 238 const struct rt6_info *rt = dst_rt6_info(dst); 239 struct net_device *dev = dst->dev; 240 241 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr); 242 if (!daddr) 243 return; 244 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) 245 return; 246 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr)) 247 return; 248 __ipv6_confirm_neigh(dev, daddr); 249 } 250 251 static struct dst_ops ip6_dst_ops_template = { 252 .family = AF_INET6, 253 .gc = ip6_dst_gc, 254 .gc_thresh = 1024, 255 .check = ip6_dst_check, 256 .default_advmss = ip6_default_advmss, 257 .mtu = ip6_mtu, 258 .cow_metrics = dst_cow_metrics_generic, 259 .destroy = ip6_dst_destroy, 260 .ifdown = ip6_dst_ifdown, 261 .negative_advice = ip6_negative_advice, 262 .link_failure = ip6_link_failure, 263 .update_pmtu = ip6_rt_update_pmtu, 264 .redirect = rt6_do_redirect, 265 .local_out = __ip6_local_out, 266 .neigh_lookup = ip6_dst_neigh_lookup, 267 .confirm_neigh = ip6_confirm_neigh, 268 }; 269 270 static struct dst_ops ip6_dst_blackhole_ops = { 271 .family = AF_INET6, 272 .default_advmss = ip6_default_advmss, 273 .neigh_lookup = ip6_dst_neigh_lookup, 274 .check = ip6_dst_check, 275 .destroy = ip6_dst_destroy, 276 .cow_metrics = dst_cow_metrics_generic, 277 .update_pmtu = dst_blackhole_update_pmtu, 278 .redirect = dst_blackhole_redirect, 279 .mtu = dst_blackhole_mtu, 280 }; 281 282 static const u32 ip6_template_metrics[RTAX_MAX] = { 283 [RTAX_HOPLIMIT - 1] = 0, 284 }; 285 286 static const struct fib6_info fib6_null_entry_template = { 287 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP), 288 .fib6_protocol = RTPROT_KERNEL, 289 .fib6_metric = ~(u32)0, 290 .fib6_ref = REFCOUNT_INIT(1), 291 .fib6_type = RTN_UNREACHABLE, 292 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics, 293 }; 294 295 static const struct rt6_info ip6_null_entry_template = { 296 .dst = { 297 .__rcuref = RCUREF_INIT(1), 298 .__use = 1, 299 .obsolete = DST_OBSOLETE_FORCE_CHK, 300 .error = -ENETUNREACH, 301 .input = ip6_pkt_discard, 302 .output = ip6_pkt_discard_out, 303 }, 304 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 305 }; 306 307 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 308 309 static const struct rt6_info ip6_prohibit_entry_template = { 310 .dst = { 311 .__rcuref = RCUREF_INIT(1), 312 .__use = 1, 313 .obsolete = DST_OBSOLETE_FORCE_CHK, 314 .error = -EACCES, 315 .input = ip6_pkt_prohibit, 316 .output = ip6_pkt_prohibit_out, 317 }, 318 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 319 }; 320 321 static const struct rt6_info ip6_blk_hole_entry_template = { 322 .dst = { 323 .__rcuref = RCUREF_INIT(1), 324 .__use = 1, 325 .obsolete = DST_OBSOLETE_FORCE_CHK, 326 .error = -EINVAL, 327 .input = dst_discard, 328 .output = dst_discard_out, 329 }, 330 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 331 }; 332 333 #endif 334 335 static void rt6_info_init(struct rt6_info *rt) 336 { 337 memset_after(rt, 0, dst); 338 } 339 340 /* allocate dst with ip6_dst_ops */ 341 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, 342 int flags) 343 { 344 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev, 345 DST_OBSOLETE_FORCE_CHK, flags); 346 347 if (rt) { 348 rt6_info_init(rt); 349 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 350 } 351 352 return rt; 353 } 354 EXPORT_SYMBOL(ip6_dst_alloc); 355 356 static void ip6_dst_destroy(struct dst_entry *dst) 357 { 358 struct rt6_info *rt = dst_rt6_info(dst); 359 struct fib6_info *from; 360 struct inet6_dev *idev; 361 362 ip_dst_metrics_put(dst); 363 rt6_uncached_list_del(rt); 364 365 idev = rt->rt6i_idev; 366 if (idev) { 367 rt->rt6i_idev = NULL; 368 in6_dev_put(idev); 369 } 370 371 from = xchg((__force struct fib6_info **)&rt->from, NULL); 372 fib6_info_release(from); 373 } 374 375 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev) 376 { 377 struct rt6_info *rt = dst_rt6_info(dst); 378 struct inet6_dev *idev = rt->rt6i_idev; 379 380 if (idev && idev->dev != blackhole_netdev) { 381 struct inet6_dev *blackhole_idev = in6_dev_get(blackhole_netdev); 382 383 if (blackhole_idev) { 384 rt->rt6i_idev = blackhole_idev; 385 in6_dev_put(idev); 386 } 387 } 388 } 389 390 static bool __rt6_check_expired(const struct rt6_info *rt) 391 { 392 if (rt->rt6i_flags & RTF_EXPIRES) 393 return time_after(jiffies, rt->dst.expires); 394 else 395 return false; 396 } 397 398 static bool rt6_check_expired(const struct rt6_info *rt) 399 { 400 struct fib6_info *from; 401 402 from = rcu_dereference(rt->from); 403 404 if (rt->rt6i_flags & RTF_EXPIRES) { 405 if (time_after(jiffies, rt->dst.expires)) 406 return true; 407 } else if (from) { 408 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK || 409 fib6_check_expired(from); 410 } 411 return false; 412 } 413 414 void fib6_select_path(const struct net *net, struct fib6_result *res, 415 struct flowi6 *fl6, int oif, bool have_oif_match, 416 const struct sk_buff *skb, int strict) 417 { 418 struct fib6_info *sibling, *next_sibling; 419 struct fib6_info *match = res->f6i; 420 421 if (!match->nh && (!match->fib6_nsiblings || have_oif_match)) 422 goto out; 423 424 if (match->nh && have_oif_match && res->nh) 425 return; 426 427 if (skb) 428 IP6CB(skb)->flags |= IP6SKB_MULTIPATH; 429 430 /* We might have already computed the hash for ICMPv6 errors. In such 431 * case it will always be non-zero. Otherwise now is the time to do it. 432 */ 433 if (!fl6->mp_hash && 434 (!match->nh || nexthop_is_multipath(match->nh))) 435 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL); 436 437 if (unlikely(match->nh)) { 438 nexthop_path_fib6_result(res, fl6->mp_hash); 439 return; 440 } 441 442 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound)) 443 goto out; 444 445 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings, 446 fib6_siblings) { 447 const struct fib6_nh *nh = sibling->fib6_nh; 448 int nh_upper_bound; 449 450 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound); 451 if (fl6->mp_hash > nh_upper_bound) 452 continue; 453 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0) 454 break; 455 match = sibling; 456 break; 457 } 458 459 out: 460 res->f6i = match; 461 res->nh = match->fib6_nh; 462 } 463 464 /* 465 * Route lookup. rcu_read_lock() should be held. 466 */ 467 468 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh, 469 const struct in6_addr *saddr, int oif, int flags) 470 { 471 const struct net_device *dev; 472 473 if (nh->fib_nh_flags & RTNH_F_DEAD) 474 return false; 475 476 dev = nh->fib_nh_dev; 477 if (oif) { 478 if (dev->ifindex == oif) 479 return true; 480 } else { 481 if (ipv6_chk_addr(net, saddr, dev, 482 flags & RT6_LOOKUP_F_IFACE)) 483 return true; 484 } 485 486 return false; 487 } 488 489 struct fib6_nh_dm_arg { 490 struct net *net; 491 const struct in6_addr *saddr; 492 int oif; 493 int flags; 494 struct fib6_nh *nh; 495 }; 496 497 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg) 498 { 499 struct fib6_nh_dm_arg *arg = _arg; 500 501 arg->nh = nh; 502 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif, 503 arg->flags); 504 } 505 506 /* returns fib6_nh from nexthop or NULL */ 507 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh, 508 struct fib6_result *res, 509 const struct in6_addr *saddr, 510 int oif, int flags) 511 { 512 struct fib6_nh_dm_arg arg = { 513 .net = net, 514 .saddr = saddr, 515 .oif = oif, 516 .flags = flags, 517 }; 518 519 if (nexthop_is_blackhole(nh)) 520 return NULL; 521 522 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg)) 523 return arg.nh; 524 525 return NULL; 526 } 527 528 static void rt6_device_match(struct net *net, struct fib6_result *res, 529 const struct in6_addr *saddr, int oif, int flags) 530 { 531 struct fib6_info *f6i = res->f6i; 532 struct fib6_info *spf6i; 533 struct fib6_nh *nh; 534 535 if (!oif && ipv6_addr_any(saddr)) { 536 if (unlikely(f6i->nh)) { 537 nh = nexthop_fib6_nh(f6i->nh); 538 if (nexthop_is_blackhole(f6i->nh)) 539 goto out_blackhole; 540 } else { 541 nh = f6i->fib6_nh; 542 } 543 if (!(nh->fib_nh_flags & RTNH_F_DEAD)) 544 goto out; 545 } 546 547 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) { 548 bool matched = false; 549 550 if (unlikely(spf6i->nh)) { 551 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr, 552 oif, flags); 553 if (nh) 554 matched = true; 555 } else { 556 nh = spf6i->fib6_nh; 557 if (__rt6_device_match(net, nh, saddr, oif, flags)) 558 matched = true; 559 } 560 if (matched) { 561 res->f6i = spf6i; 562 goto out; 563 } 564 } 565 566 if (oif && flags & RT6_LOOKUP_F_IFACE) { 567 res->f6i = net->ipv6.fib6_null_entry; 568 nh = res->f6i->fib6_nh; 569 goto out; 570 } 571 572 if (unlikely(f6i->nh)) { 573 nh = nexthop_fib6_nh(f6i->nh); 574 if (nexthop_is_blackhole(f6i->nh)) 575 goto out_blackhole; 576 } else { 577 nh = f6i->fib6_nh; 578 } 579 580 if (nh->fib_nh_flags & RTNH_F_DEAD) { 581 res->f6i = net->ipv6.fib6_null_entry; 582 nh = res->f6i->fib6_nh; 583 } 584 out: 585 res->nh = nh; 586 res->fib6_type = res->f6i->fib6_type; 587 res->fib6_flags = res->f6i->fib6_flags; 588 return; 589 590 out_blackhole: 591 res->fib6_flags |= RTF_REJECT; 592 res->fib6_type = RTN_BLACKHOLE; 593 res->nh = nh; 594 } 595 596 #ifdef CONFIG_IPV6_ROUTER_PREF 597 struct __rt6_probe_work { 598 struct work_struct work; 599 struct in6_addr target; 600 struct net_device *dev; 601 netdevice_tracker dev_tracker; 602 }; 603 604 static void rt6_probe_deferred(struct work_struct *w) 605 { 606 struct in6_addr mcaddr; 607 struct __rt6_probe_work *work = 608 container_of(w, struct __rt6_probe_work, work); 609 610 addrconf_addr_solict_mult(&work->target, &mcaddr); 611 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); 612 netdev_put(work->dev, &work->dev_tracker); 613 kfree(work); 614 } 615 616 static void rt6_probe(struct fib6_nh *fib6_nh) 617 { 618 struct __rt6_probe_work *work = NULL; 619 const struct in6_addr *nh_gw; 620 unsigned long last_probe; 621 struct neighbour *neigh; 622 struct net_device *dev; 623 struct inet6_dev *idev; 624 625 /* 626 * Okay, this does not seem to be appropriate 627 * for now, however, we need to check if it 628 * is really so; aka Router Reachability Probing. 629 * 630 * Router Reachability Probe MUST be rate-limited 631 * to no more than one per minute. 632 */ 633 if (!fib6_nh->fib_nh_gw_family) 634 return; 635 636 nh_gw = &fib6_nh->fib_nh_gw6; 637 dev = fib6_nh->fib_nh_dev; 638 rcu_read_lock(); 639 last_probe = READ_ONCE(fib6_nh->last_probe); 640 idev = __in6_dev_get(dev); 641 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); 642 if (neigh) { 643 if (READ_ONCE(neigh->nud_state) & NUD_VALID) 644 goto out; 645 646 write_lock_bh(&neigh->lock); 647 if (!(neigh->nud_state & NUD_VALID) && 648 time_after(jiffies, 649 neigh->updated + 650 READ_ONCE(idev->cnf.rtr_probe_interval))) { 651 work = kmalloc(sizeof(*work), GFP_ATOMIC); 652 if (work) 653 __neigh_set_probe_once(neigh); 654 } 655 write_unlock_bh(&neigh->lock); 656 } else if (time_after(jiffies, last_probe + 657 READ_ONCE(idev->cnf.rtr_probe_interval))) { 658 work = kmalloc(sizeof(*work), GFP_ATOMIC); 659 } 660 661 if (!work || cmpxchg(&fib6_nh->last_probe, 662 last_probe, jiffies) != last_probe) { 663 kfree(work); 664 } else { 665 INIT_WORK(&work->work, rt6_probe_deferred); 666 work->target = *nh_gw; 667 netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC); 668 work->dev = dev; 669 schedule_work(&work->work); 670 } 671 672 out: 673 rcu_read_unlock(); 674 } 675 #else 676 static inline void rt6_probe(struct fib6_nh *fib6_nh) 677 { 678 } 679 #endif 680 681 /* 682 * Default Router Selection (RFC 2461 6.3.6) 683 */ 684 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh) 685 { 686 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD; 687 struct neighbour *neigh; 688 689 rcu_read_lock(); 690 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev, 691 &fib6_nh->fib_nh_gw6); 692 if (neigh) { 693 u8 nud_state = READ_ONCE(neigh->nud_state); 694 695 if (nud_state & NUD_VALID) 696 ret = RT6_NUD_SUCCEED; 697 #ifdef CONFIG_IPV6_ROUTER_PREF 698 else if (!(nud_state & NUD_FAILED)) 699 ret = RT6_NUD_SUCCEED; 700 else 701 ret = RT6_NUD_FAIL_PROBE; 702 #endif 703 } else { 704 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ? 705 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR; 706 } 707 rcu_read_unlock(); 708 709 return ret; 710 } 711 712 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 713 int strict) 714 { 715 int m = 0; 716 717 if (!oif || nh->fib_nh_dev->ifindex == oif) 718 m = 2; 719 720 if (!m && (strict & RT6_LOOKUP_F_IFACE)) 721 return RT6_NUD_FAIL_HARD; 722 #ifdef CONFIG_IPV6_ROUTER_PREF 723 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2; 724 #endif 725 if ((strict & RT6_LOOKUP_F_REACHABLE) && 726 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) { 727 int n = rt6_check_neigh(nh); 728 if (n < 0) 729 return n; 730 } 731 return m; 732 } 733 734 static bool find_match(struct fib6_nh *nh, u32 fib6_flags, 735 int oif, int strict, int *mpri, bool *do_rr) 736 { 737 bool match_do_rr = false; 738 bool rc = false; 739 int m; 740 741 if (nh->fib_nh_flags & RTNH_F_DEAD) 742 goto out; 743 744 if (ip6_ignore_linkdown(nh->fib_nh_dev) && 745 nh->fib_nh_flags & RTNH_F_LINKDOWN && 746 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE)) 747 goto out; 748 749 m = rt6_score_route(nh, fib6_flags, oif, strict); 750 if (m == RT6_NUD_FAIL_DO_RR) { 751 match_do_rr = true; 752 m = 0; /* lowest valid score */ 753 } else if (m == RT6_NUD_FAIL_HARD) { 754 goto out; 755 } 756 757 if (strict & RT6_LOOKUP_F_REACHABLE) 758 rt6_probe(nh); 759 760 /* note that m can be RT6_NUD_FAIL_PROBE at this point */ 761 if (m > *mpri) { 762 *do_rr = match_do_rr; 763 *mpri = m; 764 rc = true; 765 } 766 out: 767 return rc; 768 } 769 770 struct fib6_nh_frl_arg { 771 u32 flags; 772 int oif; 773 int strict; 774 int *mpri; 775 bool *do_rr; 776 struct fib6_nh *nh; 777 }; 778 779 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg) 780 { 781 struct fib6_nh_frl_arg *arg = _arg; 782 783 arg->nh = nh; 784 return find_match(nh, arg->flags, arg->oif, arg->strict, 785 arg->mpri, arg->do_rr); 786 } 787 788 static void __find_rr_leaf(struct fib6_info *f6i_start, 789 struct fib6_info *nomatch, u32 metric, 790 struct fib6_result *res, struct fib6_info **cont, 791 int oif, int strict, bool *do_rr, int *mpri) 792 { 793 struct fib6_info *f6i; 794 795 for (f6i = f6i_start; 796 f6i && f6i != nomatch; 797 f6i = rcu_dereference(f6i->fib6_next)) { 798 bool matched = false; 799 struct fib6_nh *nh; 800 801 if (cont && f6i->fib6_metric != metric) { 802 *cont = f6i; 803 return; 804 } 805 806 if (fib6_check_expired(f6i)) 807 continue; 808 809 if (unlikely(f6i->nh)) { 810 struct fib6_nh_frl_arg arg = { 811 .flags = f6i->fib6_flags, 812 .oif = oif, 813 .strict = strict, 814 .mpri = mpri, 815 .do_rr = do_rr 816 }; 817 818 if (nexthop_is_blackhole(f6i->nh)) { 819 res->fib6_flags = RTF_REJECT; 820 res->fib6_type = RTN_BLACKHOLE; 821 res->f6i = f6i; 822 res->nh = nexthop_fib6_nh(f6i->nh); 823 return; 824 } 825 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match, 826 &arg)) { 827 matched = true; 828 nh = arg.nh; 829 } 830 } else { 831 nh = f6i->fib6_nh; 832 if (find_match(nh, f6i->fib6_flags, oif, strict, 833 mpri, do_rr)) 834 matched = true; 835 } 836 if (matched) { 837 res->f6i = f6i; 838 res->nh = nh; 839 res->fib6_flags = f6i->fib6_flags; 840 res->fib6_type = f6i->fib6_type; 841 } 842 } 843 } 844 845 static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf, 846 struct fib6_info *rr_head, int oif, int strict, 847 bool *do_rr, struct fib6_result *res) 848 { 849 u32 metric = rr_head->fib6_metric; 850 struct fib6_info *cont = NULL; 851 int mpri = -1; 852 853 __find_rr_leaf(rr_head, NULL, metric, res, &cont, 854 oif, strict, do_rr, &mpri); 855 856 __find_rr_leaf(leaf, rr_head, metric, res, &cont, 857 oif, strict, do_rr, &mpri); 858 859 if (res->f6i || !cont) 860 return; 861 862 __find_rr_leaf(cont, NULL, metric, res, NULL, 863 oif, strict, do_rr, &mpri); 864 } 865 866 static void rt6_select(struct net *net, struct fib6_node *fn, int oif, 867 struct fib6_result *res, int strict) 868 { 869 struct fib6_info *leaf = rcu_dereference(fn->leaf); 870 struct fib6_info *rt0; 871 bool do_rr = false; 872 int key_plen; 873 874 /* make sure this function or its helpers sets f6i */ 875 res->f6i = NULL; 876 877 if (!leaf || leaf == net->ipv6.fib6_null_entry) 878 goto out; 879 880 rt0 = rcu_dereference(fn->rr_ptr); 881 if (!rt0) 882 rt0 = leaf; 883 884 /* Double check to make sure fn is not an intermediate node 885 * and fn->leaf does not points to its child's leaf 886 * (This might happen if all routes under fn are deleted from 887 * the tree and fib6_repair_tree() is called on the node.) 888 */ 889 key_plen = rt0->fib6_dst.plen; 890 #ifdef CONFIG_IPV6_SUBTREES 891 if (rt0->fib6_src.plen) 892 key_plen = rt0->fib6_src.plen; 893 #endif 894 if (fn->fn_bit != key_plen) 895 goto out; 896 897 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res); 898 if (do_rr) { 899 struct fib6_info *next = rcu_dereference(rt0->fib6_next); 900 901 /* no entries matched; do round-robin */ 902 if (!next || next->fib6_metric != rt0->fib6_metric) 903 next = leaf; 904 905 if (next != rt0) { 906 spin_lock_bh(&leaf->fib6_table->tb6_lock); 907 /* make sure next is not being deleted from the tree */ 908 if (next->fib6_node) 909 rcu_assign_pointer(fn->rr_ptr, next); 910 spin_unlock_bh(&leaf->fib6_table->tb6_lock); 911 } 912 } 913 914 out: 915 if (!res->f6i) { 916 res->f6i = net->ipv6.fib6_null_entry; 917 res->nh = res->f6i->fib6_nh; 918 res->fib6_flags = res->f6i->fib6_flags; 919 res->fib6_type = res->f6i->fib6_type; 920 } 921 } 922 923 static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res) 924 { 925 return (res->f6i->fib6_flags & RTF_NONEXTHOP) || 926 res->nh->fib_nh_gw_family; 927 } 928 929 #ifdef CONFIG_IPV6_ROUTE_INFO 930 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, 931 const struct in6_addr *gwaddr) 932 { 933 struct net *net = dev_net(dev); 934 struct route_info *rinfo = (struct route_info *) opt; 935 struct in6_addr prefix_buf, *prefix; 936 struct fib6_table *table; 937 unsigned int pref; 938 unsigned long lifetime; 939 struct fib6_info *rt; 940 941 if (len < sizeof(struct route_info)) { 942 return -EINVAL; 943 } 944 945 /* Sanity check for prefix_len and length */ 946 if (rinfo->length > 3) { 947 return -EINVAL; 948 } else if (rinfo->prefix_len > 128) { 949 return -EINVAL; 950 } else if (rinfo->prefix_len > 64) { 951 if (rinfo->length < 2) { 952 return -EINVAL; 953 } 954 } else if (rinfo->prefix_len > 0) { 955 if (rinfo->length < 1) { 956 return -EINVAL; 957 } 958 } 959 960 pref = rinfo->route_pref; 961 if (pref == ICMPV6_ROUTER_PREF_INVALID) 962 return -EINVAL; 963 964 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ); 965 966 if (rinfo->length == 3) 967 prefix = (struct in6_addr *)rinfo->prefix; 968 else { 969 /* this function is safe */ 970 ipv6_addr_prefix(&prefix_buf, 971 (struct in6_addr *)rinfo->prefix, 972 rinfo->prefix_len); 973 prefix = &prefix_buf; 974 } 975 976 if (rinfo->prefix_len == 0) 977 rt = rt6_get_dflt_router(net, gwaddr, dev); 978 else 979 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len, 980 gwaddr, dev); 981 982 if (rt && !lifetime) { 983 ip6_del_rt(net, rt, false); 984 rt = NULL; 985 } 986 987 if (!rt && lifetime) 988 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, 989 dev, pref); 990 else if (rt) 991 rt->fib6_flags = RTF_ROUTEINFO | 992 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref); 993 994 if (rt) { 995 table = rt->fib6_table; 996 spin_lock_bh(&table->tb6_lock); 997 998 if (!addrconf_finite_timeout(lifetime)) { 999 fib6_clean_expires(rt); 1000 fib6_remove_gc_list(rt); 1001 } else { 1002 fib6_set_expires(rt, jiffies + HZ * lifetime); 1003 fib6_add_gc_list(rt); 1004 } 1005 1006 spin_unlock_bh(&table->tb6_lock); 1007 1008 fib6_info_release(rt); 1009 } 1010 return 0; 1011 } 1012 #endif 1013 1014 /* 1015 * Misc support functions 1016 */ 1017 1018 /* called with rcu_lock held */ 1019 static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) 1020 { 1021 struct net_device *dev = res->nh->fib_nh_dev; 1022 1023 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 1024 /* for copies of local routes, dst->dev needs to be the 1025 * device if it is a master device, the master device if 1026 * device is enslaved, and the loopback as the default 1027 */ 1028 if (netif_is_l3_slave(dev) && 1029 !rt6_need_strict(&res->f6i->fib6_dst.addr)) 1030 dev = l3mdev_master_dev_rcu(dev); 1031 else if (!netif_is_l3_master(dev)) 1032 dev = dev_net(dev)->loopback_dev; 1033 /* last case is netif_is_l3_master(dev) is true in which 1034 * case we want dev returned to be dev 1035 */ 1036 } 1037 1038 return dev; 1039 } 1040 1041 static const int fib6_prop[RTN_MAX + 1] = { 1042 [RTN_UNSPEC] = 0, 1043 [RTN_UNICAST] = 0, 1044 [RTN_LOCAL] = 0, 1045 [RTN_BROADCAST] = 0, 1046 [RTN_ANYCAST] = 0, 1047 [RTN_MULTICAST] = 0, 1048 [RTN_BLACKHOLE] = -EINVAL, 1049 [RTN_UNREACHABLE] = -EHOSTUNREACH, 1050 [RTN_PROHIBIT] = -EACCES, 1051 [RTN_THROW] = -EAGAIN, 1052 [RTN_NAT] = -EINVAL, 1053 [RTN_XRESOLVE] = -EINVAL, 1054 }; 1055 1056 static int ip6_rt_type_to_error(u8 fib6_type) 1057 { 1058 return fib6_prop[fib6_type]; 1059 } 1060 1061 static unsigned short fib6_info_dst_flags(struct fib6_info *rt) 1062 { 1063 unsigned short flags = 0; 1064 1065 if (rt->dst_nocount) 1066 flags |= DST_NOCOUNT; 1067 if (rt->dst_nopolicy) 1068 flags |= DST_NOPOLICY; 1069 1070 return flags; 1071 } 1072 1073 static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type) 1074 { 1075 rt->dst.error = ip6_rt_type_to_error(fib6_type); 1076 1077 switch (fib6_type) { 1078 case RTN_BLACKHOLE: 1079 rt->dst.output = dst_discard_out; 1080 rt->dst.input = dst_discard; 1081 break; 1082 case RTN_PROHIBIT: 1083 rt->dst.output = ip6_pkt_prohibit_out; 1084 rt->dst.input = ip6_pkt_prohibit; 1085 break; 1086 case RTN_THROW: 1087 case RTN_UNREACHABLE: 1088 default: 1089 rt->dst.output = ip6_pkt_discard_out; 1090 rt->dst.input = ip6_pkt_discard; 1091 break; 1092 } 1093 } 1094 1095 static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res) 1096 { 1097 struct fib6_info *f6i = res->f6i; 1098 1099 if (res->fib6_flags & RTF_REJECT) { 1100 ip6_rt_init_dst_reject(rt, res->fib6_type); 1101 return; 1102 } 1103 1104 rt->dst.error = 0; 1105 rt->dst.output = ip6_output; 1106 1107 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) { 1108 rt->dst.input = ip6_input; 1109 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 1110 rt->dst.input = ip6_mc_input; 1111 } else { 1112 rt->dst.input = ip6_forward; 1113 } 1114 1115 if (res->nh->fib_nh_lws) { 1116 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws); 1117 lwtunnel_set_redirect(&rt->dst); 1118 } 1119 1120 rt->dst.lastuse = jiffies; 1121 } 1122 1123 /* Caller must already hold reference to @from */ 1124 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from) 1125 { 1126 rt->rt6i_flags &= ~RTF_EXPIRES; 1127 rcu_assign_pointer(rt->from, from); 1128 ip_dst_init_metrics(&rt->dst, from->fib6_metrics); 1129 } 1130 1131 /* Caller must already hold reference to f6i in result */ 1132 static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res) 1133 { 1134 const struct fib6_nh *nh = res->nh; 1135 const struct net_device *dev = nh->fib_nh_dev; 1136 struct fib6_info *f6i = res->f6i; 1137 1138 ip6_rt_init_dst(rt, res); 1139 1140 rt->rt6i_dst = f6i->fib6_dst; 1141 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL; 1142 rt->rt6i_flags = res->fib6_flags; 1143 if (nh->fib_nh_gw_family) { 1144 rt->rt6i_gateway = nh->fib_nh_gw6; 1145 rt->rt6i_flags |= RTF_GATEWAY; 1146 } 1147 rt6_set_from(rt, f6i); 1148 #ifdef CONFIG_IPV6_SUBTREES 1149 rt->rt6i_src = f6i->fib6_src; 1150 #endif 1151 } 1152 1153 static struct fib6_node* fib6_backtrack(struct fib6_node *fn, 1154 struct in6_addr *saddr) 1155 { 1156 struct fib6_node *pn, *sn; 1157 while (1) { 1158 if (fn->fn_flags & RTN_TL_ROOT) 1159 return NULL; 1160 pn = rcu_dereference(fn->parent); 1161 sn = FIB6_SUBTREE(pn); 1162 if (sn && sn != fn) 1163 fn = fib6_node_lookup(sn, NULL, saddr); 1164 else 1165 fn = pn; 1166 if (fn->fn_flags & RTN_RTINFO) 1167 return fn; 1168 } 1169 } 1170 1171 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt) 1172 { 1173 struct rt6_info *rt = *prt; 1174 1175 if (dst_hold_safe(&rt->dst)) 1176 return true; 1177 if (net) { 1178 rt = net->ipv6.ip6_null_entry; 1179 dst_hold(&rt->dst); 1180 } else { 1181 rt = NULL; 1182 } 1183 *prt = rt; 1184 return false; 1185 } 1186 1187 /* called with rcu_lock held */ 1188 static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res) 1189 { 1190 struct net_device *dev = res->nh->fib_nh_dev; 1191 struct fib6_info *f6i = res->f6i; 1192 unsigned short flags; 1193 struct rt6_info *nrt; 1194 1195 if (!fib6_info_hold_safe(f6i)) 1196 goto fallback; 1197 1198 flags = fib6_info_dst_flags(f6i); 1199 nrt = ip6_dst_alloc(dev_net(dev), dev, flags); 1200 if (!nrt) { 1201 fib6_info_release(f6i); 1202 goto fallback; 1203 } 1204 1205 ip6_rt_copy_init(nrt, res); 1206 return nrt; 1207 1208 fallback: 1209 nrt = dev_net(dev)->ipv6.ip6_null_entry; 1210 dst_hold(&nrt->dst); 1211 return nrt; 1212 } 1213 1214 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, 1215 struct fib6_table *table, 1216 struct flowi6 *fl6, 1217 const struct sk_buff *skb, 1218 int flags) 1219 { 1220 struct fib6_result res = {}; 1221 struct fib6_node *fn; 1222 struct rt6_info *rt; 1223 1224 rcu_read_lock(); 1225 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1226 restart: 1227 res.f6i = rcu_dereference(fn->leaf); 1228 if (!res.f6i) 1229 res.f6i = net->ipv6.fib6_null_entry; 1230 else 1231 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif, 1232 flags); 1233 1234 if (res.f6i == net->ipv6.fib6_null_entry) { 1235 fn = fib6_backtrack(fn, &fl6->saddr); 1236 if (fn) 1237 goto restart; 1238 1239 rt = net->ipv6.ip6_null_entry; 1240 dst_hold(&rt->dst); 1241 goto out; 1242 } else if (res.fib6_flags & RTF_REJECT) { 1243 goto do_create; 1244 } 1245 1246 fib6_select_path(net, &res, fl6, fl6->flowi6_oif, 1247 fl6->flowi6_oif != 0, skb, flags); 1248 1249 /* Search through exception table */ 1250 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1251 if (rt) { 1252 if (ip6_hold_safe(net, &rt)) 1253 dst_use_noref(&rt->dst, jiffies); 1254 } else { 1255 do_create: 1256 rt = ip6_create_rt_rcu(&res); 1257 } 1258 1259 out: 1260 trace_fib6_table_lookup(net, &res, table, fl6); 1261 1262 rcu_read_unlock(); 1263 1264 return rt; 1265 } 1266 1267 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6, 1268 const struct sk_buff *skb, int flags) 1269 { 1270 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup); 1271 } 1272 EXPORT_SYMBOL_GPL(ip6_route_lookup); 1273 1274 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr, 1275 const struct in6_addr *saddr, int oif, 1276 const struct sk_buff *skb, int strict) 1277 { 1278 struct flowi6 fl6 = { 1279 .flowi6_oif = oif, 1280 .daddr = *daddr, 1281 }; 1282 struct dst_entry *dst; 1283 int flags = strict ? RT6_LOOKUP_F_IFACE : 0; 1284 1285 if (saddr) { 1286 memcpy(&fl6.saddr, saddr, sizeof(*saddr)); 1287 flags |= RT6_LOOKUP_F_HAS_SADDR; 1288 } 1289 1290 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup); 1291 if (dst->error == 0) 1292 return dst_rt6_info(dst); 1293 1294 dst_release(dst); 1295 1296 return NULL; 1297 } 1298 EXPORT_SYMBOL(rt6_lookup); 1299 1300 /* ip6_ins_rt is called with FREE table->tb6_lock. 1301 * It takes new route entry, the addition fails by any reason the 1302 * route is released. 1303 * Caller must hold dst before calling it. 1304 */ 1305 1306 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info, 1307 struct netlink_ext_ack *extack) 1308 { 1309 int err; 1310 struct fib6_table *table; 1311 1312 table = rt->fib6_table; 1313 spin_lock_bh(&table->tb6_lock); 1314 err = fib6_add(&table->tb6_root, rt, info, extack); 1315 spin_unlock_bh(&table->tb6_lock); 1316 1317 return err; 1318 } 1319 1320 int ip6_ins_rt(struct net *net, struct fib6_info *rt) 1321 { 1322 struct nl_info info = { .nl_net = net, }; 1323 1324 return __ip6_ins_rt(rt, &info, NULL); 1325 } 1326 1327 static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res, 1328 const struct in6_addr *daddr, 1329 const struct in6_addr *saddr) 1330 { 1331 struct fib6_info *f6i = res->f6i; 1332 struct net_device *dev; 1333 struct rt6_info *rt; 1334 1335 /* 1336 * Clone the route. 1337 */ 1338 1339 if (!fib6_info_hold_safe(f6i)) 1340 return NULL; 1341 1342 dev = ip6_rt_get_dev_rcu(res); 1343 rt = ip6_dst_alloc(dev_net(dev), dev, 0); 1344 if (!rt) { 1345 fib6_info_release(f6i); 1346 return NULL; 1347 } 1348 1349 ip6_rt_copy_init(rt, res); 1350 rt->rt6i_flags |= RTF_CACHE; 1351 rt->rt6i_dst.addr = *daddr; 1352 rt->rt6i_dst.plen = 128; 1353 1354 if (!rt6_is_gw_or_nonexthop(res)) { 1355 if (f6i->fib6_dst.plen != 128 && 1356 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr)) 1357 rt->rt6i_flags |= RTF_ANYCAST; 1358 #ifdef CONFIG_IPV6_SUBTREES 1359 if (rt->rt6i_src.plen && saddr) { 1360 rt->rt6i_src.addr = *saddr; 1361 rt->rt6i_src.plen = 128; 1362 } 1363 #endif 1364 } 1365 1366 return rt; 1367 } 1368 1369 static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res) 1370 { 1371 struct fib6_info *f6i = res->f6i; 1372 unsigned short flags = fib6_info_dst_flags(f6i); 1373 struct net_device *dev; 1374 struct rt6_info *pcpu_rt; 1375 1376 if (!fib6_info_hold_safe(f6i)) 1377 return NULL; 1378 1379 rcu_read_lock(); 1380 dev = ip6_rt_get_dev_rcu(res); 1381 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT); 1382 rcu_read_unlock(); 1383 if (!pcpu_rt) { 1384 fib6_info_release(f6i); 1385 return NULL; 1386 } 1387 ip6_rt_copy_init(pcpu_rt, res); 1388 pcpu_rt->rt6i_flags |= RTF_PCPU; 1389 1390 if (f6i->nh) 1391 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev)); 1392 1393 return pcpu_rt; 1394 } 1395 1396 static bool rt6_is_valid(const struct rt6_info *rt6) 1397 { 1398 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev)); 1399 } 1400 1401 /* It should be called with rcu_read_lock() acquired */ 1402 static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) 1403 { 1404 struct rt6_info *pcpu_rt; 1405 1406 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu); 1407 1408 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) { 1409 struct rt6_info *prev, **p; 1410 1411 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1412 prev = xchg(p, NULL); 1413 if (prev) { 1414 dst_dev_put(&prev->dst); 1415 dst_release(&prev->dst); 1416 } 1417 1418 pcpu_rt = NULL; 1419 } 1420 1421 return pcpu_rt; 1422 } 1423 1424 static struct rt6_info *rt6_make_pcpu_route(struct net *net, 1425 const struct fib6_result *res) 1426 { 1427 struct rt6_info *pcpu_rt, *prev, **p; 1428 1429 pcpu_rt = ip6_rt_pcpu_alloc(res); 1430 if (!pcpu_rt) 1431 return NULL; 1432 1433 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1434 prev = cmpxchg(p, NULL, pcpu_rt); 1435 BUG_ON(prev); 1436 1437 if (res->f6i->fib6_destroying) { 1438 struct fib6_info *from; 1439 1440 from = xchg((__force struct fib6_info **)&pcpu_rt->from, NULL); 1441 fib6_info_release(from); 1442 } 1443 1444 return pcpu_rt; 1445 } 1446 1447 /* exception hash table implementation 1448 */ 1449 static DEFINE_SPINLOCK(rt6_exception_lock); 1450 1451 /* Remove rt6_ex from hash table and free the memory 1452 * Caller must hold rt6_exception_lock 1453 */ 1454 static void rt6_remove_exception(struct rt6_exception_bucket *bucket, 1455 struct rt6_exception *rt6_ex) 1456 { 1457 struct fib6_info *from; 1458 struct net *net; 1459 1460 if (!bucket || !rt6_ex) 1461 return; 1462 1463 net = dev_net(rt6_ex->rt6i->dst.dev); 1464 net->ipv6.rt6_stats->fib_rt_cache--; 1465 1466 /* purge completely the exception to allow releasing the held resources: 1467 * some [sk] cache may keep the dst around for unlimited time 1468 */ 1469 from = xchg((__force struct fib6_info **)&rt6_ex->rt6i->from, NULL); 1470 fib6_info_release(from); 1471 dst_dev_put(&rt6_ex->rt6i->dst); 1472 1473 hlist_del_rcu(&rt6_ex->hlist); 1474 dst_release(&rt6_ex->rt6i->dst); 1475 kfree_rcu(rt6_ex, rcu); 1476 WARN_ON_ONCE(!bucket->depth); 1477 bucket->depth--; 1478 } 1479 1480 /* Remove oldest rt6_ex in bucket and free the memory 1481 * Caller must hold rt6_exception_lock 1482 */ 1483 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) 1484 { 1485 struct rt6_exception *rt6_ex, *oldest = NULL; 1486 1487 if (!bucket) 1488 return; 1489 1490 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 1491 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp)) 1492 oldest = rt6_ex; 1493 } 1494 rt6_remove_exception(bucket, oldest); 1495 } 1496 1497 static u32 rt6_exception_hash(const struct in6_addr *dst, 1498 const struct in6_addr *src) 1499 { 1500 static siphash_aligned_key_t rt6_exception_key; 1501 struct { 1502 struct in6_addr dst; 1503 struct in6_addr src; 1504 } __aligned(SIPHASH_ALIGNMENT) combined = { 1505 .dst = *dst, 1506 }; 1507 u64 val; 1508 1509 net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key)); 1510 1511 #ifdef CONFIG_IPV6_SUBTREES 1512 if (src) 1513 combined.src = *src; 1514 #endif 1515 val = siphash(&combined, sizeof(combined), &rt6_exception_key); 1516 1517 return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT); 1518 } 1519 1520 /* Helper function to find the cached rt in the hash table 1521 * and update bucket pointer to point to the bucket for this 1522 * (daddr, saddr) pair 1523 * Caller must hold rt6_exception_lock 1524 */ 1525 static struct rt6_exception * 1526 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket, 1527 const struct in6_addr *daddr, 1528 const struct in6_addr *saddr) 1529 { 1530 struct rt6_exception *rt6_ex; 1531 u32 hval; 1532 1533 if (!(*bucket) || !daddr) 1534 return NULL; 1535 1536 hval = rt6_exception_hash(daddr, saddr); 1537 *bucket += hval; 1538 1539 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) { 1540 struct rt6_info *rt6 = rt6_ex->rt6i; 1541 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1542 1543 #ifdef CONFIG_IPV6_SUBTREES 1544 if (matched && saddr) 1545 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1546 #endif 1547 if (matched) 1548 return rt6_ex; 1549 } 1550 return NULL; 1551 } 1552 1553 /* Helper function to find the cached rt in the hash table 1554 * and update bucket pointer to point to the bucket for this 1555 * (daddr, saddr) pair 1556 * Caller must hold rcu_read_lock() 1557 */ 1558 static struct rt6_exception * 1559 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket, 1560 const struct in6_addr *daddr, 1561 const struct in6_addr *saddr) 1562 { 1563 struct rt6_exception *rt6_ex; 1564 u32 hval; 1565 1566 WARN_ON_ONCE(!rcu_read_lock_held()); 1567 1568 if (!(*bucket) || !daddr) 1569 return NULL; 1570 1571 hval = rt6_exception_hash(daddr, saddr); 1572 *bucket += hval; 1573 1574 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) { 1575 struct rt6_info *rt6 = rt6_ex->rt6i; 1576 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1577 1578 #ifdef CONFIG_IPV6_SUBTREES 1579 if (matched && saddr) 1580 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1581 #endif 1582 if (matched) 1583 return rt6_ex; 1584 } 1585 return NULL; 1586 } 1587 1588 static unsigned int fib6_mtu(const struct fib6_result *res) 1589 { 1590 const struct fib6_nh *nh = res->nh; 1591 unsigned int mtu; 1592 1593 if (res->f6i->fib6_pmtu) { 1594 mtu = res->f6i->fib6_pmtu; 1595 } else { 1596 struct net_device *dev = nh->fib_nh_dev; 1597 struct inet6_dev *idev; 1598 1599 rcu_read_lock(); 1600 idev = __in6_dev_get(dev); 1601 mtu = READ_ONCE(idev->cnf.mtu6); 1602 rcu_read_unlock(); 1603 } 1604 1605 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 1606 1607 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 1608 } 1609 1610 #define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL 1611 1612 /* used when the flushed bit is not relevant, only access to the bucket 1613 * (ie., all bucket users except rt6_insert_exception); 1614 * 1615 * called under rcu lock; sometimes called with rt6_exception_lock held 1616 */ 1617 static 1618 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh, 1619 spinlock_t *lock) 1620 { 1621 struct rt6_exception_bucket *bucket; 1622 1623 if (lock) 1624 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1625 lockdep_is_held(lock)); 1626 else 1627 bucket = rcu_dereference(nh->rt6i_exception_bucket); 1628 1629 /* remove bucket flushed bit if set */ 1630 if (bucket) { 1631 unsigned long p = (unsigned long)bucket; 1632 1633 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED; 1634 bucket = (struct rt6_exception_bucket *)p; 1635 } 1636 1637 return bucket; 1638 } 1639 1640 static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket) 1641 { 1642 unsigned long p = (unsigned long)bucket; 1643 1644 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED); 1645 } 1646 1647 /* called with rt6_exception_lock held */ 1648 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh, 1649 spinlock_t *lock) 1650 { 1651 struct rt6_exception_bucket *bucket; 1652 unsigned long p; 1653 1654 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1655 lockdep_is_held(lock)); 1656 1657 p = (unsigned long)bucket; 1658 p |= FIB6_EXCEPTION_BUCKET_FLUSHED; 1659 bucket = (struct rt6_exception_bucket *)p; 1660 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1661 } 1662 1663 static int rt6_insert_exception(struct rt6_info *nrt, 1664 const struct fib6_result *res) 1665 { 1666 struct net *net = dev_net(nrt->dst.dev); 1667 struct rt6_exception_bucket *bucket; 1668 struct fib6_info *f6i = res->f6i; 1669 struct in6_addr *src_key = NULL; 1670 struct rt6_exception *rt6_ex; 1671 struct fib6_nh *nh = res->nh; 1672 int max_depth; 1673 int err = 0; 1674 1675 spin_lock_bh(&rt6_exception_lock); 1676 1677 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1678 lockdep_is_held(&rt6_exception_lock)); 1679 if (!bucket) { 1680 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket), 1681 GFP_ATOMIC); 1682 if (!bucket) { 1683 err = -ENOMEM; 1684 goto out; 1685 } 1686 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1687 } else if (fib6_nh_excptn_bucket_flushed(bucket)) { 1688 err = -EINVAL; 1689 goto out; 1690 } 1691 1692 #ifdef CONFIG_IPV6_SUBTREES 1693 /* fib6_src.plen != 0 indicates f6i is in subtree 1694 * and exception table is indexed by a hash of 1695 * both fib6_dst and fib6_src. 1696 * Otherwise, the exception table is indexed by 1697 * a hash of only fib6_dst. 1698 */ 1699 if (f6i->fib6_src.plen) 1700 src_key = &nrt->rt6i_src.addr; 1701 #endif 1702 /* rt6_mtu_change() might lower mtu on f6i. 1703 * Only insert this exception route if its mtu 1704 * is less than f6i's mtu value. 1705 */ 1706 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) { 1707 err = -EINVAL; 1708 goto out; 1709 } 1710 1711 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr, 1712 src_key); 1713 if (rt6_ex) 1714 rt6_remove_exception(bucket, rt6_ex); 1715 1716 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC); 1717 if (!rt6_ex) { 1718 err = -ENOMEM; 1719 goto out; 1720 } 1721 rt6_ex->rt6i = nrt; 1722 rt6_ex->stamp = jiffies; 1723 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain); 1724 bucket->depth++; 1725 net->ipv6.rt6_stats->fib_rt_cache++; 1726 1727 /* Randomize max depth to avoid some side channels attacks. */ 1728 max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH); 1729 while (bucket->depth > max_depth) 1730 rt6_exception_remove_oldest(bucket); 1731 1732 out: 1733 spin_unlock_bh(&rt6_exception_lock); 1734 1735 /* Update fn->fn_sernum to invalidate all cached dst */ 1736 if (!err) { 1737 spin_lock_bh(&f6i->fib6_table->tb6_lock); 1738 fib6_update_sernum(net, f6i); 1739 spin_unlock_bh(&f6i->fib6_table->tb6_lock); 1740 fib6_force_start_gc(net); 1741 } 1742 1743 return err; 1744 } 1745 1746 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from) 1747 { 1748 struct rt6_exception_bucket *bucket; 1749 struct rt6_exception *rt6_ex; 1750 struct hlist_node *tmp; 1751 int i; 1752 1753 spin_lock_bh(&rt6_exception_lock); 1754 1755 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1756 if (!bucket) 1757 goto out; 1758 1759 /* Prevent rt6_insert_exception() to recreate the bucket list */ 1760 if (!from) 1761 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock); 1762 1763 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 1764 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) { 1765 if (!from || 1766 rcu_access_pointer(rt6_ex->rt6i->from) == from) 1767 rt6_remove_exception(bucket, rt6_ex); 1768 } 1769 WARN_ON_ONCE(!from && bucket->depth); 1770 bucket++; 1771 } 1772 out: 1773 spin_unlock_bh(&rt6_exception_lock); 1774 } 1775 1776 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg) 1777 { 1778 struct fib6_info *f6i = arg; 1779 1780 fib6_nh_flush_exceptions(nh, f6i); 1781 1782 return 0; 1783 } 1784 1785 void rt6_flush_exceptions(struct fib6_info *f6i) 1786 { 1787 if (f6i->nh) 1788 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions, 1789 f6i); 1790 else 1791 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i); 1792 } 1793 1794 /* Find cached rt in the hash table inside passed in rt 1795 * Caller has to hold rcu_read_lock() 1796 */ 1797 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 1798 const struct in6_addr *daddr, 1799 const struct in6_addr *saddr) 1800 { 1801 const struct in6_addr *src_key = NULL; 1802 struct rt6_exception_bucket *bucket; 1803 struct rt6_exception *rt6_ex; 1804 struct rt6_info *ret = NULL; 1805 1806 #ifdef CONFIG_IPV6_SUBTREES 1807 /* fib6i_src.plen != 0 indicates f6i is in subtree 1808 * and exception table is indexed by a hash of 1809 * both fib6_dst and fib6_src. 1810 * However, the src addr used to create the hash 1811 * might not be exactly the passed in saddr which 1812 * is a /128 addr from the flow. 1813 * So we need to use f6i->fib6_src to redo lookup 1814 * if the passed in saddr does not find anything. 1815 * (See the logic in ip6_rt_cache_alloc() on how 1816 * rt->rt6i_src is updated.) 1817 */ 1818 if (res->f6i->fib6_src.plen) 1819 src_key = saddr; 1820 find_ex: 1821 #endif 1822 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL); 1823 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key); 1824 1825 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i)) 1826 ret = rt6_ex->rt6i; 1827 1828 #ifdef CONFIG_IPV6_SUBTREES 1829 /* Use fib6_src as src_key and redo lookup */ 1830 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) { 1831 src_key = &res->f6i->fib6_src.addr; 1832 goto find_ex; 1833 } 1834 #endif 1835 1836 return ret; 1837 } 1838 1839 /* Remove the passed in cached rt from the hash table that contains it */ 1840 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen, 1841 const struct rt6_info *rt) 1842 { 1843 const struct in6_addr *src_key = NULL; 1844 struct rt6_exception_bucket *bucket; 1845 struct rt6_exception *rt6_ex; 1846 int err; 1847 1848 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 1849 return -ENOENT; 1850 1851 spin_lock_bh(&rt6_exception_lock); 1852 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1853 1854 #ifdef CONFIG_IPV6_SUBTREES 1855 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1856 * and exception table is indexed by a hash of 1857 * both rt6i_dst and rt6i_src. 1858 * Otherwise, the exception table is indexed by 1859 * a hash of only rt6i_dst. 1860 */ 1861 if (plen) 1862 src_key = &rt->rt6i_src.addr; 1863 #endif 1864 rt6_ex = __rt6_find_exception_spinlock(&bucket, 1865 &rt->rt6i_dst.addr, 1866 src_key); 1867 if (rt6_ex) { 1868 rt6_remove_exception(bucket, rt6_ex); 1869 err = 0; 1870 } else { 1871 err = -ENOENT; 1872 } 1873 1874 spin_unlock_bh(&rt6_exception_lock); 1875 return err; 1876 } 1877 1878 struct fib6_nh_excptn_arg { 1879 struct rt6_info *rt; 1880 int plen; 1881 }; 1882 1883 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg) 1884 { 1885 struct fib6_nh_excptn_arg *arg = _arg; 1886 int err; 1887 1888 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt); 1889 if (err == 0) 1890 return 1; 1891 1892 return 0; 1893 } 1894 1895 static int rt6_remove_exception_rt(struct rt6_info *rt) 1896 { 1897 struct fib6_info *from; 1898 1899 from = rcu_dereference(rt->from); 1900 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1901 return -EINVAL; 1902 1903 if (from->nh) { 1904 struct fib6_nh_excptn_arg arg = { 1905 .rt = rt, 1906 .plen = from->fib6_src.plen 1907 }; 1908 int rc; 1909 1910 /* rc = 1 means an entry was found */ 1911 rc = nexthop_for_each_fib6_nh(from->nh, 1912 rt6_nh_remove_exception_rt, 1913 &arg); 1914 return rc ? 0 : -ENOENT; 1915 } 1916 1917 return fib6_nh_remove_exception(from->fib6_nh, 1918 from->fib6_src.plen, rt); 1919 } 1920 1921 /* Find rt6_ex which contains the passed in rt cache and 1922 * refresh its stamp 1923 */ 1924 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen, 1925 const struct rt6_info *rt) 1926 { 1927 const struct in6_addr *src_key = NULL; 1928 struct rt6_exception_bucket *bucket; 1929 struct rt6_exception *rt6_ex; 1930 1931 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 1932 #ifdef CONFIG_IPV6_SUBTREES 1933 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1934 * and exception table is indexed by a hash of 1935 * both rt6i_dst and rt6i_src. 1936 * Otherwise, the exception table is indexed by 1937 * a hash of only rt6i_dst. 1938 */ 1939 if (plen) 1940 src_key = &rt->rt6i_src.addr; 1941 #endif 1942 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key); 1943 if (rt6_ex) 1944 rt6_ex->stamp = jiffies; 1945 } 1946 1947 struct fib6_nh_match_arg { 1948 const struct net_device *dev; 1949 const struct in6_addr *gw; 1950 struct fib6_nh *match; 1951 }; 1952 1953 /* determine if fib6_nh has given device and gateway */ 1954 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg) 1955 { 1956 struct fib6_nh_match_arg *arg = _arg; 1957 1958 if (arg->dev != nh->fib_nh_dev || 1959 (arg->gw && !nh->fib_nh_gw_family) || 1960 (!arg->gw && nh->fib_nh_gw_family) || 1961 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6))) 1962 return 0; 1963 1964 arg->match = nh; 1965 1966 /* found a match, break the loop */ 1967 return 1; 1968 } 1969 1970 static void rt6_update_exception_stamp_rt(struct rt6_info *rt) 1971 { 1972 struct fib6_info *from; 1973 struct fib6_nh *fib6_nh; 1974 1975 rcu_read_lock(); 1976 1977 from = rcu_dereference(rt->from); 1978 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1979 goto unlock; 1980 1981 if (from->nh) { 1982 struct fib6_nh_match_arg arg = { 1983 .dev = rt->dst.dev, 1984 .gw = &rt->rt6i_gateway, 1985 }; 1986 1987 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg); 1988 1989 if (!arg.match) 1990 goto unlock; 1991 fib6_nh = arg.match; 1992 } else { 1993 fib6_nh = from->fib6_nh; 1994 } 1995 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt); 1996 unlock: 1997 rcu_read_unlock(); 1998 } 1999 2000 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, 2001 struct rt6_info *rt, int mtu) 2002 { 2003 /* If the new MTU is lower than the route PMTU, this new MTU will be the 2004 * lowest MTU in the path: always allow updating the route PMTU to 2005 * reflect PMTU decreases. 2006 * 2007 * If the new MTU is higher, and the route PMTU is equal to the local 2008 * MTU, this means the old MTU is the lowest in the path, so allow 2009 * updating it: if other nodes now have lower MTUs, PMTU discovery will 2010 * handle this. 2011 */ 2012 2013 if (dst_mtu(&rt->dst) >= mtu) 2014 return true; 2015 2016 if (dst_mtu(&rt->dst) == idev->cnf.mtu6) 2017 return true; 2018 2019 return false; 2020 } 2021 2022 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev, 2023 const struct fib6_nh *nh, int mtu) 2024 { 2025 struct rt6_exception_bucket *bucket; 2026 struct rt6_exception *rt6_ex; 2027 int i; 2028 2029 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2030 if (!bucket) 2031 return; 2032 2033 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2034 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 2035 struct rt6_info *entry = rt6_ex->rt6i; 2036 2037 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected 2038 * route), the metrics of its rt->from have already 2039 * been updated. 2040 */ 2041 if (dst_metric_raw(&entry->dst, RTAX_MTU) && 2042 rt6_mtu_change_route_allowed(idev, entry, mtu)) 2043 dst_metric_set(&entry->dst, RTAX_MTU, mtu); 2044 } 2045 bucket++; 2046 } 2047 } 2048 2049 #define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE) 2050 2051 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh, 2052 const struct in6_addr *gateway) 2053 { 2054 struct rt6_exception_bucket *bucket; 2055 struct rt6_exception *rt6_ex; 2056 struct hlist_node *tmp; 2057 int i; 2058 2059 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2060 return; 2061 2062 spin_lock_bh(&rt6_exception_lock); 2063 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2064 if (bucket) { 2065 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2066 hlist_for_each_entry_safe(rt6_ex, tmp, 2067 &bucket->chain, hlist) { 2068 struct rt6_info *entry = rt6_ex->rt6i; 2069 2070 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) == 2071 RTF_CACHE_GATEWAY && 2072 ipv6_addr_equal(gateway, 2073 &entry->rt6i_gateway)) { 2074 rt6_remove_exception(bucket, rt6_ex); 2075 } 2076 } 2077 bucket++; 2078 } 2079 } 2080 2081 spin_unlock_bh(&rt6_exception_lock); 2082 } 2083 2084 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket, 2085 struct rt6_exception *rt6_ex, 2086 struct fib6_gc_args *gc_args, 2087 unsigned long now) 2088 { 2089 struct rt6_info *rt = rt6_ex->rt6i; 2090 2091 /* we are pruning and obsoleting aged-out and non gateway exceptions 2092 * even if others have still references to them, so that on next 2093 * dst_check() such references can be dropped. 2094 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when 2095 * expired, independently from their aging, as per RFC 8201 section 4 2096 */ 2097 if (!(rt->rt6i_flags & RTF_EXPIRES)) { 2098 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) { 2099 pr_debug("aging clone %p\n", rt); 2100 rt6_remove_exception(bucket, rt6_ex); 2101 return; 2102 } 2103 } else if (time_after(jiffies, rt->dst.expires)) { 2104 pr_debug("purging expired route %p\n", rt); 2105 rt6_remove_exception(bucket, rt6_ex); 2106 return; 2107 } 2108 2109 if (rt->rt6i_flags & RTF_GATEWAY) { 2110 struct neighbour *neigh; 2111 2112 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway); 2113 2114 if (!(neigh && (neigh->flags & NTF_ROUTER))) { 2115 pr_debug("purging route %p via non-router but gateway\n", 2116 rt); 2117 rt6_remove_exception(bucket, rt6_ex); 2118 return; 2119 } 2120 } 2121 2122 gc_args->more++; 2123 } 2124 2125 static void fib6_nh_age_exceptions(const struct fib6_nh *nh, 2126 struct fib6_gc_args *gc_args, 2127 unsigned long now) 2128 { 2129 struct rt6_exception_bucket *bucket; 2130 struct rt6_exception *rt6_ex; 2131 struct hlist_node *tmp; 2132 int i; 2133 2134 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2135 return; 2136 2137 rcu_read_lock_bh(); 2138 spin_lock(&rt6_exception_lock); 2139 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2140 if (bucket) { 2141 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2142 hlist_for_each_entry_safe(rt6_ex, tmp, 2143 &bucket->chain, hlist) { 2144 rt6_age_examine_exception(bucket, rt6_ex, 2145 gc_args, now); 2146 } 2147 bucket++; 2148 } 2149 } 2150 spin_unlock(&rt6_exception_lock); 2151 rcu_read_unlock_bh(); 2152 } 2153 2154 struct fib6_nh_age_excptn_arg { 2155 struct fib6_gc_args *gc_args; 2156 unsigned long now; 2157 }; 2158 2159 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg) 2160 { 2161 struct fib6_nh_age_excptn_arg *arg = _arg; 2162 2163 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now); 2164 return 0; 2165 } 2166 2167 void rt6_age_exceptions(struct fib6_info *f6i, 2168 struct fib6_gc_args *gc_args, 2169 unsigned long now) 2170 { 2171 if (f6i->nh) { 2172 struct fib6_nh_age_excptn_arg arg = { 2173 .gc_args = gc_args, 2174 .now = now 2175 }; 2176 2177 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions, 2178 &arg); 2179 } else { 2180 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now); 2181 } 2182 } 2183 2184 /* must be called with rcu lock held */ 2185 int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif, 2186 struct flowi6 *fl6, struct fib6_result *res, int strict) 2187 { 2188 struct fib6_node *fn, *saved_fn; 2189 2190 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2191 saved_fn = fn; 2192 2193 redo_rt6_select: 2194 rt6_select(net, fn, oif, res, strict); 2195 if (res->f6i == net->ipv6.fib6_null_entry) { 2196 fn = fib6_backtrack(fn, &fl6->saddr); 2197 if (fn) 2198 goto redo_rt6_select; 2199 else if (strict & RT6_LOOKUP_F_REACHABLE) { 2200 /* also consider unreachable route */ 2201 strict &= ~RT6_LOOKUP_F_REACHABLE; 2202 fn = saved_fn; 2203 goto redo_rt6_select; 2204 } 2205 } 2206 2207 trace_fib6_table_lookup(net, res, table, fl6); 2208 2209 return 0; 2210 } 2211 2212 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, 2213 int oif, struct flowi6 *fl6, 2214 const struct sk_buff *skb, int flags) 2215 { 2216 struct fib6_result res = {}; 2217 struct rt6_info *rt = NULL; 2218 int strict = 0; 2219 2220 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) && 2221 !rcu_read_lock_held()); 2222 2223 strict |= flags & RT6_LOOKUP_F_IFACE; 2224 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE; 2225 if (READ_ONCE(net->ipv6.devconf_all->forwarding) == 0) 2226 strict |= RT6_LOOKUP_F_REACHABLE; 2227 2228 rcu_read_lock(); 2229 2230 fib6_table_lookup(net, table, oif, fl6, &res, strict); 2231 if (res.f6i == net->ipv6.fib6_null_entry) 2232 goto out; 2233 2234 fib6_select_path(net, &res, fl6, oif, false, skb, strict); 2235 2236 /*Search through exception table */ 2237 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 2238 if (rt) { 2239 goto out; 2240 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) && 2241 !res.nh->fib_nh_gw_family)) { 2242 /* Create a RTF_CACHE clone which will not be 2243 * owned by the fib6 tree. It is for the special case where 2244 * the daddr in the skb during the neighbor look-up is different 2245 * from the fl6->daddr used to look-up route here. 2246 */ 2247 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL); 2248 2249 if (rt) { 2250 /* 1 refcnt is taken during ip6_rt_cache_alloc(). 2251 * As rt6_uncached_list_add() does not consume refcnt, 2252 * this refcnt is always returned to the caller even 2253 * if caller sets RT6_LOOKUP_F_DST_NOREF flag. 2254 */ 2255 rt6_uncached_list_add(rt); 2256 rcu_read_unlock(); 2257 2258 return rt; 2259 } 2260 } else { 2261 /* Get a percpu copy */ 2262 local_bh_disable(); 2263 rt = rt6_get_pcpu_route(&res); 2264 2265 if (!rt) 2266 rt = rt6_make_pcpu_route(net, &res); 2267 2268 local_bh_enable(); 2269 } 2270 out: 2271 if (!rt) 2272 rt = net->ipv6.ip6_null_entry; 2273 if (!(flags & RT6_LOOKUP_F_DST_NOREF)) 2274 ip6_hold_safe(net, &rt); 2275 rcu_read_unlock(); 2276 2277 return rt; 2278 } 2279 EXPORT_SYMBOL_GPL(ip6_pol_route); 2280 2281 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, 2282 struct fib6_table *table, 2283 struct flowi6 *fl6, 2284 const struct sk_buff *skb, 2285 int flags) 2286 { 2287 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags); 2288 } 2289 2290 struct dst_entry *ip6_route_input_lookup(struct net *net, 2291 struct net_device *dev, 2292 struct flowi6 *fl6, 2293 const struct sk_buff *skb, 2294 int flags) 2295 { 2296 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG) 2297 flags |= RT6_LOOKUP_F_IFACE; 2298 2299 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input); 2300 } 2301 EXPORT_SYMBOL_GPL(ip6_route_input_lookup); 2302 2303 static void ip6_multipath_l3_keys(const struct sk_buff *skb, 2304 struct flow_keys *keys, 2305 struct flow_keys *flkeys) 2306 { 2307 const struct ipv6hdr *outer_iph = ipv6_hdr(skb); 2308 const struct ipv6hdr *key_iph = outer_iph; 2309 struct flow_keys *_flkeys = flkeys; 2310 const struct ipv6hdr *inner_iph; 2311 const struct icmp6hdr *icmph; 2312 struct ipv6hdr _inner_iph; 2313 struct icmp6hdr _icmph; 2314 2315 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6)) 2316 goto out; 2317 2318 icmph = skb_header_pointer(skb, skb_transport_offset(skb), 2319 sizeof(_icmph), &_icmph); 2320 if (!icmph) 2321 goto out; 2322 2323 if (!icmpv6_is_err(icmph->icmp6_type)) 2324 goto out; 2325 2326 inner_iph = skb_header_pointer(skb, 2327 skb_transport_offset(skb) + sizeof(*icmph), 2328 sizeof(_inner_iph), &_inner_iph); 2329 if (!inner_iph) 2330 goto out; 2331 2332 key_iph = inner_iph; 2333 _flkeys = NULL; 2334 out: 2335 if (_flkeys) { 2336 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src; 2337 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst; 2338 keys->tags.flow_label = _flkeys->tags.flow_label; 2339 keys->basic.ip_proto = _flkeys->basic.ip_proto; 2340 } else { 2341 keys->addrs.v6addrs.src = key_iph->saddr; 2342 keys->addrs.v6addrs.dst = key_iph->daddr; 2343 keys->tags.flow_label = ip6_flowlabel(key_iph); 2344 keys->basic.ip_proto = key_iph->nexthdr; 2345 } 2346 } 2347 2348 static u32 rt6_multipath_custom_hash_outer(const struct net *net, 2349 const struct sk_buff *skb, 2350 bool *p_has_inner) 2351 { 2352 u32 hash_fields = ip6_multipath_hash_fields(net); 2353 struct flow_keys keys, hash_keys; 2354 2355 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2356 return 0; 2357 2358 memset(&hash_keys, 0, sizeof(hash_keys)); 2359 skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP); 2360 2361 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2362 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2363 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2364 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2365 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2366 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2367 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2368 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2369 hash_keys.tags.flow_label = keys.tags.flow_label; 2370 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2371 hash_keys.ports.src = keys.ports.src; 2372 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2373 hash_keys.ports.dst = keys.ports.dst; 2374 2375 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION); 2376 return flow_hash_from_keys(&hash_keys); 2377 } 2378 2379 static u32 rt6_multipath_custom_hash_inner(const struct net *net, 2380 const struct sk_buff *skb, 2381 bool has_inner) 2382 { 2383 u32 hash_fields = ip6_multipath_hash_fields(net); 2384 struct flow_keys keys, hash_keys; 2385 2386 /* We assume the packet carries an encapsulation, but if none was 2387 * encountered during dissection of the outer flow, then there is no 2388 * point in calling the flow dissector again. 2389 */ 2390 if (!has_inner) 2391 return 0; 2392 2393 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)) 2394 return 0; 2395 2396 memset(&hash_keys, 0, sizeof(hash_keys)); 2397 skb_flow_dissect_flow_keys(skb, &keys, 0); 2398 2399 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION)) 2400 return 0; 2401 2402 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2403 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2404 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2405 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 2406 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2407 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 2408 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2409 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2410 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2411 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2412 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2413 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2414 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 2415 hash_keys.tags.flow_label = keys.tags.flow_label; 2416 } 2417 2418 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 2419 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2420 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 2421 hash_keys.ports.src = keys.ports.src; 2422 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 2423 hash_keys.ports.dst = keys.ports.dst; 2424 2425 return flow_hash_from_keys(&hash_keys); 2426 } 2427 2428 static u32 rt6_multipath_custom_hash_skb(const struct net *net, 2429 const struct sk_buff *skb) 2430 { 2431 u32 mhash, mhash_inner; 2432 bool has_inner = true; 2433 2434 mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner); 2435 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner); 2436 2437 return jhash_2words(mhash, mhash_inner, 0); 2438 } 2439 2440 static u32 rt6_multipath_custom_hash_fl6(const struct net *net, 2441 const struct flowi6 *fl6) 2442 { 2443 u32 hash_fields = ip6_multipath_hash_fields(net); 2444 struct flow_keys hash_keys; 2445 2446 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2447 return 0; 2448 2449 memset(&hash_keys, 0, sizeof(hash_keys)); 2450 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2451 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2452 hash_keys.addrs.v6addrs.src = fl6->saddr; 2453 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2454 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2455 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2456 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2457 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2458 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2459 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2460 hash_keys.ports.src = fl6->fl6_sport; 2461 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2462 hash_keys.ports.dst = fl6->fl6_dport; 2463 2464 return flow_hash_from_keys(&hash_keys); 2465 } 2466 2467 /* if skb is set it will be used and fl6 can be NULL */ 2468 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6, 2469 const struct sk_buff *skb, struct flow_keys *flkeys) 2470 { 2471 struct flow_keys hash_keys; 2472 u32 mhash = 0; 2473 2474 switch (ip6_multipath_hash_policy(net)) { 2475 case 0: 2476 memset(&hash_keys, 0, sizeof(hash_keys)); 2477 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2478 if (skb) { 2479 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2480 } else { 2481 hash_keys.addrs.v6addrs.src = fl6->saddr; 2482 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2483 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2484 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2485 } 2486 mhash = flow_hash_from_keys(&hash_keys); 2487 break; 2488 case 1: 2489 if (skb) { 2490 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 2491 struct flow_keys keys; 2492 2493 /* short-circuit if we already have L4 hash present */ 2494 if (skb->l4_hash) 2495 return skb_get_hash_raw(skb) >> 1; 2496 2497 memset(&hash_keys, 0, sizeof(hash_keys)); 2498 2499 if (!flkeys) { 2500 skb_flow_dissect_flow_keys(skb, &keys, flag); 2501 flkeys = &keys; 2502 } 2503 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2504 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2505 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2506 hash_keys.ports.src = flkeys->ports.src; 2507 hash_keys.ports.dst = flkeys->ports.dst; 2508 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2509 } else { 2510 memset(&hash_keys, 0, sizeof(hash_keys)); 2511 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2512 hash_keys.addrs.v6addrs.src = fl6->saddr; 2513 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2514 hash_keys.ports.src = fl6->fl6_sport; 2515 hash_keys.ports.dst = fl6->fl6_dport; 2516 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2517 } 2518 mhash = flow_hash_from_keys(&hash_keys); 2519 break; 2520 case 2: 2521 memset(&hash_keys, 0, sizeof(hash_keys)); 2522 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2523 if (skb) { 2524 struct flow_keys keys; 2525 2526 if (!flkeys) { 2527 skb_flow_dissect_flow_keys(skb, &keys, 0); 2528 flkeys = &keys; 2529 } 2530 2531 /* Inner can be v4 or v6 */ 2532 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2533 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2534 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 2535 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 2536 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2537 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2538 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2539 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2540 hash_keys.tags.flow_label = flkeys->tags.flow_label; 2541 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2542 } else { 2543 /* Same as case 0 */ 2544 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2545 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2546 } 2547 } else { 2548 /* Same as case 0 */ 2549 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2550 hash_keys.addrs.v6addrs.src = fl6->saddr; 2551 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2552 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2553 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2554 } 2555 mhash = flow_hash_from_keys(&hash_keys); 2556 break; 2557 case 3: 2558 if (skb) 2559 mhash = rt6_multipath_custom_hash_skb(net, skb); 2560 else 2561 mhash = rt6_multipath_custom_hash_fl6(net, fl6); 2562 break; 2563 } 2564 2565 return mhash >> 1; 2566 } 2567 2568 /* Called with rcu held */ 2569 void ip6_route_input(struct sk_buff *skb) 2570 { 2571 const struct ipv6hdr *iph = ipv6_hdr(skb); 2572 struct net *net = dev_net(skb->dev); 2573 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF; 2574 struct ip_tunnel_info *tun_info; 2575 struct flowi6 fl6 = { 2576 .flowi6_iif = skb->dev->ifindex, 2577 .daddr = iph->daddr, 2578 .saddr = iph->saddr, 2579 .flowlabel = ip6_flowinfo(iph), 2580 .flowi6_mark = skb->mark, 2581 .flowi6_proto = iph->nexthdr, 2582 }; 2583 struct flow_keys *flkeys = NULL, _flkeys; 2584 2585 tun_info = skb_tunnel_info(skb); 2586 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2587 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id; 2588 2589 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys)) 2590 flkeys = &_flkeys; 2591 2592 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6)) 2593 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys); 2594 skb_dst_drop(skb); 2595 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev, 2596 &fl6, skb, flags)); 2597 } 2598 2599 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, 2600 struct fib6_table *table, 2601 struct flowi6 *fl6, 2602 const struct sk_buff *skb, 2603 int flags) 2604 { 2605 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags); 2606 } 2607 2608 static struct dst_entry *ip6_route_output_flags_noref(struct net *net, 2609 const struct sock *sk, 2610 struct flowi6 *fl6, 2611 int flags) 2612 { 2613 bool any_src; 2614 2615 if (ipv6_addr_type(&fl6->daddr) & 2616 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) { 2617 struct dst_entry *dst; 2618 2619 /* This function does not take refcnt on the dst */ 2620 dst = l3mdev_link_scope_lookup(net, fl6); 2621 if (dst) 2622 return dst; 2623 } 2624 2625 fl6->flowi6_iif = LOOPBACK_IFINDEX; 2626 2627 flags |= RT6_LOOKUP_F_DST_NOREF; 2628 any_src = ipv6_addr_any(&fl6->saddr); 2629 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) || 2630 (fl6->flowi6_oif && any_src)) 2631 flags |= RT6_LOOKUP_F_IFACE; 2632 2633 if (!any_src) 2634 flags |= RT6_LOOKUP_F_HAS_SADDR; 2635 else if (sk) 2636 flags |= rt6_srcprefs2flags(READ_ONCE(inet6_sk(sk)->srcprefs)); 2637 2638 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output); 2639 } 2640 2641 struct dst_entry *ip6_route_output_flags(struct net *net, 2642 const struct sock *sk, 2643 struct flowi6 *fl6, 2644 int flags) 2645 { 2646 struct dst_entry *dst; 2647 struct rt6_info *rt6; 2648 2649 rcu_read_lock(); 2650 dst = ip6_route_output_flags_noref(net, sk, fl6, flags); 2651 rt6 = dst_rt6_info(dst); 2652 /* For dst cached in uncached_list, refcnt is already taken. */ 2653 if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) { 2654 dst = &net->ipv6.ip6_null_entry->dst; 2655 dst_hold(dst); 2656 } 2657 rcu_read_unlock(); 2658 2659 return dst; 2660 } 2661 EXPORT_SYMBOL_GPL(ip6_route_output_flags); 2662 2663 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2664 { 2665 struct rt6_info *rt, *ort = dst_rt6_info(dst_orig); 2666 struct net_device *loopback_dev = net->loopback_dev; 2667 struct dst_entry *new = NULL; 2668 2669 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 2670 DST_OBSOLETE_DEAD, 0); 2671 if (rt) { 2672 rt6_info_init(rt); 2673 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 2674 2675 new = &rt->dst; 2676 new->__use = 1; 2677 new->input = dst_discard; 2678 new->output = dst_discard_out; 2679 2680 dst_copy_metrics(new, &ort->dst); 2681 2682 rt->rt6i_idev = in6_dev_get(loopback_dev); 2683 rt->rt6i_gateway = ort->rt6i_gateway; 2684 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU; 2685 2686 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key)); 2687 #ifdef CONFIG_IPV6_SUBTREES 2688 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key)); 2689 #endif 2690 } 2691 2692 dst_release(dst_orig); 2693 return new ? new : ERR_PTR(-ENOMEM); 2694 } 2695 2696 /* 2697 * Destination cache support functions 2698 */ 2699 2700 static bool fib6_check(struct fib6_info *f6i, u32 cookie) 2701 { 2702 u32 rt_cookie = 0; 2703 2704 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie) 2705 return false; 2706 2707 if (fib6_check_expired(f6i)) 2708 return false; 2709 2710 return true; 2711 } 2712 2713 static struct dst_entry *rt6_check(struct rt6_info *rt, 2714 struct fib6_info *from, 2715 u32 cookie) 2716 { 2717 u32 rt_cookie = 0; 2718 2719 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) || 2720 rt_cookie != cookie) 2721 return NULL; 2722 2723 if (rt6_check_expired(rt)) 2724 return NULL; 2725 2726 return &rt->dst; 2727 } 2728 2729 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, 2730 struct fib6_info *from, 2731 u32 cookie) 2732 { 2733 if (!__rt6_check_expired(rt) && 2734 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 2735 fib6_check(from, cookie)) 2736 return &rt->dst; 2737 else 2738 return NULL; 2739 } 2740 2741 INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, 2742 u32 cookie) 2743 { 2744 struct dst_entry *dst_ret; 2745 struct fib6_info *from; 2746 struct rt6_info *rt; 2747 2748 rt = dst_rt6_info(dst); 2749 2750 if (rt->sernum) 2751 return rt6_is_valid(rt) ? dst : NULL; 2752 2753 rcu_read_lock(); 2754 2755 /* All IPV6 dsts are created with ->obsolete set to the value 2756 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 2757 * into this function always. 2758 */ 2759 2760 from = rcu_dereference(rt->from); 2761 2762 if (from && (rt->rt6i_flags & RTF_PCPU || 2763 unlikely(!list_empty(&rt->dst.rt_uncached)))) 2764 dst_ret = rt6_dst_from_check(rt, from, cookie); 2765 else 2766 dst_ret = rt6_check(rt, from, cookie); 2767 2768 rcu_read_unlock(); 2769 2770 return dst_ret; 2771 } 2772 EXPORT_INDIRECT_CALLABLE(ip6_dst_check); 2773 2774 static void ip6_negative_advice(struct sock *sk, 2775 struct dst_entry *dst) 2776 { 2777 struct rt6_info *rt = dst_rt6_info(dst); 2778 2779 if (rt->rt6i_flags & RTF_CACHE) { 2780 rcu_read_lock(); 2781 if (rt6_check_expired(rt)) { 2782 /* counteract the dst_release() in sk_dst_reset() */ 2783 dst_hold(dst); 2784 sk_dst_reset(sk); 2785 2786 rt6_remove_exception_rt(rt); 2787 } 2788 rcu_read_unlock(); 2789 return; 2790 } 2791 sk_dst_reset(sk); 2792 } 2793 2794 static void ip6_link_failure(struct sk_buff *skb) 2795 { 2796 struct rt6_info *rt; 2797 2798 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 2799 2800 rt = dst_rt6_info(skb_dst(skb)); 2801 if (rt) { 2802 rcu_read_lock(); 2803 if (rt->rt6i_flags & RTF_CACHE) { 2804 rt6_remove_exception_rt(rt); 2805 } else { 2806 struct fib6_info *from; 2807 struct fib6_node *fn; 2808 2809 from = rcu_dereference(rt->from); 2810 if (from) { 2811 fn = rcu_dereference(from->fib6_node); 2812 if (fn && (rt->rt6i_flags & RTF_DEFAULT)) 2813 WRITE_ONCE(fn->fn_sernum, -1); 2814 } 2815 } 2816 rcu_read_unlock(); 2817 } 2818 } 2819 2820 static void rt6_update_expires(struct rt6_info *rt0, int timeout) 2821 { 2822 if (!(rt0->rt6i_flags & RTF_EXPIRES)) { 2823 struct fib6_info *from; 2824 2825 rcu_read_lock(); 2826 from = rcu_dereference(rt0->from); 2827 if (from) 2828 rt0->dst.expires = from->expires; 2829 rcu_read_unlock(); 2830 } 2831 2832 dst_set_expires(&rt0->dst, timeout); 2833 rt0->rt6i_flags |= RTF_EXPIRES; 2834 } 2835 2836 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) 2837 { 2838 struct net *net = dev_net(rt->dst.dev); 2839 2840 dst_metric_set(&rt->dst, RTAX_MTU, mtu); 2841 rt->rt6i_flags |= RTF_MODIFIED; 2842 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); 2843 } 2844 2845 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) 2846 { 2847 return !(rt->rt6i_flags & RTF_CACHE) && 2848 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from)); 2849 } 2850 2851 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk, 2852 const struct ipv6hdr *iph, u32 mtu, 2853 bool confirm_neigh) 2854 { 2855 const struct in6_addr *daddr, *saddr; 2856 struct rt6_info *rt6 = dst_rt6_info(dst); 2857 2858 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU) 2859 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it. 2860 * [see also comment in rt6_mtu_change_route()] 2861 */ 2862 2863 if (iph) { 2864 daddr = &iph->daddr; 2865 saddr = &iph->saddr; 2866 } else if (sk) { 2867 daddr = &sk->sk_v6_daddr; 2868 saddr = &inet6_sk(sk)->saddr; 2869 } else { 2870 daddr = NULL; 2871 saddr = NULL; 2872 } 2873 2874 if (confirm_neigh) 2875 dst_confirm_neigh(dst, daddr); 2876 2877 if (mtu < IPV6_MIN_MTU) 2878 return; 2879 if (mtu >= dst_mtu(dst)) 2880 return; 2881 2882 if (!rt6_cache_allowed_for_pmtu(rt6)) { 2883 rt6_do_update_pmtu(rt6, mtu); 2884 /* update rt6_ex->stamp for cache */ 2885 if (rt6->rt6i_flags & RTF_CACHE) 2886 rt6_update_exception_stamp_rt(rt6); 2887 } else if (daddr) { 2888 struct fib6_result res = {}; 2889 struct rt6_info *nrt6; 2890 2891 rcu_read_lock(); 2892 res.f6i = rcu_dereference(rt6->from); 2893 if (!res.f6i) 2894 goto out_unlock; 2895 2896 res.fib6_flags = res.f6i->fib6_flags; 2897 res.fib6_type = res.f6i->fib6_type; 2898 2899 if (res.f6i->nh) { 2900 struct fib6_nh_match_arg arg = { 2901 .dev = dst->dev, 2902 .gw = &rt6->rt6i_gateway, 2903 }; 2904 2905 nexthop_for_each_fib6_nh(res.f6i->nh, 2906 fib6_nh_find_match, &arg); 2907 2908 /* fib6_info uses a nexthop that does not have fib6_nh 2909 * using the dst->dev + gw. Should be impossible. 2910 */ 2911 if (!arg.match) 2912 goto out_unlock; 2913 2914 res.nh = arg.match; 2915 } else { 2916 res.nh = res.f6i->fib6_nh; 2917 } 2918 2919 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr); 2920 if (nrt6) { 2921 rt6_do_update_pmtu(nrt6, mtu); 2922 if (rt6_insert_exception(nrt6, &res)) 2923 dst_release_immediate(&nrt6->dst); 2924 } 2925 out_unlock: 2926 rcu_read_unlock(); 2927 } 2928 } 2929 2930 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 2931 struct sk_buff *skb, u32 mtu, 2932 bool confirm_neigh) 2933 { 2934 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu, 2935 confirm_neigh); 2936 } 2937 2938 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, 2939 int oif, u32 mark, kuid_t uid) 2940 { 2941 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 2942 struct dst_entry *dst; 2943 struct flowi6 fl6 = { 2944 .flowi6_oif = oif, 2945 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark), 2946 .daddr = iph->daddr, 2947 .saddr = iph->saddr, 2948 .flowlabel = ip6_flowinfo(iph), 2949 .flowi6_uid = uid, 2950 }; 2951 2952 dst = ip6_route_output(net, NULL, &fl6); 2953 if (!dst->error) 2954 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true); 2955 dst_release(dst); 2956 } 2957 EXPORT_SYMBOL_GPL(ip6_update_pmtu); 2958 2959 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu) 2960 { 2961 int oif = sk->sk_bound_dev_if; 2962 struct dst_entry *dst; 2963 2964 if (!oif && skb->dev) 2965 oif = l3mdev_master_ifindex(skb->dev); 2966 2967 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, READ_ONCE(sk->sk_mark), 2968 sk->sk_uid); 2969 2970 dst = __sk_dst_get(sk); 2971 if (!dst || !dst->obsolete || 2972 dst->ops->check(dst, inet6_sk(sk)->dst_cookie)) 2973 return; 2974 2975 bh_lock_sock(sk); 2976 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 2977 ip6_datagram_dst_update(sk, false); 2978 bh_unlock_sock(sk); 2979 } 2980 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu); 2981 2982 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst, 2983 const struct flowi6 *fl6) 2984 { 2985 #ifdef CONFIG_IPV6_SUBTREES 2986 struct ipv6_pinfo *np = inet6_sk(sk); 2987 #endif 2988 2989 ip6_dst_store(sk, dst, 2990 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ? 2991 &sk->sk_v6_daddr : NULL, 2992 #ifdef CONFIG_IPV6_SUBTREES 2993 ipv6_addr_equal(&fl6->saddr, &np->saddr) ? 2994 &np->saddr : 2995 #endif 2996 NULL); 2997 } 2998 2999 static bool ip6_redirect_nh_match(const struct fib6_result *res, 3000 struct flowi6 *fl6, 3001 const struct in6_addr *gw, 3002 struct rt6_info **ret) 3003 { 3004 const struct fib6_nh *nh = res->nh; 3005 3006 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family || 3007 fl6->flowi6_oif != nh->fib_nh_dev->ifindex) 3008 return false; 3009 3010 /* rt_cache's gateway might be different from its 'parent' 3011 * in the case of an ip redirect. 3012 * So we keep searching in the exception table if the gateway 3013 * is different. 3014 */ 3015 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) { 3016 struct rt6_info *rt_cache; 3017 3018 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr); 3019 if (rt_cache && 3020 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) { 3021 *ret = rt_cache; 3022 return true; 3023 } 3024 return false; 3025 } 3026 return true; 3027 } 3028 3029 struct fib6_nh_rd_arg { 3030 struct fib6_result *res; 3031 struct flowi6 *fl6; 3032 const struct in6_addr *gw; 3033 struct rt6_info **ret; 3034 }; 3035 3036 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg) 3037 { 3038 struct fib6_nh_rd_arg *arg = _arg; 3039 3040 arg->res->nh = nh; 3041 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret); 3042 } 3043 3044 /* Handle redirects */ 3045 struct ip6rd_flowi { 3046 struct flowi6 fl6; 3047 struct in6_addr gateway; 3048 }; 3049 3050 INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, 3051 struct fib6_table *table, 3052 struct flowi6 *fl6, 3053 const struct sk_buff *skb, 3054 int flags) 3055 { 3056 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6; 3057 struct rt6_info *ret = NULL; 3058 struct fib6_result res = {}; 3059 struct fib6_nh_rd_arg arg = { 3060 .res = &res, 3061 .fl6 = fl6, 3062 .gw = &rdfl->gateway, 3063 .ret = &ret 3064 }; 3065 struct fib6_info *rt; 3066 struct fib6_node *fn; 3067 3068 /* Get the "current" route for this destination and 3069 * check if the redirect has come from appropriate router. 3070 * 3071 * RFC 4861 specifies that redirects should only be 3072 * accepted if they come from the nexthop to the target. 3073 * Due to the way the routes are chosen, this notion 3074 * is a bit fuzzy and one might need to check all possible 3075 * routes. 3076 */ 3077 3078 rcu_read_lock(); 3079 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 3080 restart: 3081 for_each_fib6_node_rt_rcu(fn) { 3082 res.f6i = rt; 3083 if (fib6_check_expired(rt)) 3084 continue; 3085 if (rt->fib6_flags & RTF_REJECT) 3086 break; 3087 if (unlikely(rt->nh)) { 3088 if (nexthop_is_blackhole(rt->nh)) 3089 continue; 3090 /* on match, res->nh is filled in and potentially ret */ 3091 if (nexthop_for_each_fib6_nh(rt->nh, 3092 fib6_nh_redirect_match, 3093 &arg)) 3094 goto out; 3095 } else { 3096 res.nh = rt->fib6_nh; 3097 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway, 3098 &ret)) 3099 goto out; 3100 } 3101 } 3102 3103 if (!rt) 3104 rt = net->ipv6.fib6_null_entry; 3105 else if (rt->fib6_flags & RTF_REJECT) { 3106 ret = net->ipv6.ip6_null_entry; 3107 goto out; 3108 } 3109 3110 if (rt == net->ipv6.fib6_null_entry) { 3111 fn = fib6_backtrack(fn, &fl6->saddr); 3112 if (fn) 3113 goto restart; 3114 } 3115 3116 res.f6i = rt; 3117 res.nh = rt->fib6_nh; 3118 out: 3119 if (ret) { 3120 ip6_hold_safe(net, &ret); 3121 } else { 3122 res.fib6_flags = res.f6i->fib6_flags; 3123 res.fib6_type = res.f6i->fib6_type; 3124 ret = ip6_create_rt_rcu(&res); 3125 } 3126 3127 rcu_read_unlock(); 3128 3129 trace_fib6_table_lookup(net, &res, table, fl6); 3130 return ret; 3131 }; 3132 3133 static struct dst_entry *ip6_route_redirect(struct net *net, 3134 const struct flowi6 *fl6, 3135 const struct sk_buff *skb, 3136 const struct in6_addr *gateway) 3137 { 3138 int flags = RT6_LOOKUP_F_HAS_SADDR; 3139 struct ip6rd_flowi rdfl; 3140 3141 rdfl.fl6 = *fl6; 3142 rdfl.gateway = *gateway; 3143 3144 return fib6_rule_lookup(net, &rdfl.fl6, skb, 3145 flags, __ip6_route_redirect); 3146 } 3147 3148 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, 3149 kuid_t uid) 3150 { 3151 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 3152 struct dst_entry *dst; 3153 struct flowi6 fl6 = { 3154 .flowi6_iif = LOOPBACK_IFINDEX, 3155 .flowi6_oif = oif, 3156 .flowi6_mark = mark, 3157 .daddr = iph->daddr, 3158 .saddr = iph->saddr, 3159 .flowlabel = ip6_flowinfo(iph), 3160 .flowi6_uid = uid, 3161 }; 3162 3163 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr); 3164 rt6_do_redirect(dst, NULL, skb); 3165 dst_release(dst); 3166 } 3167 EXPORT_SYMBOL_GPL(ip6_redirect); 3168 3169 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif) 3170 { 3171 const struct ipv6hdr *iph = ipv6_hdr(skb); 3172 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb); 3173 struct dst_entry *dst; 3174 struct flowi6 fl6 = { 3175 .flowi6_iif = LOOPBACK_IFINDEX, 3176 .flowi6_oif = oif, 3177 .daddr = msg->dest, 3178 .saddr = iph->daddr, 3179 .flowi6_uid = sock_net_uid(net, NULL), 3180 }; 3181 3182 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr); 3183 rt6_do_redirect(dst, NULL, skb); 3184 dst_release(dst); 3185 } 3186 3187 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) 3188 { 3189 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, 3190 READ_ONCE(sk->sk_mark), sk->sk_uid); 3191 } 3192 EXPORT_SYMBOL_GPL(ip6_sk_redirect); 3193 3194 static unsigned int ip6_default_advmss(const struct dst_entry *dst) 3195 { 3196 struct net_device *dev = dst->dev; 3197 unsigned int mtu = dst_mtu(dst); 3198 struct net *net = dev_net(dev); 3199 3200 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 3201 3202 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) 3203 mtu = net->ipv6.sysctl.ip6_rt_min_advmss; 3204 3205 /* 3206 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and 3207 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size. 3208 * IPV6_MAXPLEN is also valid and means: "any MSS, 3209 * rely only on pmtu discovery" 3210 */ 3211 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr)) 3212 mtu = IPV6_MAXPLEN; 3213 return mtu; 3214 } 3215 3216 INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst) 3217 { 3218 return ip6_dst_mtu_maybe_forward(dst, false); 3219 } 3220 EXPORT_INDIRECT_CALLABLE(ip6_mtu); 3221 3222 /* MTU selection: 3223 * 1. mtu on route is locked - use it 3224 * 2. mtu from nexthop exception 3225 * 3. mtu from egress device 3226 * 3227 * based on ip6_dst_mtu_forward and exception logic of 3228 * rt6_find_cached_rt; called with rcu_read_lock 3229 */ 3230 u32 ip6_mtu_from_fib6(const struct fib6_result *res, 3231 const struct in6_addr *daddr, 3232 const struct in6_addr *saddr) 3233 { 3234 const struct fib6_nh *nh = res->nh; 3235 struct fib6_info *f6i = res->f6i; 3236 struct inet6_dev *idev; 3237 struct rt6_info *rt; 3238 u32 mtu = 0; 3239 3240 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) { 3241 mtu = f6i->fib6_pmtu; 3242 if (mtu) 3243 goto out; 3244 } 3245 3246 rt = rt6_find_cached_rt(res, daddr, saddr); 3247 if (unlikely(rt)) { 3248 mtu = dst_metric_raw(&rt->dst, RTAX_MTU); 3249 } else { 3250 struct net_device *dev = nh->fib_nh_dev; 3251 3252 mtu = IPV6_MIN_MTU; 3253 idev = __in6_dev_get(dev); 3254 if (idev) 3255 mtu = max_t(u32, mtu, READ_ONCE(idev->cnf.mtu6)); 3256 } 3257 3258 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 3259 out: 3260 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 3261 } 3262 3263 struct dst_entry *icmp6_dst_alloc(struct net_device *dev, 3264 struct flowi6 *fl6) 3265 { 3266 struct dst_entry *dst; 3267 struct rt6_info *rt; 3268 struct inet6_dev *idev = in6_dev_get(dev); 3269 struct net *net = dev_net(dev); 3270 3271 if (unlikely(!idev)) 3272 return ERR_PTR(-ENODEV); 3273 3274 rt = ip6_dst_alloc(net, dev, 0); 3275 if (unlikely(!rt)) { 3276 in6_dev_put(idev); 3277 dst = ERR_PTR(-ENOMEM); 3278 goto out; 3279 } 3280 3281 rt->dst.input = ip6_input; 3282 rt->dst.output = ip6_output; 3283 rt->rt6i_gateway = fl6->daddr; 3284 rt->rt6i_dst.addr = fl6->daddr; 3285 rt->rt6i_dst.plen = 128; 3286 rt->rt6i_idev = idev; 3287 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0); 3288 3289 /* Add this dst into uncached_list so that rt6_disable_ip() can 3290 * do proper release of the net_device 3291 */ 3292 rt6_uncached_list_add(rt); 3293 3294 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0); 3295 3296 out: 3297 return dst; 3298 } 3299 3300 static void ip6_dst_gc(struct dst_ops *ops) 3301 { 3302 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); 3303 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; 3304 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; 3305 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; 3306 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; 3307 unsigned int val; 3308 int entries; 3309 3310 if (time_after(rt_last_gc + rt_min_interval, jiffies)) 3311 goto out; 3312 3313 fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true); 3314 entries = dst_entries_get_slow(ops); 3315 if (entries < ops->gc_thresh) 3316 atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1); 3317 out: 3318 val = atomic_read(&net->ipv6.ip6_rt_gc_expire); 3319 atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); 3320 } 3321 3322 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg, 3323 const struct in6_addr *gw_addr, u32 tbid, 3324 int flags, struct fib6_result *res) 3325 { 3326 struct flowi6 fl6 = { 3327 .flowi6_oif = cfg->fc_ifindex, 3328 .daddr = *gw_addr, 3329 .saddr = cfg->fc_prefsrc, 3330 }; 3331 struct fib6_table *table; 3332 int err; 3333 3334 table = fib6_get_table(net, tbid); 3335 if (!table) 3336 return -EINVAL; 3337 3338 if (!ipv6_addr_any(&cfg->fc_prefsrc)) 3339 flags |= RT6_LOOKUP_F_HAS_SADDR; 3340 3341 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE; 3342 3343 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags); 3344 if (!err && res->f6i != net->ipv6.fib6_null_entry) 3345 fib6_select_path(net, res, &fl6, cfg->fc_ifindex, 3346 cfg->fc_ifindex != 0, NULL, flags); 3347 3348 return err; 3349 } 3350 3351 static int ip6_route_check_nh_onlink(struct net *net, 3352 struct fib6_config *cfg, 3353 const struct net_device *dev, 3354 struct netlink_ext_ack *extack) 3355 { 3356 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; 3357 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3358 struct fib6_result res = {}; 3359 int err; 3360 3361 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res); 3362 if (!err && !(res.fib6_flags & RTF_REJECT) && 3363 /* ignore match if it is the default route */ 3364 !ipv6_addr_any(&res.f6i->fib6_dst.addr) && 3365 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) { 3366 NL_SET_ERR_MSG(extack, 3367 "Nexthop has invalid gateway or device mismatch"); 3368 err = -EINVAL; 3369 } 3370 3371 return err; 3372 } 3373 3374 static int ip6_route_check_nh(struct net *net, 3375 struct fib6_config *cfg, 3376 struct net_device **_dev, 3377 netdevice_tracker *dev_tracker, 3378 struct inet6_dev **idev) 3379 { 3380 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3381 struct net_device *dev = _dev ? *_dev : NULL; 3382 int flags = RT6_LOOKUP_F_IFACE; 3383 struct fib6_result res = {}; 3384 int err = -EHOSTUNREACH; 3385 3386 if (cfg->fc_table) { 3387 err = ip6_nh_lookup_table(net, cfg, gw_addr, 3388 cfg->fc_table, flags, &res); 3389 /* gw_addr can not require a gateway or resolve to a reject 3390 * route. If a device is given, it must match the result. 3391 */ 3392 if (err || res.fib6_flags & RTF_REJECT || 3393 res.nh->fib_nh_gw_family || 3394 (dev && dev != res.nh->fib_nh_dev)) 3395 err = -EHOSTUNREACH; 3396 } 3397 3398 if (err < 0) { 3399 struct flowi6 fl6 = { 3400 .flowi6_oif = cfg->fc_ifindex, 3401 .daddr = *gw_addr, 3402 }; 3403 3404 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags); 3405 if (err || res.fib6_flags & RTF_REJECT || 3406 res.nh->fib_nh_gw_family) 3407 err = -EHOSTUNREACH; 3408 3409 if (err) 3410 return err; 3411 3412 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex, 3413 cfg->fc_ifindex != 0, NULL, flags); 3414 } 3415 3416 err = 0; 3417 if (dev) { 3418 if (dev != res.nh->fib_nh_dev) 3419 err = -EHOSTUNREACH; 3420 } else { 3421 *_dev = dev = res.nh->fib_nh_dev; 3422 netdev_hold(dev, dev_tracker, GFP_ATOMIC); 3423 *idev = in6_dev_get(dev); 3424 } 3425 3426 return err; 3427 } 3428 3429 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg, 3430 struct net_device **_dev, 3431 netdevice_tracker *dev_tracker, 3432 struct inet6_dev **idev, 3433 struct netlink_ext_ack *extack) 3434 { 3435 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3436 int gwa_type = ipv6_addr_type(gw_addr); 3437 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true; 3438 const struct net_device *dev = *_dev; 3439 bool need_addr_check = !dev; 3440 int err = -EINVAL; 3441 3442 /* if gw_addr is local we will fail to detect this in case 3443 * address is still TENTATIVE (DAD in progress). rt6_lookup() 3444 * will return already-added prefix route via interface that 3445 * prefix route was assigned to, which might be non-loopback. 3446 */ 3447 if (dev && 3448 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3449 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3450 goto out; 3451 } 3452 3453 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) { 3454 /* IPv6 strictly inhibits using not link-local 3455 * addresses as nexthop address. 3456 * Otherwise, router will not able to send redirects. 3457 * It is very good, but in some (rare!) circumstances 3458 * (SIT, PtP, NBMA NOARP links) it is handy to allow 3459 * some exceptions. --ANK 3460 * We allow IPv4-mapped nexthops to support RFC4798-type 3461 * addressing 3462 */ 3463 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) { 3464 NL_SET_ERR_MSG(extack, "Invalid gateway address"); 3465 goto out; 3466 } 3467 3468 rcu_read_lock(); 3469 3470 if (cfg->fc_flags & RTNH_F_ONLINK) 3471 err = ip6_route_check_nh_onlink(net, cfg, dev, extack); 3472 else 3473 err = ip6_route_check_nh(net, cfg, _dev, dev_tracker, 3474 idev); 3475 3476 rcu_read_unlock(); 3477 3478 if (err) 3479 goto out; 3480 } 3481 3482 /* reload in case device was changed */ 3483 dev = *_dev; 3484 3485 err = -EINVAL; 3486 if (!dev) { 3487 NL_SET_ERR_MSG(extack, "Egress device not specified"); 3488 goto out; 3489 } else if (dev->flags & IFF_LOOPBACK) { 3490 NL_SET_ERR_MSG(extack, 3491 "Egress device can not be loopback device for this route"); 3492 goto out; 3493 } 3494 3495 /* if we did not check gw_addr above, do so now that the 3496 * egress device has been resolved. 3497 */ 3498 if (need_addr_check && 3499 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3500 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3501 goto out; 3502 } 3503 3504 err = 0; 3505 out: 3506 return err; 3507 } 3508 3509 static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type) 3510 { 3511 if ((flags & RTF_REJECT) || 3512 (dev && (dev->flags & IFF_LOOPBACK) && 3513 !(addr_type & IPV6_ADDR_LOOPBACK) && 3514 !(flags & (RTF_ANYCAST | RTF_LOCAL)))) 3515 return true; 3516 3517 return false; 3518 } 3519 3520 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, 3521 struct fib6_config *cfg, gfp_t gfp_flags, 3522 struct netlink_ext_ack *extack) 3523 { 3524 netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker; 3525 struct net_device *dev = NULL; 3526 struct inet6_dev *idev = NULL; 3527 int addr_type; 3528 int err; 3529 3530 fib6_nh->fib_nh_family = AF_INET6; 3531 #ifdef CONFIG_IPV6_ROUTER_PREF 3532 fib6_nh->last_probe = jiffies; 3533 #endif 3534 if (cfg->fc_is_fdb) { 3535 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3536 fib6_nh->fib_nh_gw_family = AF_INET6; 3537 return 0; 3538 } 3539 3540 err = -ENODEV; 3541 if (cfg->fc_ifindex) { 3542 dev = netdev_get_by_index(net, cfg->fc_ifindex, 3543 dev_tracker, gfp_flags); 3544 if (!dev) 3545 goto out; 3546 idev = in6_dev_get(dev); 3547 if (!idev) 3548 goto out; 3549 } 3550 3551 if (cfg->fc_flags & RTNH_F_ONLINK) { 3552 if (!dev) { 3553 NL_SET_ERR_MSG(extack, 3554 "Nexthop device required for onlink"); 3555 goto out; 3556 } 3557 3558 if (!(dev->flags & IFF_UP)) { 3559 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3560 err = -ENETDOWN; 3561 goto out; 3562 } 3563 3564 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK; 3565 } 3566 3567 fib6_nh->fib_nh_weight = 1; 3568 3569 /* We cannot add true routes via loopback here, 3570 * they would result in kernel looping; promote them to reject routes 3571 */ 3572 addr_type = ipv6_addr_type(&cfg->fc_dst); 3573 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { 3574 /* hold loopback dev/idev if we haven't done so. */ 3575 if (dev != net->loopback_dev) { 3576 if (dev) { 3577 netdev_put(dev, dev_tracker); 3578 in6_dev_put(idev); 3579 } 3580 dev = net->loopback_dev; 3581 netdev_hold(dev, dev_tracker, gfp_flags); 3582 idev = in6_dev_get(dev); 3583 if (!idev) { 3584 err = -ENODEV; 3585 goto out; 3586 } 3587 } 3588 goto pcpu_alloc; 3589 } 3590 3591 if (cfg->fc_flags & RTF_GATEWAY) { 3592 err = ip6_validate_gw(net, cfg, &dev, dev_tracker, 3593 &idev, extack); 3594 if (err) 3595 goto out; 3596 3597 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3598 fib6_nh->fib_nh_gw_family = AF_INET6; 3599 } 3600 3601 err = -ENODEV; 3602 if (!dev) 3603 goto out; 3604 3605 if (idev->cnf.disable_ipv6) { 3606 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device"); 3607 err = -EACCES; 3608 goto out; 3609 } 3610 3611 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) { 3612 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3613 err = -ENETDOWN; 3614 goto out; 3615 } 3616 3617 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) && 3618 !netif_carrier_ok(dev)) 3619 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 3620 3621 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap, 3622 cfg->fc_encap_type, cfg, gfp_flags, extack); 3623 if (err) 3624 goto out; 3625 3626 pcpu_alloc: 3627 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags); 3628 if (!fib6_nh->rt6i_pcpu) { 3629 err = -ENOMEM; 3630 goto out; 3631 } 3632 3633 fib6_nh->fib_nh_dev = dev; 3634 fib6_nh->fib_nh_oif = dev->ifindex; 3635 err = 0; 3636 out: 3637 if (idev) 3638 in6_dev_put(idev); 3639 3640 if (err) { 3641 lwtstate_put(fib6_nh->fib_nh_lws); 3642 fib6_nh->fib_nh_lws = NULL; 3643 netdev_put(dev, dev_tracker); 3644 } 3645 3646 return err; 3647 } 3648 3649 void fib6_nh_release(struct fib6_nh *fib6_nh) 3650 { 3651 struct rt6_exception_bucket *bucket; 3652 3653 rcu_read_lock(); 3654 3655 fib6_nh_flush_exceptions(fib6_nh, NULL); 3656 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL); 3657 if (bucket) { 3658 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL); 3659 kfree(bucket); 3660 } 3661 3662 rcu_read_unlock(); 3663 3664 fib6_nh_release_dsts(fib6_nh); 3665 free_percpu(fib6_nh->rt6i_pcpu); 3666 3667 fib_nh_common_release(&fib6_nh->nh_common); 3668 } 3669 3670 void fib6_nh_release_dsts(struct fib6_nh *fib6_nh) 3671 { 3672 int cpu; 3673 3674 if (!fib6_nh->rt6i_pcpu) 3675 return; 3676 3677 for_each_possible_cpu(cpu) { 3678 struct rt6_info *pcpu_rt, **ppcpu_rt; 3679 3680 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); 3681 pcpu_rt = xchg(ppcpu_rt, NULL); 3682 if (pcpu_rt) { 3683 dst_dev_put(&pcpu_rt->dst); 3684 dst_release(&pcpu_rt->dst); 3685 } 3686 } 3687 } 3688 3689 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg, 3690 gfp_t gfp_flags, 3691 struct netlink_ext_ack *extack) 3692 { 3693 struct net *net = cfg->fc_nlinfo.nl_net; 3694 struct fib6_info *rt = NULL; 3695 struct nexthop *nh = NULL; 3696 struct fib6_table *table; 3697 struct fib6_nh *fib6_nh; 3698 int err = -EINVAL; 3699 int addr_type; 3700 3701 /* RTF_PCPU is an internal flag; can not be set by userspace */ 3702 if (cfg->fc_flags & RTF_PCPU) { 3703 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU"); 3704 goto out; 3705 } 3706 3707 /* RTF_CACHE is an internal flag; can not be set by userspace */ 3708 if (cfg->fc_flags & RTF_CACHE) { 3709 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE"); 3710 goto out; 3711 } 3712 3713 if (cfg->fc_type > RTN_MAX) { 3714 NL_SET_ERR_MSG(extack, "Invalid route type"); 3715 goto out; 3716 } 3717 3718 if (cfg->fc_dst_len > 128) { 3719 NL_SET_ERR_MSG(extack, "Invalid prefix length"); 3720 goto out; 3721 } 3722 if (cfg->fc_src_len > 128) { 3723 NL_SET_ERR_MSG(extack, "Invalid source address length"); 3724 goto out; 3725 } 3726 #ifndef CONFIG_IPV6_SUBTREES 3727 if (cfg->fc_src_len) { 3728 NL_SET_ERR_MSG(extack, 3729 "Specifying source address requires IPV6_SUBTREES to be enabled"); 3730 goto out; 3731 } 3732 #endif 3733 if (cfg->fc_nh_id) { 3734 nh = nexthop_find_by_id(net, cfg->fc_nh_id); 3735 if (!nh) { 3736 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 3737 goto out; 3738 } 3739 err = fib6_check_nexthop(nh, cfg, extack); 3740 if (err) 3741 goto out; 3742 } 3743 3744 err = -ENOBUFS; 3745 if (cfg->fc_nlinfo.nlh && 3746 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) { 3747 table = fib6_get_table(net, cfg->fc_table); 3748 if (!table) { 3749 pr_warn("NLM_F_CREATE should be specified when creating new route\n"); 3750 table = fib6_new_table(net, cfg->fc_table); 3751 } 3752 } else { 3753 table = fib6_new_table(net, cfg->fc_table); 3754 } 3755 3756 if (!table) 3757 goto out; 3758 3759 err = -ENOMEM; 3760 rt = fib6_info_alloc(gfp_flags, !nh); 3761 if (!rt) 3762 goto out; 3763 3764 rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len, 3765 extack); 3766 if (IS_ERR(rt->fib6_metrics)) { 3767 err = PTR_ERR(rt->fib6_metrics); 3768 /* Do not leave garbage there. */ 3769 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics; 3770 goto out_free; 3771 } 3772 3773 if (cfg->fc_flags & RTF_ADDRCONF) 3774 rt->dst_nocount = true; 3775 3776 if (cfg->fc_flags & RTF_EXPIRES) 3777 fib6_set_expires(rt, jiffies + 3778 clock_t_to_jiffies(cfg->fc_expires)); 3779 3780 if (cfg->fc_protocol == RTPROT_UNSPEC) 3781 cfg->fc_protocol = RTPROT_BOOT; 3782 rt->fib6_protocol = cfg->fc_protocol; 3783 3784 rt->fib6_table = table; 3785 rt->fib6_metric = cfg->fc_metric; 3786 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST; 3787 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY; 3788 3789 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len); 3790 rt->fib6_dst.plen = cfg->fc_dst_len; 3791 3792 #ifdef CONFIG_IPV6_SUBTREES 3793 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len); 3794 rt->fib6_src.plen = cfg->fc_src_len; 3795 #endif 3796 if (nh) { 3797 if (rt->fib6_src.plen) { 3798 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing"); 3799 goto out_free; 3800 } 3801 if (!nexthop_get(nh)) { 3802 NL_SET_ERR_MSG(extack, "Nexthop has been deleted"); 3803 goto out_free; 3804 } 3805 rt->nh = nh; 3806 fib6_nh = nexthop_fib6_nh(rt->nh); 3807 } else { 3808 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack); 3809 if (err) 3810 goto out; 3811 3812 fib6_nh = rt->fib6_nh; 3813 3814 /* We cannot add true routes via loopback here, they would 3815 * result in kernel looping; promote them to reject routes 3816 */ 3817 addr_type = ipv6_addr_type(&cfg->fc_dst); 3818 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev, 3819 addr_type)) 3820 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP; 3821 } 3822 3823 if (!ipv6_addr_any(&cfg->fc_prefsrc)) { 3824 struct net_device *dev = fib6_nh->fib_nh_dev; 3825 3826 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) { 3827 NL_SET_ERR_MSG(extack, "Invalid source address"); 3828 err = -EINVAL; 3829 goto out; 3830 } 3831 rt->fib6_prefsrc.addr = cfg->fc_prefsrc; 3832 rt->fib6_prefsrc.plen = 128; 3833 } else 3834 rt->fib6_prefsrc.plen = 0; 3835 3836 return rt; 3837 out: 3838 fib6_info_release(rt); 3839 return ERR_PTR(err); 3840 out_free: 3841 ip_fib_metrics_put(rt->fib6_metrics); 3842 kfree(rt); 3843 return ERR_PTR(err); 3844 } 3845 3846 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags, 3847 struct netlink_ext_ack *extack) 3848 { 3849 struct fib6_info *rt; 3850 int err; 3851 3852 rt = ip6_route_info_create(cfg, gfp_flags, extack); 3853 if (IS_ERR(rt)) 3854 return PTR_ERR(rt); 3855 3856 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack); 3857 fib6_info_release(rt); 3858 3859 return err; 3860 } 3861 3862 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info) 3863 { 3864 struct net *net = info->nl_net; 3865 struct fib6_table *table; 3866 int err; 3867 3868 if (rt == net->ipv6.fib6_null_entry) { 3869 err = -ENOENT; 3870 goto out; 3871 } 3872 3873 table = rt->fib6_table; 3874 spin_lock_bh(&table->tb6_lock); 3875 err = fib6_del(rt, info); 3876 spin_unlock_bh(&table->tb6_lock); 3877 3878 out: 3879 fib6_info_release(rt); 3880 return err; 3881 } 3882 3883 int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify) 3884 { 3885 struct nl_info info = { 3886 .nl_net = net, 3887 .skip_notify = skip_notify 3888 }; 3889 3890 return __ip6_del_rt(rt, &info); 3891 } 3892 3893 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) 3894 { 3895 struct nl_info *info = &cfg->fc_nlinfo; 3896 struct net *net = info->nl_net; 3897 struct sk_buff *skb = NULL; 3898 struct fib6_table *table; 3899 int err = -ENOENT; 3900 3901 if (rt == net->ipv6.fib6_null_entry) 3902 goto out_put; 3903 table = rt->fib6_table; 3904 spin_lock_bh(&table->tb6_lock); 3905 3906 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { 3907 struct fib6_info *sibling, *next_sibling; 3908 struct fib6_node *fn; 3909 3910 /* prefer to send a single notification with all hops */ 3911 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 3912 if (skb) { 3913 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 3914 3915 if (rt6_fill_node(net, skb, rt, NULL, 3916 NULL, NULL, 0, RTM_DELROUTE, 3917 info->portid, seq, 0) < 0) { 3918 kfree_skb(skb); 3919 skb = NULL; 3920 } else 3921 info->skip_notify = 1; 3922 } 3923 3924 /* 'rt' points to the first sibling route. If it is not the 3925 * leaf, then we do not need to send a notification. Otherwise, 3926 * we need to check if the last sibling has a next route or not 3927 * and emit a replace or delete notification, respectively. 3928 */ 3929 info->skip_notify_kernel = 1; 3930 fn = rcu_dereference_protected(rt->fib6_node, 3931 lockdep_is_held(&table->tb6_lock)); 3932 if (rcu_access_pointer(fn->leaf) == rt) { 3933 struct fib6_info *last_sibling, *replace_rt; 3934 3935 last_sibling = list_last_entry(&rt->fib6_siblings, 3936 struct fib6_info, 3937 fib6_siblings); 3938 replace_rt = rcu_dereference_protected( 3939 last_sibling->fib6_next, 3940 lockdep_is_held(&table->tb6_lock)); 3941 if (replace_rt) 3942 call_fib6_entry_notifiers_replace(net, 3943 replace_rt); 3944 else 3945 call_fib6_multipath_entry_notifiers(net, 3946 FIB_EVENT_ENTRY_DEL, 3947 rt, rt->fib6_nsiblings, 3948 NULL); 3949 } 3950 list_for_each_entry_safe(sibling, next_sibling, 3951 &rt->fib6_siblings, 3952 fib6_siblings) { 3953 err = fib6_del(sibling, info); 3954 if (err) 3955 goto out_unlock; 3956 } 3957 } 3958 3959 err = fib6_del(rt, info); 3960 out_unlock: 3961 spin_unlock_bh(&table->tb6_lock); 3962 out_put: 3963 fib6_info_release(rt); 3964 3965 if (skb) { 3966 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 3967 info->nlh, gfp_any()); 3968 } 3969 return err; 3970 } 3971 3972 static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg) 3973 { 3974 int rc = -ESRCH; 3975 3976 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex) 3977 goto out; 3978 3979 if (cfg->fc_flags & RTF_GATEWAY && 3980 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway)) 3981 goto out; 3982 3983 rc = rt6_remove_exception_rt(rt); 3984 out: 3985 return rc; 3986 } 3987 3988 static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt, 3989 struct fib6_nh *nh) 3990 { 3991 struct fib6_result res = { 3992 .f6i = rt, 3993 .nh = nh, 3994 }; 3995 struct rt6_info *rt_cache; 3996 3997 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src); 3998 if (rt_cache) 3999 return __ip6_del_cached_rt(rt_cache, cfg); 4000 4001 return 0; 4002 } 4003 4004 struct fib6_nh_del_cached_rt_arg { 4005 struct fib6_config *cfg; 4006 struct fib6_info *f6i; 4007 }; 4008 4009 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg) 4010 { 4011 struct fib6_nh_del_cached_rt_arg *arg = _arg; 4012 int rc; 4013 4014 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh); 4015 return rc != -ESRCH ? rc : 0; 4016 } 4017 4018 static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i) 4019 { 4020 struct fib6_nh_del_cached_rt_arg arg = { 4021 .cfg = cfg, 4022 .f6i = f6i 4023 }; 4024 4025 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg); 4026 } 4027 4028 static int ip6_route_del(struct fib6_config *cfg, 4029 struct netlink_ext_ack *extack) 4030 { 4031 struct fib6_table *table; 4032 struct fib6_info *rt; 4033 struct fib6_node *fn; 4034 int err = -ESRCH; 4035 4036 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table); 4037 if (!table) { 4038 NL_SET_ERR_MSG(extack, "FIB table does not exist"); 4039 return err; 4040 } 4041 4042 rcu_read_lock(); 4043 4044 fn = fib6_locate(&table->tb6_root, 4045 &cfg->fc_dst, cfg->fc_dst_len, 4046 &cfg->fc_src, cfg->fc_src_len, 4047 !(cfg->fc_flags & RTF_CACHE)); 4048 4049 if (fn) { 4050 for_each_fib6_node_rt_rcu(fn) { 4051 struct fib6_nh *nh; 4052 4053 if (rt->nh && cfg->fc_nh_id && 4054 rt->nh->id != cfg->fc_nh_id) 4055 continue; 4056 4057 if (cfg->fc_flags & RTF_CACHE) { 4058 int rc = 0; 4059 4060 if (rt->nh) { 4061 rc = ip6_del_cached_rt_nh(cfg, rt); 4062 } else if (cfg->fc_nh_id) { 4063 continue; 4064 } else { 4065 nh = rt->fib6_nh; 4066 rc = ip6_del_cached_rt(cfg, rt, nh); 4067 } 4068 if (rc != -ESRCH) { 4069 rcu_read_unlock(); 4070 return rc; 4071 } 4072 continue; 4073 } 4074 4075 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric) 4076 continue; 4077 if (cfg->fc_protocol && 4078 cfg->fc_protocol != rt->fib6_protocol) 4079 continue; 4080 4081 if (rt->nh) { 4082 if (!fib6_info_hold_safe(rt)) 4083 continue; 4084 rcu_read_unlock(); 4085 4086 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4087 } 4088 if (cfg->fc_nh_id) 4089 continue; 4090 4091 nh = rt->fib6_nh; 4092 if (cfg->fc_ifindex && 4093 (!nh->fib_nh_dev || 4094 nh->fib_nh_dev->ifindex != cfg->fc_ifindex)) 4095 continue; 4096 if (cfg->fc_flags & RTF_GATEWAY && 4097 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6)) 4098 continue; 4099 if (!fib6_info_hold_safe(rt)) 4100 continue; 4101 rcu_read_unlock(); 4102 4103 /* if gateway was specified only delete the one hop */ 4104 if (cfg->fc_flags & RTF_GATEWAY) 4105 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4106 4107 return __ip6_del_rt_siblings(rt, cfg); 4108 } 4109 } 4110 rcu_read_unlock(); 4111 4112 return err; 4113 } 4114 4115 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 4116 { 4117 struct netevent_redirect netevent; 4118 struct rt6_info *rt, *nrt = NULL; 4119 struct fib6_result res = {}; 4120 struct ndisc_options ndopts; 4121 struct inet6_dev *in6_dev; 4122 struct neighbour *neigh; 4123 struct rd_msg *msg; 4124 int optlen, on_link; 4125 u8 *lladdr; 4126 4127 optlen = skb_tail_pointer(skb) - skb_transport_header(skb); 4128 optlen -= sizeof(*msg); 4129 4130 if (optlen < 0) { 4131 net_dbg_ratelimited("rt6_do_redirect: packet too short\n"); 4132 return; 4133 } 4134 4135 msg = (struct rd_msg *)icmp6_hdr(skb); 4136 4137 if (ipv6_addr_is_multicast(&msg->dest)) { 4138 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n"); 4139 return; 4140 } 4141 4142 on_link = 0; 4143 if (ipv6_addr_equal(&msg->dest, &msg->target)) { 4144 on_link = 1; 4145 } else if (ipv6_addr_type(&msg->target) != 4146 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) { 4147 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n"); 4148 return; 4149 } 4150 4151 in6_dev = __in6_dev_get(skb->dev); 4152 if (!in6_dev) 4153 return; 4154 if (READ_ONCE(in6_dev->cnf.forwarding) || 4155 !READ_ONCE(in6_dev->cnf.accept_redirects)) 4156 return; 4157 4158 /* RFC2461 8.1: 4159 * The IP source address of the Redirect MUST be the same as the current 4160 * first-hop router for the specified ICMP Destination Address. 4161 */ 4162 4163 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) { 4164 net_dbg_ratelimited("rt6_redirect: invalid ND options\n"); 4165 return; 4166 } 4167 4168 lladdr = NULL; 4169 if (ndopts.nd_opts_tgt_lladdr) { 4170 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, 4171 skb->dev); 4172 if (!lladdr) { 4173 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n"); 4174 return; 4175 } 4176 } 4177 4178 rt = dst_rt6_info(dst); 4179 if (rt->rt6i_flags & RTF_REJECT) { 4180 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n"); 4181 return; 4182 } 4183 4184 /* Redirect received -> path was valid. 4185 * Look, redirects are sent only in response to data packets, 4186 * so that this nexthop apparently is reachable. --ANK 4187 */ 4188 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr); 4189 4190 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1); 4191 if (!neigh) 4192 return; 4193 4194 /* 4195 * We have finally decided to accept it. 4196 */ 4197 4198 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE, 4199 NEIGH_UPDATE_F_WEAK_OVERRIDE| 4200 NEIGH_UPDATE_F_OVERRIDE| 4201 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER| 4202 NEIGH_UPDATE_F_ISROUTER)), 4203 NDISC_REDIRECT, &ndopts); 4204 4205 rcu_read_lock(); 4206 res.f6i = rcu_dereference(rt->from); 4207 if (!res.f6i) 4208 goto out; 4209 4210 if (res.f6i->nh) { 4211 struct fib6_nh_match_arg arg = { 4212 .dev = dst->dev, 4213 .gw = &rt->rt6i_gateway, 4214 }; 4215 4216 nexthop_for_each_fib6_nh(res.f6i->nh, 4217 fib6_nh_find_match, &arg); 4218 4219 /* fib6_info uses a nexthop that does not have fib6_nh 4220 * using the dst->dev. Should be impossible 4221 */ 4222 if (!arg.match) 4223 goto out; 4224 res.nh = arg.match; 4225 } else { 4226 res.nh = res.f6i->fib6_nh; 4227 } 4228 4229 res.fib6_flags = res.f6i->fib6_flags; 4230 res.fib6_type = res.f6i->fib6_type; 4231 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL); 4232 if (!nrt) 4233 goto out; 4234 4235 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE; 4236 if (on_link) 4237 nrt->rt6i_flags &= ~RTF_GATEWAY; 4238 4239 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key; 4240 4241 /* rt6_insert_exception() will take care of duplicated exceptions */ 4242 if (rt6_insert_exception(nrt, &res)) { 4243 dst_release_immediate(&nrt->dst); 4244 goto out; 4245 } 4246 4247 netevent.old = &rt->dst; 4248 netevent.new = &nrt->dst; 4249 netevent.daddr = &msg->dest; 4250 netevent.neigh = neigh; 4251 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent); 4252 4253 out: 4254 rcu_read_unlock(); 4255 neigh_release(neigh); 4256 } 4257 4258 #ifdef CONFIG_IPV6_ROUTE_INFO 4259 static struct fib6_info *rt6_get_route_info(struct net *net, 4260 const struct in6_addr *prefix, int prefixlen, 4261 const struct in6_addr *gwaddr, 4262 struct net_device *dev) 4263 { 4264 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4265 int ifindex = dev->ifindex; 4266 struct fib6_node *fn; 4267 struct fib6_info *rt = NULL; 4268 struct fib6_table *table; 4269 4270 table = fib6_get_table(net, tb_id); 4271 if (!table) 4272 return NULL; 4273 4274 rcu_read_lock(); 4275 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true); 4276 if (!fn) 4277 goto out; 4278 4279 for_each_fib6_node_rt_rcu(fn) { 4280 /* these routes do not use nexthops */ 4281 if (rt->nh) 4282 continue; 4283 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex) 4284 continue; 4285 if (!(rt->fib6_flags & RTF_ROUTEINFO) || 4286 !rt->fib6_nh->fib_nh_gw_family) 4287 continue; 4288 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr)) 4289 continue; 4290 if (!fib6_info_hold_safe(rt)) 4291 continue; 4292 break; 4293 } 4294 out: 4295 rcu_read_unlock(); 4296 return rt; 4297 } 4298 4299 static struct fib6_info *rt6_add_route_info(struct net *net, 4300 const struct in6_addr *prefix, int prefixlen, 4301 const struct in6_addr *gwaddr, 4302 struct net_device *dev, 4303 unsigned int pref) 4304 { 4305 struct fib6_config cfg = { 4306 .fc_metric = IP6_RT_PRIO_USER, 4307 .fc_ifindex = dev->ifindex, 4308 .fc_dst_len = prefixlen, 4309 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO | 4310 RTF_UP | RTF_PREF(pref), 4311 .fc_protocol = RTPROT_RA, 4312 .fc_type = RTN_UNICAST, 4313 .fc_nlinfo.portid = 0, 4314 .fc_nlinfo.nlh = NULL, 4315 .fc_nlinfo.nl_net = net, 4316 }; 4317 4318 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4319 cfg.fc_dst = *prefix; 4320 cfg.fc_gateway = *gwaddr; 4321 4322 /* We should treat it as a default route if prefix length is 0. */ 4323 if (!prefixlen) 4324 cfg.fc_flags |= RTF_DEFAULT; 4325 4326 ip6_route_add(&cfg, GFP_ATOMIC, NULL); 4327 4328 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev); 4329 } 4330 #endif 4331 4332 struct fib6_info *rt6_get_dflt_router(struct net *net, 4333 const struct in6_addr *addr, 4334 struct net_device *dev) 4335 { 4336 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; 4337 struct fib6_info *rt; 4338 struct fib6_table *table; 4339 4340 table = fib6_get_table(net, tb_id); 4341 if (!table) 4342 return NULL; 4343 4344 rcu_read_lock(); 4345 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4346 struct fib6_nh *nh; 4347 4348 /* RA routes do not use nexthops */ 4349 if (rt->nh) 4350 continue; 4351 4352 nh = rt->fib6_nh; 4353 if (dev == nh->fib_nh_dev && 4354 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) && 4355 ipv6_addr_equal(&nh->fib_nh_gw6, addr)) 4356 break; 4357 } 4358 if (rt && !fib6_info_hold_safe(rt)) 4359 rt = NULL; 4360 rcu_read_unlock(); 4361 return rt; 4362 } 4363 4364 struct fib6_info *rt6_add_dflt_router(struct net *net, 4365 const struct in6_addr *gwaddr, 4366 struct net_device *dev, 4367 unsigned int pref, 4368 u32 defrtr_usr_metric, 4369 int lifetime) 4370 { 4371 struct fib6_config cfg = { 4372 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, 4373 .fc_metric = defrtr_usr_metric, 4374 .fc_ifindex = dev->ifindex, 4375 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | 4376 RTF_UP | RTF_EXPIRES | RTF_PREF(pref), 4377 .fc_protocol = RTPROT_RA, 4378 .fc_type = RTN_UNICAST, 4379 .fc_nlinfo.portid = 0, 4380 .fc_nlinfo.nlh = NULL, 4381 .fc_nlinfo.nl_net = net, 4382 .fc_expires = jiffies_to_clock_t(lifetime * HZ), 4383 }; 4384 4385 cfg.fc_gateway = *gwaddr; 4386 4387 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) { 4388 struct fib6_table *table; 4389 4390 table = fib6_get_table(dev_net(dev), cfg.fc_table); 4391 if (table) 4392 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER; 4393 } 4394 4395 return rt6_get_dflt_router(net, gwaddr, dev); 4396 } 4397 4398 static void __rt6_purge_dflt_routers(struct net *net, 4399 struct fib6_table *table) 4400 { 4401 struct fib6_info *rt; 4402 4403 restart: 4404 rcu_read_lock(); 4405 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4406 struct net_device *dev = fib6_info_nh_dev(rt); 4407 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL; 4408 4409 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) && 4410 (!idev || idev->cnf.accept_ra != 2) && 4411 fib6_info_hold_safe(rt)) { 4412 rcu_read_unlock(); 4413 ip6_del_rt(net, rt, false); 4414 goto restart; 4415 } 4416 } 4417 rcu_read_unlock(); 4418 4419 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; 4420 } 4421 4422 void rt6_purge_dflt_routers(struct net *net) 4423 { 4424 struct fib6_table *table; 4425 struct hlist_head *head; 4426 unsigned int h; 4427 4428 rcu_read_lock(); 4429 4430 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { 4431 head = &net->ipv6.fib_table_hash[h]; 4432 hlist_for_each_entry_rcu(table, head, tb6_hlist) { 4433 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) 4434 __rt6_purge_dflt_routers(net, table); 4435 } 4436 } 4437 4438 rcu_read_unlock(); 4439 } 4440 4441 static void rtmsg_to_fib6_config(struct net *net, 4442 struct in6_rtmsg *rtmsg, 4443 struct fib6_config *cfg) 4444 { 4445 *cfg = (struct fib6_config){ 4446 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ? 4447 : RT6_TABLE_MAIN, 4448 .fc_ifindex = rtmsg->rtmsg_ifindex, 4449 .fc_metric = rtmsg->rtmsg_metric, 4450 .fc_expires = rtmsg->rtmsg_info, 4451 .fc_dst_len = rtmsg->rtmsg_dst_len, 4452 .fc_src_len = rtmsg->rtmsg_src_len, 4453 .fc_flags = rtmsg->rtmsg_flags, 4454 .fc_type = rtmsg->rtmsg_type, 4455 4456 .fc_nlinfo.nl_net = net, 4457 4458 .fc_dst = rtmsg->rtmsg_dst, 4459 .fc_src = rtmsg->rtmsg_src, 4460 .fc_gateway = rtmsg->rtmsg_gateway, 4461 }; 4462 } 4463 4464 int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg) 4465 { 4466 struct fib6_config cfg; 4467 int err; 4468 4469 if (cmd != SIOCADDRT && cmd != SIOCDELRT) 4470 return -EINVAL; 4471 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 4472 return -EPERM; 4473 4474 rtmsg_to_fib6_config(net, rtmsg, &cfg); 4475 4476 rtnl_lock(); 4477 switch (cmd) { 4478 case SIOCADDRT: 4479 /* Only do the default setting of fc_metric in route adding */ 4480 if (cfg.fc_metric == 0) 4481 cfg.fc_metric = IP6_RT_PRIO_USER; 4482 err = ip6_route_add(&cfg, GFP_KERNEL, NULL); 4483 break; 4484 case SIOCDELRT: 4485 err = ip6_route_del(&cfg, NULL); 4486 break; 4487 } 4488 rtnl_unlock(); 4489 return err; 4490 } 4491 4492 /* 4493 * Drop the packet on the floor 4494 */ 4495 4496 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes) 4497 { 4498 struct dst_entry *dst = skb_dst(skb); 4499 struct net *net = dev_net(dst->dev); 4500 struct inet6_dev *idev; 4501 SKB_DR(reason); 4502 int type; 4503 4504 if (netif_is_l3_master(skb->dev) || 4505 dst->dev == net->loopback_dev) 4506 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 4507 else 4508 idev = ip6_dst_idev(dst); 4509 4510 switch (ipstats_mib_noroutes) { 4511 case IPSTATS_MIB_INNOROUTES: 4512 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr); 4513 if (type == IPV6_ADDR_ANY) { 4514 SKB_DR_SET(reason, IP_INADDRERRORS); 4515 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 4516 break; 4517 } 4518 SKB_DR_SET(reason, IP_INNOROUTES); 4519 fallthrough; 4520 case IPSTATS_MIB_OUTNOROUTES: 4521 SKB_DR_OR(reason, IP_OUTNOROUTES); 4522 IP6_INC_STATS(net, idev, ipstats_mib_noroutes); 4523 break; 4524 } 4525 4526 /* Start over by dropping the dst for l3mdev case */ 4527 if (netif_is_l3_master(skb->dev)) 4528 skb_dst_drop(skb); 4529 4530 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0); 4531 kfree_skb_reason(skb, reason); 4532 return 0; 4533 } 4534 4535 static int ip6_pkt_discard(struct sk_buff *skb) 4536 { 4537 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES); 4538 } 4539 4540 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4541 { 4542 skb->dev = skb_dst(skb)->dev; 4543 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES); 4544 } 4545 4546 static int ip6_pkt_prohibit(struct sk_buff *skb) 4547 { 4548 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES); 4549 } 4550 4551 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4552 { 4553 skb->dev = skb_dst(skb)->dev; 4554 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES); 4555 } 4556 4557 /* 4558 * Allocate a dst for local (unicast / anycast) address. 4559 */ 4560 4561 struct fib6_info *addrconf_f6i_alloc(struct net *net, 4562 struct inet6_dev *idev, 4563 const struct in6_addr *addr, 4564 bool anycast, gfp_t gfp_flags, 4565 struct netlink_ext_ack *extack) 4566 { 4567 struct fib6_config cfg = { 4568 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL, 4569 .fc_ifindex = idev->dev->ifindex, 4570 .fc_flags = RTF_UP | RTF_NONEXTHOP, 4571 .fc_dst = *addr, 4572 .fc_dst_len = 128, 4573 .fc_protocol = RTPROT_KERNEL, 4574 .fc_nlinfo.nl_net = net, 4575 .fc_ignore_dev_down = true, 4576 }; 4577 struct fib6_info *f6i; 4578 4579 if (anycast) { 4580 cfg.fc_type = RTN_ANYCAST; 4581 cfg.fc_flags |= RTF_ANYCAST; 4582 } else { 4583 cfg.fc_type = RTN_LOCAL; 4584 cfg.fc_flags |= RTF_LOCAL; 4585 } 4586 4587 f6i = ip6_route_info_create(&cfg, gfp_flags, extack); 4588 if (!IS_ERR(f6i)) { 4589 f6i->dst_nocount = true; 4590 4591 if (!anycast && 4592 (READ_ONCE(net->ipv6.devconf_all->disable_policy) || 4593 READ_ONCE(idev->cnf.disable_policy))) 4594 f6i->dst_nopolicy = true; 4595 } 4596 4597 return f6i; 4598 } 4599 4600 /* remove deleted ip from prefsrc entries */ 4601 struct arg_dev_net_ip { 4602 struct net *net; 4603 struct in6_addr *addr; 4604 }; 4605 4606 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg) 4607 { 4608 struct net *net = ((struct arg_dev_net_ip *)arg)->net; 4609 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr; 4610 4611 if (!rt->nh && 4612 rt != net->ipv6.fib6_null_entry && 4613 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr) && 4614 !ipv6_chk_addr(net, addr, rt->fib6_nh->fib_nh_dev, 0)) { 4615 spin_lock_bh(&rt6_exception_lock); 4616 /* remove prefsrc entry */ 4617 rt->fib6_prefsrc.plen = 0; 4618 spin_unlock_bh(&rt6_exception_lock); 4619 } 4620 return 0; 4621 } 4622 4623 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp) 4624 { 4625 struct net *net = dev_net(ifp->idev->dev); 4626 struct arg_dev_net_ip adni = { 4627 .net = net, 4628 .addr = &ifp->addr, 4629 }; 4630 fib6_clean_all(net, fib6_remove_prefsrc, &adni); 4631 } 4632 4633 #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT) 4634 4635 /* Remove routers and update dst entries when gateway turn into host. */ 4636 static int fib6_clean_tohost(struct fib6_info *rt, void *arg) 4637 { 4638 struct in6_addr *gateway = (struct in6_addr *)arg; 4639 struct fib6_nh *nh; 4640 4641 /* RA routes do not use nexthops */ 4642 if (rt->nh) 4643 return 0; 4644 4645 nh = rt->fib6_nh; 4646 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) && 4647 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6)) 4648 return -1; 4649 4650 /* Further clean up cached routes in exception table. 4651 * This is needed because cached route may have a different 4652 * gateway than its 'parent' in the case of an ip redirect. 4653 */ 4654 fib6_nh_exceptions_clean_tohost(nh, gateway); 4655 4656 return 0; 4657 } 4658 4659 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway) 4660 { 4661 fib6_clean_all(net, fib6_clean_tohost, gateway); 4662 } 4663 4664 struct arg_netdev_event { 4665 const struct net_device *dev; 4666 union { 4667 unsigned char nh_flags; 4668 unsigned long event; 4669 }; 4670 }; 4671 4672 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt) 4673 { 4674 struct fib6_info *iter; 4675 struct fib6_node *fn; 4676 4677 fn = rcu_dereference_protected(rt->fib6_node, 4678 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4679 iter = rcu_dereference_protected(fn->leaf, 4680 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4681 while (iter) { 4682 if (iter->fib6_metric == rt->fib6_metric && 4683 rt6_qualify_for_ecmp(iter)) 4684 return iter; 4685 iter = rcu_dereference_protected(iter->fib6_next, 4686 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4687 } 4688 4689 return NULL; 4690 } 4691 4692 /* only called for fib entries with builtin fib6_nh */ 4693 static bool rt6_is_dead(const struct fib6_info *rt) 4694 { 4695 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD || 4696 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN && 4697 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev))) 4698 return true; 4699 4700 return false; 4701 } 4702 4703 static int rt6_multipath_total_weight(const struct fib6_info *rt) 4704 { 4705 struct fib6_info *iter; 4706 int total = 0; 4707 4708 if (!rt6_is_dead(rt)) 4709 total += rt->fib6_nh->fib_nh_weight; 4710 4711 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 4712 if (!rt6_is_dead(iter)) 4713 total += iter->fib6_nh->fib_nh_weight; 4714 } 4715 4716 return total; 4717 } 4718 4719 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) 4720 { 4721 int upper_bound = -1; 4722 4723 if (!rt6_is_dead(rt)) { 4724 *weight += rt->fib6_nh->fib_nh_weight; 4725 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, 4726 total) - 1; 4727 } 4728 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound); 4729 } 4730 4731 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total) 4732 { 4733 struct fib6_info *iter; 4734 int weight = 0; 4735 4736 rt6_upper_bound_set(rt, &weight, total); 4737 4738 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4739 rt6_upper_bound_set(iter, &weight, total); 4740 } 4741 4742 void rt6_multipath_rebalance(struct fib6_info *rt) 4743 { 4744 struct fib6_info *first; 4745 int total; 4746 4747 /* In case the entire multipath route was marked for flushing, 4748 * then there is no need to rebalance upon the removal of every 4749 * sibling route. 4750 */ 4751 if (!rt->fib6_nsiblings || rt->should_flush) 4752 return; 4753 4754 /* During lookup routes are evaluated in order, so we need to 4755 * make sure upper bounds are assigned from the first sibling 4756 * onwards. 4757 */ 4758 first = rt6_multipath_first_sibling(rt); 4759 if (WARN_ON_ONCE(!first)) 4760 return; 4761 4762 total = rt6_multipath_total_weight(first); 4763 rt6_multipath_upper_bound_set(first, total); 4764 } 4765 4766 static int fib6_ifup(struct fib6_info *rt, void *p_arg) 4767 { 4768 const struct arg_netdev_event *arg = p_arg; 4769 struct net *net = dev_net(arg->dev); 4770 4771 if (rt != net->ipv6.fib6_null_entry && !rt->nh && 4772 rt->fib6_nh->fib_nh_dev == arg->dev) { 4773 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags; 4774 fib6_update_sernum_upto_root(net, rt); 4775 rt6_multipath_rebalance(rt); 4776 } 4777 4778 return 0; 4779 } 4780 4781 void rt6_sync_up(struct net_device *dev, unsigned char nh_flags) 4782 { 4783 struct arg_netdev_event arg = { 4784 .dev = dev, 4785 { 4786 .nh_flags = nh_flags, 4787 }, 4788 }; 4789 4790 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev)) 4791 arg.nh_flags |= RTNH_F_LINKDOWN; 4792 4793 fib6_clean_all(dev_net(dev), fib6_ifup, &arg); 4794 } 4795 4796 /* only called for fib entries with inline fib6_nh */ 4797 static bool rt6_multipath_uses_dev(const struct fib6_info *rt, 4798 const struct net_device *dev) 4799 { 4800 struct fib6_info *iter; 4801 4802 if (rt->fib6_nh->fib_nh_dev == dev) 4803 return true; 4804 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4805 if (iter->fib6_nh->fib_nh_dev == dev) 4806 return true; 4807 4808 return false; 4809 } 4810 4811 static void rt6_multipath_flush(struct fib6_info *rt) 4812 { 4813 struct fib6_info *iter; 4814 4815 rt->should_flush = 1; 4816 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4817 iter->should_flush = 1; 4818 } 4819 4820 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt, 4821 const struct net_device *down_dev) 4822 { 4823 struct fib6_info *iter; 4824 unsigned int dead = 0; 4825 4826 if (rt->fib6_nh->fib_nh_dev == down_dev || 4827 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4828 dead++; 4829 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4830 if (iter->fib6_nh->fib_nh_dev == down_dev || 4831 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4832 dead++; 4833 4834 return dead; 4835 } 4836 4837 static void rt6_multipath_nh_flags_set(struct fib6_info *rt, 4838 const struct net_device *dev, 4839 unsigned char nh_flags) 4840 { 4841 struct fib6_info *iter; 4842 4843 if (rt->fib6_nh->fib_nh_dev == dev) 4844 rt->fib6_nh->fib_nh_flags |= nh_flags; 4845 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4846 if (iter->fib6_nh->fib_nh_dev == dev) 4847 iter->fib6_nh->fib_nh_flags |= nh_flags; 4848 } 4849 4850 /* called with write lock held for table with rt */ 4851 static int fib6_ifdown(struct fib6_info *rt, void *p_arg) 4852 { 4853 const struct arg_netdev_event *arg = p_arg; 4854 const struct net_device *dev = arg->dev; 4855 struct net *net = dev_net(dev); 4856 4857 if (rt == net->ipv6.fib6_null_entry || rt->nh) 4858 return 0; 4859 4860 switch (arg->event) { 4861 case NETDEV_UNREGISTER: 4862 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4863 case NETDEV_DOWN: 4864 if (rt->should_flush) 4865 return -1; 4866 if (!rt->fib6_nsiblings) 4867 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4868 if (rt6_multipath_uses_dev(rt, dev)) { 4869 unsigned int count; 4870 4871 count = rt6_multipath_dead_count(rt, dev); 4872 if (rt->fib6_nsiblings + 1 == count) { 4873 rt6_multipath_flush(rt); 4874 return -1; 4875 } 4876 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD | 4877 RTNH_F_LINKDOWN); 4878 fib6_update_sernum(net, rt); 4879 rt6_multipath_rebalance(rt); 4880 } 4881 return -2; 4882 case NETDEV_CHANGE: 4883 if (rt->fib6_nh->fib_nh_dev != dev || 4884 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 4885 break; 4886 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 4887 rt6_multipath_rebalance(rt); 4888 break; 4889 } 4890 4891 return 0; 4892 } 4893 4894 void rt6_sync_down_dev(struct net_device *dev, unsigned long event) 4895 { 4896 struct arg_netdev_event arg = { 4897 .dev = dev, 4898 { 4899 .event = event, 4900 }, 4901 }; 4902 struct net *net = dev_net(dev); 4903 4904 if (net->ipv6.sysctl.skip_notify_on_dev_down) 4905 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); 4906 else 4907 fib6_clean_all(net, fib6_ifdown, &arg); 4908 } 4909 4910 void rt6_disable_ip(struct net_device *dev, unsigned long event) 4911 { 4912 rt6_sync_down_dev(dev, event); 4913 rt6_uncached_list_flush_dev(dev); 4914 neigh_ifdown(&nd_tbl, dev); 4915 } 4916 4917 struct rt6_mtu_change_arg { 4918 struct net_device *dev; 4919 unsigned int mtu; 4920 struct fib6_info *f6i; 4921 }; 4922 4923 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) 4924 { 4925 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg; 4926 struct fib6_info *f6i = arg->f6i; 4927 4928 /* For administrative MTU increase, there is no way to discover 4929 * IPv6 PMTU increase, so PMTU increase should be updated here. 4930 * Since RFC 1981 doesn't include administrative MTU increase 4931 * update PMTU increase is a MUST. (i.e. jumbo frame) 4932 */ 4933 if (nh->fib_nh_dev == arg->dev) { 4934 struct inet6_dev *idev = __in6_dev_get(arg->dev); 4935 u32 mtu = f6i->fib6_pmtu; 4936 4937 if (mtu >= arg->mtu || 4938 (mtu < arg->mtu && mtu == idev->cnf.mtu6)) 4939 fib6_metric_set(f6i, RTAX_MTU, arg->mtu); 4940 4941 spin_lock_bh(&rt6_exception_lock); 4942 rt6_exceptions_update_pmtu(idev, nh, arg->mtu); 4943 spin_unlock_bh(&rt6_exception_lock); 4944 } 4945 4946 return 0; 4947 } 4948 4949 static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg) 4950 { 4951 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg; 4952 struct inet6_dev *idev; 4953 4954 /* In IPv6 pmtu discovery is not optional, 4955 so that RTAX_MTU lock cannot disable it. 4956 We still use this lock to block changes 4957 caused by addrconf/ndisc. 4958 */ 4959 4960 idev = __in6_dev_get(arg->dev); 4961 if (!idev) 4962 return 0; 4963 4964 if (fib6_metric_locked(f6i, RTAX_MTU)) 4965 return 0; 4966 4967 arg->f6i = f6i; 4968 if (f6i->nh) { 4969 /* fib6_nh_mtu_change only returns 0, so this is safe */ 4970 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change, 4971 arg); 4972 } 4973 4974 return fib6_nh_mtu_change(f6i->fib6_nh, arg); 4975 } 4976 4977 void rt6_mtu_change(struct net_device *dev, unsigned int mtu) 4978 { 4979 struct rt6_mtu_change_arg arg = { 4980 .dev = dev, 4981 .mtu = mtu, 4982 }; 4983 4984 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg); 4985 } 4986 4987 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = { 4988 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 }, 4989 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) }, 4990 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) }, 4991 [RTA_OIF] = { .type = NLA_U32 }, 4992 [RTA_IIF] = { .type = NLA_U32 }, 4993 [RTA_PRIORITY] = { .type = NLA_U32 }, 4994 [RTA_METRICS] = { .type = NLA_NESTED }, 4995 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, 4996 [RTA_PREF] = { .type = NLA_U8 }, 4997 [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, 4998 [RTA_ENCAP] = { .type = NLA_NESTED }, 4999 [RTA_EXPIRES] = { .type = NLA_U32 }, 5000 [RTA_UID] = { .type = NLA_U32 }, 5001 [RTA_MARK] = { .type = NLA_U32 }, 5002 [RTA_TABLE] = { .type = NLA_U32 }, 5003 [RTA_IP_PROTO] = { .type = NLA_U8 }, 5004 [RTA_SPORT] = { .type = NLA_U16 }, 5005 [RTA_DPORT] = { .type = NLA_U16 }, 5006 [RTA_NH_ID] = { .type = NLA_U32 }, 5007 }; 5008 5009 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, 5010 struct fib6_config *cfg, 5011 struct netlink_ext_ack *extack) 5012 { 5013 struct rtmsg *rtm; 5014 struct nlattr *tb[RTA_MAX+1]; 5015 unsigned int pref; 5016 int err; 5017 5018 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5019 rtm_ipv6_policy, extack); 5020 if (err < 0) 5021 goto errout; 5022 5023 err = -EINVAL; 5024 rtm = nlmsg_data(nlh); 5025 5026 if (rtm->rtm_tos) { 5027 NL_SET_ERR_MSG(extack, 5028 "Invalid dsfield (tos): option not available for IPv6"); 5029 goto errout; 5030 } 5031 5032 *cfg = (struct fib6_config){ 5033 .fc_table = rtm->rtm_table, 5034 .fc_dst_len = rtm->rtm_dst_len, 5035 .fc_src_len = rtm->rtm_src_len, 5036 .fc_flags = RTF_UP, 5037 .fc_protocol = rtm->rtm_protocol, 5038 .fc_type = rtm->rtm_type, 5039 5040 .fc_nlinfo.portid = NETLINK_CB(skb).portid, 5041 .fc_nlinfo.nlh = nlh, 5042 .fc_nlinfo.nl_net = sock_net(skb->sk), 5043 }; 5044 5045 if (rtm->rtm_type == RTN_UNREACHABLE || 5046 rtm->rtm_type == RTN_BLACKHOLE || 5047 rtm->rtm_type == RTN_PROHIBIT || 5048 rtm->rtm_type == RTN_THROW) 5049 cfg->fc_flags |= RTF_REJECT; 5050 5051 if (rtm->rtm_type == RTN_LOCAL) 5052 cfg->fc_flags |= RTF_LOCAL; 5053 5054 if (rtm->rtm_flags & RTM_F_CLONED) 5055 cfg->fc_flags |= RTF_CACHE; 5056 5057 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK); 5058 5059 if (tb[RTA_NH_ID]) { 5060 if (tb[RTA_GATEWAY] || tb[RTA_OIF] || 5061 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) { 5062 NL_SET_ERR_MSG(extack, 5063 "Nexthop specification and nexthop id are mutually exclusive"); 5064 goto errout; 5065 } 5066 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]); 5067 } 5068 5069 if (tb[RTA_GATEWAY]) { 5070 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]); 5071 cfg->fc_flags |= RTF_GATEWAY; 5072 } 5073 if (tb[RTA_VIA]) { 5074 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute"); 5075 goto errout; 5076 } 5077 5078 if (tb[RTA_DST]) { 5079 int plen = (rtm->rtm_dst_len + 7) >> 3; 5080 5081 if (nla_len(tb[RTA_DST]) < plen) 5082 goto errout; 5083 5084 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen); 5085 } 5086 5087 if (tb[RTA_SRC]) { 5088 int plen = (rtm->rtm_src_len + 7) >> 3; 5089 5090 if (nla_len(tb[RTA_SRC]) < plen) 5091 goto errout; 5092 5093 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen); 5094 } 5095 5096 if (tb[RTA_PREFSRC]) 5097 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]); 5098 5099 if (tb[RTA_OIF]) 5100 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]); 5101 5102 if (tb[RTA_PRIORITY]) 5103 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]); 5104 5105 if (tb[RTA_METRICS]) { 5106 cfg->fc_mx = nla_data(tb[RTA_METRICS]); 5107 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]); 5108 } 5109 5110 if (tb[RTA_TABLE]) 5111 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]); 5112 5113 if (tb[RTA_MULTIPATH]) { 5114 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]); 5115 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]); 5116 5117 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp, 5118 cfg->fc_mp_len, extack); 5119 if (err < 0) 5120 goto errout; 5121 } 5122 5123 if (tb[RTA_PREF]) { 5124 pref = nla_get_u8(tb[RTA_PREF]); 5125 if (pref != ICMPV6_ROUTER_PREF_LOW && 5126 pref != ICMPV6_ROUTER_PREF_HIGH) 5127 pref = ICMPV6_ROUTER_PREF_MEDIUM; 5128 cfg->fc_flags |= RTF_PREF(pref); 5129 } 5130 5131 if (tb[RTA_ENCAP]) 5132 cfg->fc_encap = tb[RTA_ENCAP]; 5133 5134 if (tb[RTA_ENCAP_TYPE]) { 5135 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]); 5136 5137 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack); 5138 if (err < 0) 5139 goto errout; 5140 } 5141 5142 if (tb[RTA_EXPIRES]) { 5143 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ); 5144 5145 if (addrconf_finite_timeout(timeout)) { 5146 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ); 5147 cfg->fc_flags |= RTF_EXPIRES; 5148 } 5149 } 5150 5151 err = 0; 5152 errout: 5153 return err; 5154 } 5155 5156 struct rt6_nh { 5157 struct fib6_info *fib6_info; 5158 struct fib6_config r_cfg; 5159 struct list_head next; 5160 }; 5161 5162 static int ip6_route_info_append(struct net *net, 5163 struct list_head *rt6_nh_list, 5164 struct fib6_info *rt, 5165 struct fib6_config *r_cfg) 5166 { 5167 struct rt6_nh *nh; 5168 int err = -EEXIST; 5169 5170 list_for_each_entry(nh, rt6_nh_list, next) { 5171 /* check if fib6_info already exists */ 5172 if (rt6_duplicate_nexthop(nh->fib6_info, rt)) 5173 return err; 5174 } 5175 5176 nh = kzalloc(sizeof(*nh), GFP_KERNEL); 5177 if (!nh) 5178 return -ENOMEM; 5179 nh->fib6_info = rt; 5180 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg)); 5181 list_add_tail(&nh->next, rt6_nh_list); 5182 5183 return 0; 5184 } 5185 5186 static void ip6_route_mpath_notify(struct fib6_info *rt, 5187 struct fib6_info *rt_last, 5188 struct nl_info *info, 5189 __u16 nlflags) 5190 { 5191 /* if this is an APPEND route, then rt points to the first route 5192 * inserted and rt_last points to last route inserted. Userspace 5193 * wants a consistent dump of the route which starts at the first 5194 * nexthop. Since sibling routes are always added at the end of 5195 * the list, find the first sibling of the last route appended 5196 */ 5197 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) { 5198 rt = list_first_entry(&rt_last->fib6_siblings, 5199 struct fib6_info, 5200 fib6_siblings); 5201 } 5202 5203 if (rt) 5204 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags); 5205 } 5206 5207 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt) 5208 { 5209 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt); 5210 bool should_notify = false; 5211 struct fib6_info *leaf; 5212 struct fib6_node *fn; 5213 5214 rcu_read_lock(); 5215 fn = rcu_dereference(rt->fib6_node); 5216 if (!fn) 5217 goto out; 5218 5219 leaf = rcu_dereference(fn->leaf); 5220 if (!leaf) 5221 goto out; 5222 5223 if (rt == leaf || 5224 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric && 5225 rt6_qualify_for_ecmp(leaf))) 5226 should_notify = true; 5227 out: 5228 rcu_read_unlock(); 5229 5230 return should_notify; 5231 } 5232 5233 static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla, 5234 struct netlink_ext_ack *extack) 5235 { 5236 if (nla_len(nla) < sizeof(*gw)) { 5237 NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY"); 5238 return -EINVAL; 5239 } 5240 5241 *gw = nla_get_in6_addr(nla); 5242 5243 return 0; 5244 } 5245 5246 static int ip6_route_multipath_add(struct fib6_config *cfg, 5247 struct netlink_ext_ack *extack) 5248 { 5249 struct fib6_info *rt_notif = NULL, *rt_last = NULL; 5250 struct nl_info *info = &cfg->fc_nlinfo; 5251 struct fib6_config r_cfg; 5252 struct rtnexthop *rtnh; 5253 struct fib6_info *rt; 5254 struct rt6_nh *err_nh; 5255 struct rt6_nh *nh, *nh_safe; 5256 __u16 nlflags; 5257 int remaining; 5258 int attrlen; 5259 int err = 1; 5260 int nhn = 0; 5261 int replace = (cfg->fc_nlinfo.nlh && 5262 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE)); 5263 LIST_HEAD(rt6_nh_list); 5264 5265 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE; 5266 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND) 5267 nlflags |= NLM_F_APPEND; 5268 5269 remaining = cfg->fc_mp_len; 5270 rtnh = (struct rtnexthop *)cfg->fc_mp; 5271 5272 /* Parse a Multipath Entry and build a list (rt6_nh_list) of 5273 * fib6_info structs per nexthop 5274 */ 5275 while (rtnh_ok(rtnh, remaining)) { 5276 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5277 if (rtnh->rtnh_ifindex) 5278 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5279 5280 attrlen = rtnh_attrlen(rtnh); 5281 if (attrlen > 0) { 5282 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5283 5284 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5285 if (nla) { 5286 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla, 5287 extack); 5288 if (err) 5289 goto cleanup; 5290 5291 r_cfg.fc_flags |= RTF_GATEWAY; 5292 } 5293 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP); 5294 5295 /* RTA_ENCAP_TYPE length checked in 5296 * lwtunnel_valid_encap_type_attr 5297 */ 5298 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); 5299 if (nla) 5300 r_cfg.fc_encap_type = nla_get_u16(nla); 5301 } 5302 5303 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK); 5304 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack); 5305 if (IS_ERR(rt)) { 5306 err = PTR_ERR(rt); 5307 rt = NULL; 5308 goto cleanup; 5309 } 5310 if (!rt6_qualify_for_ecmp(rt)) { 5311 err = -EINVAL; 5312 NL_SET_ERR_MSG(extack, 5313 "Device only routes can not be added for IPv6 using the multipath API."); 5314 fib6_info_release(rt); 5315 goto cleanup; 5316 } 5317 5318 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1; 5319 5320 err = ip6_route_info_append(info->nl_net, &rt6_nh_list, 5321 rt, &r_cfg); 5322 if (err) { 5323 fib6_info_release(rt); 5324 goto cleanup; 5325 } 5326 5327 rtnh = rtnh_next(rtnh, &remaining); 5328 } 5329 5330 if (list_empty(&rt6_nh_list)) { 5331 NL_SET_ERR_MSG(extack, 5332 "Invalid nexthop configuration - no valid nexthops"); 5333 return -EINVAL; 5334 } 5335 5336 /* for add and replace send one notification with all nexthops. 5337 * Skip the notification in fib6_add_rt2node and send one with 5338 * the full route when done 5339 */ 5340 info->skip_notify = 1; 5341 5342 /* For add and replace, send one notification with all nexthops. For 5343 * append, send one notification with all appended nexthops. 5344 */ 5345 info->skip_notify_kernel = 1; 5346 5347 err_nh = NULL; 5348 list_for_each_entry(nh, &rt6_nh_list, next) { 5349 err = __ip6_ins_rt(nh->fib6_info, info, extack); 5350 5351 if (err) { 5352 if (replace && nhn) 5353 NL_SET_ERR_MSG_MOD(extack, 5354 "multipath route replace failed (check consistency of installed routes)"); 5355 err_nh = nh; 5356 goto add_errout; 5357 } 5358 /* save reference to last route successfully inserted */ 5359 rt_last = nh->fib6_info; 5360 5361 /* save reference to first route for notification */ 5362 if (!rt_notif) 5363 rt_notif = nh->fib6_info; 5364 5365 /* Because each route is added like a single route we remove 5366 * these flags after the first nexthop: if there is a collision, 5367 * we have already failed to add the first nexthop: 5368 * fib6_add_rt2node() has rejected it; when replacing, old 5369 * nexthops have been replaced by first new, the rest should 5370 * be added to it. 5371 */ 5372 if (cfg->fc_nlinfo.nlh) { 5373 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL | 5374 NLM_F_REPLACE); 5375 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE; 5376 } 5377 nhn++; 5378 } 5379 5380 /* An in-kernel notification should only be sent in case the new 5381 * multipath route is added as the first route in the node, or if 5382 * it was appended to it. We pass 'rt_notif' since it is the first 5383 * sibling and might allow us to skip some checks in the replace case. 5384 */ 5385 if (ip6_route_mpath_should_notify(rt_notif)) { 5386 enum fib_event_type fib_event; 5387 5388 if (rt_notif->fib6_nsiblings != nhn - 1) 5389 fib_event = FIB_EVENT_ENTRY_APPEND; 5390 else 5391 fib_event = FIB_EVENT_ENTRY_REPLACE; 5392 5393 err = call_fib6_multipath_entry_notifiers(info->nl_net, 5394 fib_event, rt_notif, 5395 nhn - 1, extack); 5396 if (err) { 5397 /* Delete all the siblings that were just added */ 5398 err_nh = NULL; 5399 goto add_errout; 5400 } 5401 } 5402 5403 /* success ... tell user about new route */ 5404 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5405 goto cleanup; 5406 5407 add_errout: 5408 /* send notification for routes that were added so that 5409 * the delete notifications sent by ip6_route_del are 5410 * coherent 5411 */ 5412 if (rt_notif) 5413 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5414 5415 /* Delete routes that were already added */ 5416 list_for_each_entry(nh, &rt6_nh_list, next) { 5417 if (err_nh == nh) 5418 break; 5419 ip6_route_del(&nh->r_cfg, extack); 5420 } 5421 5422 cleanup: 5423 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) { 5424 fib6_info_release(nh->fib6_info); 5425 list_del(&nh->next); 5426 kfree(nh); 5427 } 5428 5429 return err; 5430 } 5431 5432 static int ip6_route_multipath_del(struct fib6_config *cfg, 5433 struct netlink_ext_ack *extack) 5434 { 5435 struct fib6_config r_cfg; 5436 struct rtnexthop *rtnh; 5437 int last_err = 0; 5438 int remaining; 5439 int attrlen; 5440 int err; 5441 5442 remaining = cfg->fc_mp_len; 5443 rtnh = (struct rtnexthop *)cfg->fc_mp; 5444 5445 /* Parse a Multipath Entry */ 5446 while (rtnh_ok(rtnh, remaining)) { 5447 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5448 if (rtnh->rtnh_ifindex) 5449 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5450 5451 attrlen = rtnh_attrlen(rtnh); 5452 if (attrlen > 0) { 5453 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5454 5455 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5456 if (nla) { 5457 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla, 5458 extack); 5459 if (err) { 5460 last_err = err; 5461 goto next_rtnh; 5462 } 5463 5464 r_cfg.fc_flags |= RTF_GATEWAY; 5465 } 5466 } 5467 err = ip6_route_del(&r_cfg, extack); 5468 if (err) 5469 last_err = err; 5470 5471 next_rtnh: 5472 rtnh = rtnh_next(rtnh, &remaining); 5473 } 5474 5475 return last_err; 5476 } 5477 5478 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5479 struct netlink_ext_ack *extack) 5480 { 5481 struct fib6_config cfg; 5482 int err; 5483 5484 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5485 if (err < 0) 5486 return err; 5487 5488 if (cfg.fc_nh_id && 5489 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) { 5490 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 5491 return -EINVAL; 5492 } 5493 5494 if (cfg.fc_mp) 5495 return ip6_route_multipath_del(&cfg, extack); 5496 else { 5497 cfg.fc_delete_all_nh = 1; 5498 return ip6_route_del(&cfg, extack); 5499 } 5500 } 5501 5502 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5503 struct netlink_ext_ack *extack) 5504 { 5505 struct fib6_config cfg; 5506 int err; 5507 5508 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5509 if (err < 0) 5510 return err; 5511 5512 if (cfg.fc_metric == 0) 5513 cfg.fc_metric = IP6_RT_PRIO_USER; 5514 5515 if (cfg.fc_mp) 5516 return ip6_route_multipath_add(&cfg, extack); 5517 else 5518 return ip6_route_add(&cfg, GFP_KERNEL, extack); 5519 } 5520 5521 /* add the overhead of this fib6_nh to nexthop_len */ 5522 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg) 5523 { 5524 int *nexthop_len = arg; 5525 5526 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */ 5527 + NLA_ALIGN(sizeof(struct rtnexthop)) 5528 + nla_total_size(16); /* RTA_GATEWAY */ 5529 5530 if (nh->fib_nh_lws) { 5531 /* RTA_ENCAP_TYPE */ 5532 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5533 /* RTA_ENCAP */ 5534 *nexthop_len += nla_total_size(2); 5535 } 5536 5537 return 0; 5538 } 5539 5540 static size_t rt6_nlmsg_size(struct fib6_info *f6i) 5541 { 5542 int nexthop_len; 5543 5544 if (f6i->nh) { 5545 nexthop_len = nla_total_size(4); /* RTA_NH_ID */ 5546 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size, 5547 &nexthop_len); 5548 } else { 5549 struct fib6_info *sibling, *next_sibling; 5550 struct fib6_nh *nh = f6i->fib6_nh; 5551 5552 nexthop_len = 0; 5553 if (f6i->fib6_nsiblings) { 5554 rt6_nh_nlmsg_size(nh, &nexthop_len); 5555 5556 list_for_each_entry_safe(sibling, next_sibling, 5557 &f6i->fib6_siblings, fib6_siblings) { 5558 rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len); 5559 } 5560 } 5561 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5562 } 5563 5564 return NLMSG_ALIGN(sizeof(struct rtmsg)) 5565 + nla_total_size(16) /* RTA_SRC */ 5566 + nla_total_size(16) /* RTA_DST */ 5567 + nla_total_size(16) /* RTA_GATEWAY */ 5568 + nla_total_size(16) /* RTA_PREFSRC */ 5569 + nla_total_size(4) /* RTA_TABLE */ 5570 + nla_total_size(4) /* RTA_IIF */ 5571 + nla_total_size(4) /* RTA_OIF */ 5572 + nla_total_size(4) /* RTA_PRIORITY */ 5573 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */ 5574 + nla_total_size(sizeof(struct rta_cacheinfo)) 5575 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */ 5576 + nla_total_size(1) /* RTA_PREF */ 5577 + nexthop_len; 5578 } 5579 5580 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh, 5581 unsigned char *flags) 5582 { 5583 if (nexthop_is_multipath(nh)) { 5584 struct nlattr *mp; 5585 5586 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5587 if (!mp) 5588 goto nla_put_failure; 5589 5590 if (nexthop_mpath_fill_node(skb, nh, AF_INET6)) 5591 goto nla_put_failure; 5592 5593 nla_nest_end(skb, mp); 5594 } else { 5595 struct fib6_nh *fib6_nh; 5596 5597 fib6_nh = nexthop_fib6_nh(nh); 5598 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6, 5599 flags, false) < 0) 5600 goto nla_put_failure; 5601 } 5602 5603 return 0; 5604 5605 nla_put_failure: 5606 return -EMSGSIZE; 5607 } 5608 5609 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 5610 struct fib6_info *rt, struct dst_entry *dst, 5611 struct in6_addr *dest, struct in6_addr *src, 5612 int iif, int type, u32 portid, u32 seq, 5613 unsigned int flags) 5614 { 5615 struct rt6_info *rt6 = dst_rt6_info(dst); 5616 struct rt6key *rt6_dst, *rt6_src; 5617 u32 *pmetrics, table, rt6_flags; 5618 unsigned char nh_flags = 0; 5619 struct nlmsghdr *nlh; 5620 struct rtmsg *rtm; 5621 long expires = 0; 5622 5623 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags); 5624 if (!nlh) 5625 return -EMSGSIZE; 5626 5627 if (rt6) { 5628 rt6_dst = &rt6->rt6i_dst; 5629 rt6_src = &rt6->rt6i_src; 5630 rt6_flags = rt6->rt6i_flags; 5631 } else { 5632 rt6_dst = &rt->fib6_dst; 5633 rt6_src = &rt->fib6_src; 5634 rt6_flags = rt->fib6_flags; 5635 } 5636 5637 rtm = nlmsg_data(nlh); 5638 rtm->rtm_family = AF_INET6; 5639 rtm->rtm_dst_len = rt6_dst->plen; 5640 rtm->rtm_src_len = rt6_src->plen; 5641 rtm->rtm_tos = 0; 5642 if (rt->fib6_table) 5643 table = rt->fib6_table->tb6_id; 5644 else 5645 table = RT6_TABLE_UNSPEC; 5646 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT; 5647 if (nla_put_u32(skb, RTA_TABLE, table)) 5648 goto nla_put_failure; 5649 5650 rtm->rtm_type = rt->fib6_type; 5651 rtm->rtm_flags = 0; 5652 rtm->rtm_scope = RT_SCOPE_UNIVERSE; 5653 rtm->rtm_protocol = rt->fib6_protocol; 5654 5655 if (rt6_flags & RTF_CACHE) 5656 rtm->rtm_flags |= RTM_F_CLONED; 5657 5658 if (dest) { 5659 if (nla_put_in6_addr(skb, RTA_DST, dest)) 5660 goto nla_put_failure; 5661 rtm->rtm_dst_len = 128; 5662 } else if (rtm->rtm_dst_len) 5663 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr)) 5664 goto nla_put_failure; 5665 #ifdef CONFIG_IPV6_SUBTREES 5666 if (src) { 5667 if (nla_put_in6_addr(skb, RTA_SRC, src)) 5668 goto nla_put_failure; 5669 rtm->rtm_src_len = 128; 5670 } else if (rtm->rtm_src_len && 5671 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr)) 5672 goto nla_put_failure; 5673 #endif 5674 if (iif) { 5675 #ifdef CONFIG_IPV6_MROUTE 5676 if (ipv6_addr_is_multicast(&rt6_dst->addr)) { 5677 int err = ip6mr_get_route(net, skb, rtm, portid); 5678 5679 if (err == 0) 5680 return 0; 5681 if (err < 0) 5682 goto nla_put_failure; 5683 } else 5684 #endif 5685 if (nla_put_u32(skb, RTA_IIF, iif)) 5686 goto nla_put_failure; 5687 } else if (dest) { 5688 struct in6_addr saddr_buf; 5689 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 && 5690 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5691 goto nla_put_failure; 5692 } 5693 5694 if (rt->fib6_prefsrc.plen) { 5695 struct in6_addr saddr_buf; 5696 saddr_buf = rt->fib6_prefsrc.addr; 5697 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5698 goto nla_put_failure; 5699 } 5700 5701 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics; 5702 if (rtnetlink_put_metrics(skb, pmetrics) < 0) 5703 goto nla_put_failure; 5704 5705 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric)) 5706 goto nla_put_failure; 5707 5708 /* For multipath routes, walk the siblings list and add 5709 * each as a nexthop within RTA_MULTIPATH. 5710 */ 5711 if (rt6) { 5712 if (rt6_flags & RTF_GATEWAY && 5713 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway)) 5714 goto nla_put_failure; 5715 5716 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex)) 5717 goto nla_put_failure; 5718 5719 if (dst->lwtstate && 5720 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 5721 goto nla_put_failure; 5722 } else if (rt->fib6_nsiblings) { 5723 struct fib6_info *sibling, *next_sibling; 5724 struct nlattr *mp; 5725 5726 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5727 if (!mp) 5728 goto nla_put_failure; 5729 5730 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common, 5731 rt->fib6_nh->fib_nh_weight, AF_INET6, 5732 0) < 0) 5733 goto nla_put_failure; 5734 5735 list_for_each_entry_safe(sibling, next_sibling, 5736 &rt->fib6_siblings, fib6_siblings) { 5737 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common, 5738 sibling->fib6_nh->fib_nh_weight, 5739 AF_INET6, 0) < 0) 5740 goto nla_put_failure; 5741 } 5742 5743 nla_nest_end(skb, mp); 5744 } else if (rt->nh) { 5745 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id)) 5746 goto nla_put_failure; 5747 5748 if (nexthop_is_blackhole(rt->nh)) 5749 rtm->rtm_type = RTN_BLACKHOLE; 5750 5751 if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) && 5752 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0) 5753 goto nla_put_failure; 5754 5755 rtm->rtm_flags |= nh_flags; 5756 } else { 5757 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6, 5758 &nh_flags, false) < 0) 5759 goto nla_put_failure; 5760 5761 rtm->rtm_flags |= nh_flags; 5762 } 5763 5764 if (rt6_flags & RTF_EXPIRES) { 5765 expires = dst ? dst->expires : rt->expires; 5766 expires -= jiffies; 5767 } 5768 5769 if (!dst) { 5770 if (READ_ONCE(rt->offload)) 5771 rtm->rtm_flags |= RTM_F_OFFLOAD; 5772 if (READ_ONCE(rt->trap)) 5773 rtm->rtm_flags |= RTM_F_TRAP; 5774 if (READ_ONCE(rt->offload_failed)) 5775 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED; 5776 } 5777 5778 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0) 5779 goto nla_put_failure; 5780 5781 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags))) 5782 goto nla_put_failure; 5783 5784 5785 nlmsg_end(skb, nlh); 5786 return 0; 5787 5788 nla_put_failure: 5789 nlmsg_cancel(skb, nlh); 5790 return -EMSGSIZE; 5791 } 5792 5793 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg) 5794 { 5795 const struct net_device *dev = arg; 5796 5797 if (nh->fib_nh_dev == dev) 5798 return 1; 5799 5800 return 0; 5801 } 5802 5803 static bool fib6_info_uses_dev(const struct fib6_info *f6i, 5804 const struct net_device *dev) 5805 { 5806 if (f6i->nh) { 5807 struct net_device *_dev = (struct net_device *)dev; 5808 5809 return !!nexthop_for_each_fib6_nh(f6i->nh, 5810 fib6_info_nh_uses_dev, 5811 _dev); 5812 } 5813 5814 if (f6i->fib6_nh->fib_nh_dev == dev) 5815 return true; 5816 5817 if (f6i->fib6_nsiblings) { 5818 struct fib6_info *sibling, *next_sibling; 5819 5820 list_for_each_entry_safe(sibling, next_sibling, 5821 &f6i->fib6_siblings, fib6_siblings) { 5822 if (sibling->fib6_nh->fib_nh_dev == dev) 5823 return true; 5824 } 5825 } 5826 5827 return false; 5828 } 5829 5830 struct fib6_nh_exception_dump_walker { 5831 struct rt6_rtnl_dump_arg *dump; 5832 struct fib6_info *rt; 5833 unsigned int flags; 5834 unsigned int skip; 5835 unsigned int count; 5836 }; 5837 5838 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) 5839 { 5840 struct fib6_nh_exception_dump_walker *w = arg; 5841 struct rt6_rtnl_dump_arg *dump = w->dump; 5842 struct rt6_exception_bucket *bucket; 5843 struct rt6_exception *rt6_ex; 5844 int i, err; 5845 5846 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 5847 if (!bucket) 5848 return 0; 5849 5850 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 5851 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 5852 if (w->skip) { 5853 w->skip--; 5854 continue; 5855 } 5856 5857 /* Expiration of entries doesn't bump sernum, insertion 5858 * does. Removal is triggered by insertion, so we can 5859 * rely on the fact that if entries change between two 5860 * partial dumps, this node is scanned again completely, 5861 * see rt6_insert_exception() and fib6_dump_table(). 5862 * 5863 * Count expired entries we go through as handled 5864 * entries that we'll skip next time, in case of partial 5865 * node dump. Otherwise, if entries expire meanwhile, 5866 * we'll skip the wrong amount. 5867 */ 5868 if (rt6_check_expired(rt6_ex->rt6i)) { 5869 w->count++; 5870 continue; 5871 } 5872 5873 err = rt6_fill_node(dump->net, dump->skb, w->rt, 5874 &rt6_ex->rt6i->dst, NULL, NULL, 0, 5875 RTM_NEWROUTE, 5876 NETLINK_CB(dump->cb->skb).portid, 5877 dump->cb->nlh->nlmsg_seq, w->flags); 5878 if (err) 5879 return err; 5880 5881 w->count++; 5882 } 5883 bucket++; 5884 } 5885 5886 return 0; 5887 } 5888 5889 /* Return -1 if done with node, number of handled routes on partial dump */ 5890 int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip) 5891 { 5892 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg; 5893 struct fib_dump_filter *filter = &arg->filter; 5894 unsigned int flags = NLM_F_MULTI; 5895 struct net *net = arg->net; 5896 int count = 0; 5897 5898 if (rt == net->ipv6.fib6_null_entry) 5899 return -1; 5900 5901 if ((filter->flags & RTM_F_PREFIX) && 5902 !(rt->fib6_flags & RTF_PREFIX_RT)) { 5903 /* success since this is not a prefix route */ 5904 return -1; 5905 } 5906 if (filter->filter_set && 5907 ((filter->rt_type && rt->fib6_type != filter->rt_type) || 5908 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) || 5909 (filter->protocol && rt->fib6_protocol != filter->protocol))) { 5910 return -1; 5911 } 5912 5913 if (filter->filter_set || 5914 !filter->dump_routes || !filter->dump_exceptions) { 5915 flags |= NLM_F_DUMP_FILTERED; 5916 } 5917 5918 if (filter->dump_routes) { 5919 if (skip) { 5920 skip--; 5921 } else { 5922 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 5923 0, RTM_NEWROUTE, 5924 NETLINK_CB(arg->cb->skb).portid, 5925 arg->cb->nlh->nlmsg_seq, flags)) { 5926 return 0; 5927 } 5928 count++; 5929 } 5930 } 5931 5932 if (filter->dump_exceptions) { 5933 struct fib6_nh_exception_dump_walker w = { .dump = arg, 5934 .rt = rt, 5935 .flags = flags, 5936 .skip = skip, 5937 .count = 0 }; 5938 int err; 5939 5940 rcu_read_lock(); 5941 if (rt->nh) { 5942 err = nexthop_for_each_fib6_nh(rt->nh, 5943 rt6_nh_dump_exceptions, 5944 &w); 5945 } else { 5946 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w); 5947 } 5948 rcu_read_unlock(); 5949 5950 if (err) 5951 return count + w.count; 5952 } 5953 5954 return -1; 5955 } 5956 5957 static int inet6_rtm_valid_getroute_req(struct sk_buff *skb, 5958 const struct nlmsghdr *nlh, 5959 struct nlattr **tb, 5960 struct netlink_ext_ack *extack) 5961 { 5962 struct rtmsg *rtm; 5963 int i, err; 5964 5965 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 5966 NL_SET_ERR_MSG_MOD(extack, 5967 "Invalid header for get route request"); 5968 return -EINVAL; 5969 } 5970 5971 if (!netlink_strict_get_check(skb)) 5972 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5973 rtm_ipv6_policy, extack); 5974 5975 rtm = nlmsg_data(nlh); 5976 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) || 5977 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) || 5978 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope || 5979 rtm->rtm_type) { 5980 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request"); 5981 return -EINVAL; 5982 } 5983 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) { 5984 NL_SET_ERR_MSG_MOD(extack, 5985 "Invalid flags for get route request"); 5986 return -EINVAL; 5987 } 5988 5989 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 5990 rtm_ipv6_policy, extack); 5991 if (err) 5992 return err; 5993 5994 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 5995 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 5996 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6"); 5997 return -EINVAL; 5998 } 5999 6000 for (i = 0; i <= RTA_MAX; i++) { 6001 if (!tb[i]) 6002 continue; 6003 6004 switch (i) { 6005 case RTA_SRC: 6006 case RTA_DST: 6007 case RTA_IIF: 6008 case RTA_OIF: 6009 case RTA_MARK: 6010 case RTA_UID: 6011 case RTA_SPORT: 6012 case RTA_DPORT: 6013 case RTA_IP_PROTO: 6014 break; 6015 default: 6016 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request"); 6017 return -EINVAL; 6018 } 6019 } 6020 6021 return 0; 6022 } 6023 6024 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 6025 struct netlink_ext_ack *extack) 6026 { 6027 struct net *net = sock_net(in_skb->sk); 6028 struct nlattr *tb[RTA_MAX+1]; 6029 int err, iif = 0, oif = 0; 6030 struct fib6_info *from; 6031 struct dst_entry *dst; 6032 struct rt6_info *rt; 6033 struct sk_buff *skb; 6034 struct rtmsg *rtm; 6035 struct flowi6 fl6 = {}; 6036 bool fibmatch; 6037 6038 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 6039 if (err < 0) 6040 goto errout; 6041 6042 err = -EINVAL; 6043 rtm = nlmsg_data(nlh); 6044 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0); 6045 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH); 6046 6047 if (tb[RTA_SRC]) { 6048 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr)) 6049 goto errout; 6050 6051 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]); 6052 } 6053 6054 if (tb[RTA_DST]) { 6055 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr)) 6056 goto errout; 6057 6058 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]); 6059 } 6060 6061 if (tb[RTA_IIF]) 6062 iif = nla_get_u32(tb[RTA_IIF]); 6063 6064 if (tb[RTA_OIF]) 6065 oif = nla_get_u32(tb[RTA_OIF]); 6066 6067 if (tb[RTA_MARK]) 6068 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]); 6069 6070 if (tb[RTA_UID]) 6071 fl6.flowi6_uid = make_kuid(current_user_ns(), 6072 nla_get_u32(tb[RTA_UID])); 6073 else 6074 fl6.flowi6_uid = iif ? INVALID_UID : current_uid(); 6075 6076 if (tb[RTA_SPORT]) 6077 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]); 6078 6079 if (tb[RTA_DPORT]) 6080 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]); 6081 6082 if (tb[RTA_IP_PROTO]) { 6083 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 6084 &fl6.flowi6_proto, AF_INET6, 6085 extack); 6086 if (err) 6087 goto errout; 6088 } 6089 6090 if (iif) { 6091 struct net_device *dev; 6092 int flags = 0; 6093 6094 rcu_read_lock(); 6095 6096 dev = dev_get_by_index_rcu(net, iif); 6097 if (!dev) { 6098 rcu_read_unlock(); 6099 err = -ENODEV; 6100 goto errout; 6101 } 6102 6103 fl6.flowi6_iif = iif; 6104 6105 if (!ipv6_addr_any(&fl6.saddr)) 6106 flags |= RT6_LOOKUP_F_HAS_SADDR; 6107 6108 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags); 6109 6110 rcu_read_unlock(); 6111 } else { 6112 fl6.flowi6_oif = oif; 6113 6114 dst = ip6_route_output(net, NULL, &fl6); 6115 } 6116 6117 6118 rt = dst_rt6_info(dst); 6119 if (rt->dst.error) { 6120 err = rt->dst.error; 6121 ip6_rt_put(rt); 6122 goto errout; 6123 } 6124 6125 if (rt == net->ipv6.ip6_null_entry) { 6126 err = rt->dst.error; 6127 ip6_rt_put(rt); 6128 goto errout; 6129 } 6130 6131 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 6132 if (!skb) { 6133 ip6_rt_put(rt); 6134 err = -ENOBUFS; 6135 goto errout; 6136 } 6137 6138 skb_dst_set(skb, &rt->dst); 6139 6140 rcu_read_lock(); 6141 from = rcu_dereference(rt->from); 6142 if (from) { 6143 if (fibmatch) 6144 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, 6145 iif, RTM_NEWROUTE, 6146 NETLINK_CB(in_skb).portid, 6147 nlh->nlmsg_seq, 0); 6148 else 6149 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr, 6150 &fl6.saddr, iif, RTM_NEWROUTE, 6151 NETLINK_CB(in_skb).portid, 6152 nlh->nlmsg_seq, 0); 6153 } else { 6154 err = -ENETUNREACH; 6155 } 6156 rcu_read_unlock(); 6157 6158 if (err < 0) { 6159 kfree_skb(skb); 6160 goto errout; 6161 } 6162 6163 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 6164 errout: 6165 return err; 6166 } 6167 6168 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, 6169 unsigned int nlm_flags) 6170 { 6171 struct sk_buff *skb; 6172 struct net *net = info->nl_net; 6173 u32 seq; 6174 int err; 6175 6176 err = -ENOBUFS; 6177 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6178 6179 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6180 if (!skb) 6181 goto errout; 6182 6183 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6184 event, info->portid, seq, nlm_flags); 6185 if (err < 0) { 6186 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6187 WARN_ON(err == -EMSGSIZE); 6188 kfree_skb(skb); 6189 goto errout; 6190 } 6191 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6192 info->nlh, gfp_any()); 6193 return; 6194 errout: 6195 if (err < 0) 6196 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6197 } 6198 6199 void fib6_rt_update(struct net *net, struct fib6_info *rt, 6200 struct nl_info *info) 6201 { 6202 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6203 struct sk_buff *skb; 6204 int err = -ENOBUFS; 6205 6206 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6207 if (!skb) 6208 goto errout; 6209 6210 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6211 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE); 6212 if (err < 0) { 6213 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6214 WARN_ON(err == -EMSGSIZE); 6215 kfree_skb(skb); 6216 goto errout; 6217 } 6218 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6219 info->nlh, gfp_any()); 6220 return; 6221 errout: 6222 if (err < 0) 6223 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6224 } 6225 6226 void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, 6227 bool offload, bool trap, bool offload_failed) 6228 { 6229 struct sk_buff *skb; 6230 int err; 6231 6232 if (READ_ONCE(f6i->offload) == offload && 6233 READ_ONCE(f6i->trap) == trap && 6234 READ_ONCE(f6i->offload_failed) == offload_failed) 6235 return; 6236 6237 WRITE_ONCE(f6i->offload, offload); 6238 WRITE_ONCE(f6i->trap, trap); 6239 6240 /* 2 means send notifications only if offload_failed was changed. */ 6241 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && 6242 READ_ONCE(f6i->offload_failed) == offload_failed) 6243 return; 6244 6245 WRITE_ONCE(f6i->offload_failed, offload_failed); 6246 6247 if (!rcu_access_pointer(f6i->fib6_node)) 6248 /* The route was removed from the tree, do not send 6249 * notification. 6250 */ 6251 return; 6252 6253 if (!net->ipv6.sysctl.fib_notify_on_flag_change) 6254 return; 6255 6256 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); 6257 if (!skb) { 6258 err = -ENOBUFS; 6259 goto errout; 6260 } 6261 6262 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0, 6263 0, 0); 6264 if (err < 0) { 6265 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6266 WARN_ON(err == -EMSGSIZE); 6267 kfree_skb(skb); 6268 goto errout; 6269 } 6270 6271 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL); 6272 return; 6273 6274 errout: 6275 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6276 } 6277 EXPORT_SYMBOL(fib6_info_hw_flags_set); 6278 6279 static int ip6_route_dev_notify(struct notifier_block *this, 6280 unsigned long event, void *ptr) 6281 { 6282 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6283 struct net *net = dev_net(dev); 6284 6285 if (!(dev->flags & IFF_LOOPBACK)) 6286 return NOTIFY_OK; 6287 6288 if (event == NETDEV_REGISTER) { 6289 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev; 6290 net->ipv6.ip6_null_entry->dst.dev = dev; 6291 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev); 6292 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6293 net->ipv6.ip6_prohibit_entry->dst.dev = dev; 6294 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev); 6295 net->ipv6.ip6_blk_hole_entry->dst.dev = dev; 6296 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev); 6297 #endif 6298 } else if (event == NETDEV_UNREGISTER && 6299 dev->reg_state != NETREG_UNREGISTERED) { 6300 /* NETDEV_UNREGISTER could be fired for multiple times by 6301 * netdev_wait_allrefs(). Make sure we only call this once. 6302 */ 6303 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev); 6304 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6305 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev); 6306 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev); 6307 #endif 6308 } 6309 6310 return NOTIFY_OK; 6311 } 6312 6313 /* 6314 * /proc 6315 */ 6316 6317 #ifdef CONFIG_PROC_FS 6318 static int rt6_stats_seq_show(struct seq_file *seq, void *v) 6319 { 6320 struct net *net = (struct net *)seq->private; 6321 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n", 6322 net->ipv6.rt6_stats->fib_nodes, 6323 net->ipv6.rt6_stats->fib_route_nodes, 6324 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc), 6325 net->ipv6.rt6_stats->fib_rt_entries, 6326 net->ipv6.rt6_stats->fib_rt_cache, 6327 dst_entries_get_slow(&net->ipv6.ip6_dst_ops), 6328 net->ipv6.rt6_stats->fib_discarded_routes); 6329 6330 return 0; 6331 } 6332 #endif /* CONFIG_PROC_FS */ 6333 6334 #ifdef CONFIG_SYSCTL 6335 6336 static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write, 6337 void *buffer, size_t *lenp, loff_t *ppos) 6338 { 6339 struct net *net; 6340 int delay; 6341 int ret; 6342 if (!write) 6343 return -EINVAL; 6344 6345 net = (struct net *)ctl->extra1; 6346 delay = net->ipv6.sysctl.flush_delay; 6347 ret = proc_dointvec(ctl, write, buffer, lenp, ppos); 6348 if (ret) 6349 return ret; 6350 6351 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); 6352 return 0; 6353 } 6354 6355 static struct ctl_table ipv6_route_table_template[] = { 6356 { 6357 .procname = "max_size", 6358 .data = &init_net.ipv6.sysctl.ip6_rt_max_size, 6359 .maxlen = sizeof(int), 6360 .mode = 0644, 6361 .proc_handler = proc_dointvec, 6362 }, 6363 { 6364 .procname = "gc_thresh", 6365 .data = &ip6_dst_ops_template.gc_thresh, 6366 .maxlen = sizeof(int), 6367 .mode = 0644, 6368 .proc_handler = proc_dointvec, 6369 }, 6370 { 6371 .procname = "flush", 6372 .data = &init_net.ipv6.sysctl.flush_delay, 6373 .maxlen = sizeof(int), 6374 .mode = 0200, 6375 .proc_handler = ipv6_sysctl_rtcache_flush 6376 }, 6377 { 6378 .procname = "gc_min_interval", 6379 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6380 .maxlen = sizeof(int), 6381 .mode = 0644, 6382 .proc_handler = proc_dointvec_jiffies, 6383 }, 6384 { 6385 .procname = "gc_timeout", 6386 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout, 6387 .maxlen = sizeof(int), 6388 .mode = 0644, 6389 .proc_handler = proc_dointvec_jiffies, 6390 }, 6391 { 6392 .procname = "gc_interval", 6393 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval, 6394 .maxlen = sizeof(int), 6395 .mode = 0644, 6396 .proc_handler = proc_dointvec_jiffies, 6397 }, 6398 { 6399 .procname = "gc_elasticity", 6400 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity, 6401 .maxlen = sizeof(int), 6402 .mode = 0644, 6403 .proc_handler = proc_dointvec, 6404 }, 6405 { 6406 .procname = "mtu_expires", 6407 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires, 6408 .maxlen = sizeof(int), 6409 .mode = 0644, 6410 .proc_handler = proc_dointvec_jiffies, 6411 }, 6412 { 6413 .procname = "min_adv_mss", 6414 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss, 6415 .maxlen = sizeof(int), 6416 .mode = 0644, 6417 .proc_handler = proc_dointvec, 6418 }, 6419 { 6420 .procname = "gc_min_interval_ms", 6421 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6422 .maxlen = sizeof(int), 6423 .mode = 0644, 6424 .proc_handler = proc_dointvec_ms_jiffies, 6425 }, 6426 { 6427 .procname = "skip_notify_on_dev_down", 6428 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down, 6429 .maxlen = sizeof(u8), 6430 .mode = 0644, 6431 .proc_handler = proc_dou8vec_minmax, 6432 .extra1 = SYSCTL_ZERO, 6433 .extra2 = SYSCTL_ONE, 6434 }, 6435 }; 6436 6437 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net) 6438 { 6439 struct ctl_table *table; 6440 6441 table = kmemdup(ipv6_route_table_template, 6442 sizeof(ipv6_route_table_template), 6443 GFP_KERNEL); 6444 6445 if (table) { 6446 table[0].data = &net->ipv6.sysctl.ip6_rt_max_size; 6447 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh; 6448 table[2].data = &net->ipv6.sysctl.flush_delay; 6449 table[2].extra1 = net; 6450 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6451 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout; 6452 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval; 6453 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity; 6454 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires; 6455 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss; 6456 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6457 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down; 6458 } 6459 6460 return table; 6461 } 6462 6463 size_t ipv6_route_sysctl_table_size(struct net *net) 6464 { 6465 /* Don't export sysctls to unprivileged users */ 6466 if (net->user_ns != &init_user_ns) 6467 return 1; 6468 6469 return ARRAY_SIZE(ipv6_route_table_template); 6470 } 6471 #endif 6472 6473 static int __net_init ip6_route_net_init(struct net *net) 6474 { 6475 int ret = -ENOMEM; 6476 6477 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template, 6478 sizeof(net->ipv6.ip6_dst_ops)); 6479 6480 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0) 6481 goto out_ip6_dst_ops; 6482 6483 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true); 6484 if (!net->ipv6.fib6_null_entry) 6485 goto out_ip6_dst_entries; 6486 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template, 6487 sizeof(*net->ipv6.fib6_null_entry)); 6488 6489 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template, 6490 sizeof(*net->ipv6.ip6_null_entry), 6491 GFP_KERNEL); 6492 if (!net->ipv6.ip6_null_entry) 6493 goto out_fib6_null_entry; 6494 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6495 dst_init_metrics(&net->ipv6.ip6_null_entry->dst, 6496 ip6_template_metrics, true); 6497 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->dst.rt_uncached); 6498 6499 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6500 net->ipv6.fib6_has_custom_rules = false; 6501 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template, 6502 sizeof(*net->ipv6.ip6_prohibit_entry), 6503 GFP_KERNEL); 6504 if (!net->ipv6.ip6_prohibit_entry) 6505 goto out_ip6_null_entry; 6506 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6507 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst, 6508 ip6_template_metrics, true); 6509 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->dst.rt_uncached); 6510 6511 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template, 6512 sizeof(*net->ipv6.ip6_blk_hole_entry), 6513 GFP_KERNEL); 6514 if (!net->ipv6.ip6_blk_hole_entry) 6515 goto out_ip6_prohibit_entry; 6516 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6517 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst, 6518 ip6_template_metrics, true); 6519 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->dst.rt_uncached); 6520 #ifdef CONFIG_IPV6_SUBTREES 6521 net->ipv6.fib6_routes_require_src = 0; 6522 #endif 6523 #endif 6524 6525 net->ipv6.sysctl.flush_delay = 0; 6526 net->ipv6.sysctl.ip6_rt_max_size = INT_MAX; 6527 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2; 6528 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ; 6529 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ; 6530 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9; 6531 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ; 6532 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40; 6533 net->ipv6.sysctl.skip_notify_on_dev_down = 0; 6534 6535 atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ); 6536 6537 ret = 0; 6538 out: 6539 return ret; 6540 6541 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6542 out_ip6_prohibit_entry: 6543 kfree(net->ipv6.ip6_prohibit_entry); 6544 out_ip6_null_entry: 6545 kfree(net->ipv6.ip6_null_entry); 6546 #endif 6547 out_fib6_null_entry: 6548 kfree(net->ipv6.fib6_null_entry); 6549 out_ip6_dst_entries: 6550 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6551 out_ip6_dst_ops: 6552 goto out; 6553 } 6554 6555 static void __net_exit ip6_route_net_exit(struct net *net) 6556 { 6557 kfree(net->ipv6.fib6_null_entry); 6558 kfree(net->ipv6.ip6_null_entry); 6559 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6560 kfree(net->ipv6.ip6_prohibit_entry); 6561 kfree(net->ipv6.ip6_blk_hole_entry); 6562 #endif 6563 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6564 } 6565 6566 static int __net_init ip6_route_net_init_late(struct net *net) 6567 { 6568 #ifdef CONFIG_PROC_FS 6569 if (!proc_create_net("ipv6_route", 0, net->proc_net, 6570 &ipv6_route_seq_ops, 6571 sizeof(struct ipv6_route_iter))) 6572 return -ENOMEM; 6573 6574 if (!proc_create_net_single("rt6_stats", 0444, net->proc_net, 6575 rt6_stats_seq_show, NULL)) { 6576 remove_proc_entry("ipv6_route", net->proc_net); 6577 return -ENOMEM; 6578 } 6579 #endif 6580 return 0; 6581 } 6582 6583 static void __net_exit ip6_route_net_exit_late(struct net *net) 6584 { 6585 #ifdef CONFIG_PROC_FS 6586 remove_proc_entry("ipv6_route", net->proc_net); 6587 remove_proc_entry("rt6_stats", net->proc_net); 6588 #endif 6589 } 6590 6591 static struct pernet_operations ip6_route_net_ops = { 6592 .init = ip6_route_net_init, 6593 .exit = ip6_route_net_exit, 6594 }; 6595 6596 static int __net_init ipv6_inetpeer_init(struct net *net) 6597 { 6598 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 6599 6600 if (!bp) 6601 return -ENOMEM; 6602 inet_peer_base_init(bp); 6603 net->ipv6.peers = bp; 6604 return 0; 6605 } 6606 6607 static void __net_exit ipv6_inetpeer_exit(struct net *net) 6608 { 6609 struct inet_peer_base *bp = net->ipv6.peers; 6610 6611 net->ipv6.peers = NULL; 6612 inetpeer_invalidate_tree(bp); 6613 kfree(bp); 6614 } 6615 6616 static struct pernet_operations ipv6_inetpeer_ops = { 6617 .init = ipv6_inetpeer_init, 6618 .exit = ipv6_inetpeer_exit, 6619 }; 6620 6621 static struct pernet_operations ip6_route_net_late_ops = { 6622 .init = ip6_route_net_init_late, 6623 .exit = ip6_route_net_exit_late, 6624 }; 6625 6626 static struct notifier_block ip6_route_dev_notifier = { 6627 .notifier_call = ip6_route_dev_notify, 6628 .priority = ADDRCONF_NOTIFY_PRIORITY - 10, 6629 }; 6630 6631 void __init ip6_route_init_special_entries(void) 6632 { 6633 /* Registering of the loopback is done before this portion of code, 6634 * the loopback reference in rt6_info will not be taken, do it 6635 * manually for init_net */ 6636 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev; 6637 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev; 6638 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6639 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6640 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev; 6641 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6642 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev; 6643 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6644 #endif 6645 } 6646 6647 #if IS_BUILTIN(CONFIG_IPV6) 6648 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6649 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt) 6650 6651 BTF_ID_LIST(btf_fib6_info_id) 6652 BTF_ID(struct, fib6_info) 6653 6654 static const struct bpf_iter_seq_info ipv6_route_seq_info = { 6655 .seq_ops = &ipv6_route_seq_ops, 6656 .init_seq_private = bpf_iter_init_seq_net, 6657 .fini_seq_private = bpf_iter_fini_seq_net, 6658 .seq_priv_size = sizeof(struct ipv6_route_iter), 6659 }; 6660 6661 static struct bpf_iter_reg ipv6_route_reg_info = { 6662 .target = "ipv6_route", 6663 .ctx_arg_info_size = 1, 6664 .ctx_arg_info = { 6665 { offsetof(struct bpf_iter__ipv6_route, rt), 6666 PTR_TO_BTF_ID_OR_NULL }, 6667 }, 6668 .seq_info = &ipv6_route_seq_info, 6669 }; 6670 6671 static int __init bpf_iter_register(void) 6672 { 6673 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id; 6674 return bpf_iter_reg_target(&ipv6_route_reg_info); 6675 } 6676 6677 static void bpf_iter_unregister(void) 6678 { 6679 bpf_iter_unreg_target(&ipv6_route_reg_info); 6680 } 6681 #endif 6682 #endif 6683 6684 int __init ip6_route_init(void) 6685 { 6686 int ret; 6687 int cpu; 6688 6689 ret = -ENOMEM; 6690 ip6_dst_ops_template.kmem_cachep = 6691 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0, 6692 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL); 6693 if (!ip6_dst_ops_template.kmem_cachep) 6694 goto out; 6695 6696 ret = dst_entries_init(&ip6_dst_blackhole_ops); 6697 if (ret) 6698 goto out_kmem_cache; 6699 6700 ret = register_pernet_subsys(&ipv6_inetpeer_ops); 6701 if (ret) 6702 goto out_dst_entries; 6703 6704 ret = register_pernet_subsys(&ip6_route_net_ops); 6705 if (ret) 6706 goto out_register_inetpeer; 6707 6708 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep; 6709 6710 ret = fib6_init(); 6711 if (ret) 6712 goto out_register_subsys; 6713 6714 ret = xfrm6_init(); 6715 if (ret) 6716 goto out_fib6_init; 6717 6718 ret = fib6_rules_init(); 6719 if (ret) 6720 goto xfrm6_init; 6721 6722 ret = register_pernet_subsys(&ip6_route_net_late_ops); 6723 if (ret) 6724 goto fib6_rules_init; 6725 6726 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE, 6727 inet6_rtm_newroute, NULL, 0); 6728 if (ret < 0) 6729 goto out_register_late_subsys; 6730 6731 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE, 6732 inet6_rtm_delroute, NULL, 0); 6733 if (ret < 0) 6734 goto out_register_late_subsys; 6735 6736 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE, 6737 inet6_rtm_getroute, NULL, 6738 RTNL_FLAG_DOIT_UNLOCKED); 6739 if (ret < 0) 6740 goto out_register_late_subsys; 6741 6742 ret = register_netdevice_notifier(&ip6_route_dev_notifier); 6743 if (ret) 6744 goto out_register_late_subsys; 6745 6746 #if IS_BUILTIN(CONFIG_IPV6) 6747 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6748 ret = bpf_iter_register(); 6749 if (ret) 6750 goto out_register_late_subsys; 6751 #endif 6752 #endif 6753 6754 for_each_possible_cpu(cpu) { 6755 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 6756 6757 INIT_LIST_HEAD(&ul->head); 6758 INIT_LIST_HEAD(&ul->quarantine); 6759 spin_lock_init(&ul->lock); 6760 } 6761 6762 out: 6763 return ret; 6764 6765 out_register_late_subsys: 6766 rtnl_unregister_all(PF_INET6); 6767 unregister_pernet_subsys(&ip6_route_net_late_ops); 6768 fib6_rules_init: 6769 fib6_rules_cleanup(); 6770 xfrm6_init: 6771 xfrm6_fini(); 6772 out_fib6_init: 6773 fib6_gc_cleanup(); 6774 out_register_subsys: 6775 unregister_pernet_subsys(&ip6_route_net_ops); 6776 out_register_inetpeer: 6777 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6778 out_dst_entries: 6779 dst_entries_destroy(&ip6_dst_blackhole_ops); 6780 out_kmem_cache: 6781 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6782 goto out; 6783 } 6784 6785 void ip6_route_cleanup(void) 6786 { 6787 #if IS_BUILTIN(CONFIG_IPV6) 6788 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6789 bpf_iter_unregister(); 6790 #endif 6791 #endif 6792 unregister_netdevice_notifier(&ip6_route_dev_notifier); 6793 unregister_pernet_subsys(&ip6_route_net_late_ops); 6794 fib6_rules_cleanup(); 6795 xfrm6_fini(); 6796 fib6_gc_cleanup(); 6797 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6798 unregister_pernet_subsys(&ip6_route_net_ops); 6799 dst_entries_destroy(&ip6_dst_blackhole_ops); 6800 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6801 } 6802