1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux INET6 implementation 4 * FIB front-end. 5 * 6 * Authors: 7 * Pedro Roque <roque@di.fc.ul.pt> 8 */ 9 10 /* Changes: 11 * 12 * YOSHIFUJI Hideaki @USAGI 13 * reworked default router selection. 14 * - respect outgoing interface 15 * - select from (probably) reachable routers (i.e. 16 * routers in REACHABLE, STALE, DELAY or PROBE states). 17 * - always select the same router if it is (probably) 18 * reachable. otherwise, round-robin the list. 19 * Ville Nuorvala 20 * Fixed routing subtrees. 21 */ 22 23 #define pr_fmt(fmt) "IPv6: " fmt 24 25 #include <linux/capability.h> 26 #include <linux/errno.h> 27 #include <linux/export.h> 28 #include <linux/types.h> 29 #include <linux/times.h> 30 #include <linux/socket.h> 31 #include <linux/sockios.h> 32 #include <linux/net.h> 33 #include <linux/route.h> 34 #include <linux/netdevice.h> 35 #include <linux/in6.h> 36 #include <linux/mroute6.h> 37 #include <linux/init.h> 38 #include <linux/if_arp.h> 39 #include <linux/proc_fs.h> 40 #include <linux/seq_file.h> 41 #include <linux/nsproxy.h> 42 #include <linux/slab.h> 43 #include <linux/jhash.h> 44 #include <linux/siphash.h> 45 #include <net/net_namespace.h> 46 #include <net/snmp.h> 47 #include <net/ipv6.h> 48 #include <net/ip6_fib.h> 49 #include <net/ip6_route.h> 50 #include <net/ndisc.h> 51 #include <net/addrconf.h> 52 #include <net/tcp.h> 53 #include <linux/rtnetlink.h> 54 #include <net/dst.h> 55 #include <net/dst_metadata.h> 56 #include <net/xfrm.h> 57 #include <net/netevent.h> 58 #include <net/netlink.h> 59 #include <net/rtnh.h> 60 #include <net/lwtunnel.h> 61 #include <net/ip_tunnels.h> 62 #include <net/l3mdev.h> 63 #include <net/ip.h> 64 #include <linux/uaccess.h> 65 #include <linux/btf_ids.h> 66 67 #ifdef CONFIG_SYSCTL 68 #include <linux/sysctl.h> 69 #endif 70 71 static int ip6_rt_type_to_error(u8 fib6_type); 72 73 #define CREATE_TRACE_POINTS 74 #include <trace/events/fib6.h> 75 EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup); 76 #undef CREATE_TRACE_POINTS 77 78 enum rt6_nud_state { 79 RT6_NUD_FAIL_HARD = -3, 80 RT6_NUD_FAIL_PROBE = -2, 81 RT6_NUD_FAIL_DO_RR = -1, 82 RT6_NUD_SUCCEED = 1 83 }; 84 85 INDIRECT_CALLABLE_SCOPE 86 struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie); 87 static unsigned int ip6_default_advmss(const struct dst_entry *dst); 88 INDIRECT_CALLABLE_SCOPE 89 unsigned int ip6_mtu(const struct dst_entry *dst); 90 static struct dst_entry *ip6_negative_advice(struct dst_entry *); 91 static void ip6_dst_destroy(struct dst_entry *); 92 static void ip6_dst_ifdown(struct dst_entry *, 93 struct net_device *dev, int how); 94 static void ip6_dst_gc(struct dst_ops *ops); 95 96 static int ip6_pkt_discard(struct sk_buff *skb); 97 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb); 98 static int ip6_pkt_prohibit(struct sk_buff *skb); 99 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb); 100 static void ip6_link_failure(struct sk_buff *skb); 101 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 102 struct sk_buff *skb, u32 mtu, 103 bool confirm_neigh); 104 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, 105 struct sk_buff *skb); 106 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 107 int strict); 108 static size_t rt6_nlmsg_size(struct fib6_info *f6i); 109 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 110 struct fib6_info *rt, struct dst_entry *dst, 111 struct in6_addr *dest, struct in6_addr *src, 112 int iif, int type, u32 portid, u32 seq, 113 unsigned int flags); 114 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 115 const struct in6_addr *daddr, 116 const struct in6_addr *saddr); 117 118 #ifdef CONFIG_IPV6_ROUTE_INFO 119 static struct fib6_info *rt6_add_route_info(struct net *net, 120 const struct in6_addr *prefix, int prefixlen, 121 const struct in6_addr *gwaddr, 122 struct net_device *dev, 123 unsigned int pref); 124 static struct fib6_info *rt6_get_route_info(struct net *net, 125 const struct in6_addr *prefix, int prefixlen, 126 const struct in6_addr *gwaddr, 127 struct net_device *dev); 128 #endif 129 130 struct uncached_list { 131 spinlock_t lock; 132 struct list_head head; 133 struct list_head quarantine; 134 }; 135 136 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); 137 138 void rt6_uncached_list_add(struct rt6_info *rt) 139 { 140 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); 141 142 rt->rt6i_uncached_list = ul; 143 144 spin_lock_bh(&ul->lock); 145 list_add_tail(&rt->rt6i_uncached, &ul->head); 146 spin_unlock_bh(&ul->lock); 147 } 148 149 void rt6_uncached_list_del(struct rt6_info *rt) 150 { 151 if (!list_empty(&rt->rt6i_uncached)) { 152 struct uncached_list *ul = rt->rt6i_uncached_list; 153 154 spin_lock_bh(&ul->lock); 155 list_del_init(&rt->rt6i_uncached); 156 spin_unlock_bh(&ul->lock); 157 } 158 } 159 160 static void rt6_uncached_list_flush_dev(struct net_device *dev) 161 { 162 int cpu; 163 164 for_each_possible_cpu(cpu) { 165 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 166 struct rt6_info *rt, *safe; 167 168 if (list_empty(&ul->head)) 169 continue; 170 171 spin_lock_bh(&ul->lock); 172 list_for_each_entry_safe(rt, safe, &ul->head, rt6i_uncached) { 173 struct inet6_dev *rt_idev = rt->rt6i_idev; 174 struct net_device *rt_dev = rt->dst.dev; 175 bool handled = false; 176 177 if (rt_idev->dev == dev) { 178 rt->rt6i_idev = in6_dev_get(blackhole_netdev); 179 in6_dev_put(rt_idev); 180 handled = true; 181 } 182 183 if (rt_dev == dev) { 184 rt->dst.dev = blackhole_netdev; 185 netdev_ref_replace(rt_dev, blackhole_netdev, 186 &rt->dst.dev_tracker, 187 GFP_ATOMIC); 188 handled = true; 189 } 190 if (handled) 191 list_move(&rt->rt6i_uncached, 192 &ul->quarantine); 193 } 194 spin_unlock_bh(&ul->lock); 195 } 196 } 197 198 static inline const void *choose_neigh_daddr(const struct in6_addr *p, 199 struct sk_buff *skb, 200 const void *daddr) 201 { 202 if (!ipv6_addr_any(p)) 203 return (const void *) p; 204 else if (skb) 205 return &ipv6_hdr(skb)->daddr; 206 return daddr; 207 } 208 209 struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw, 210 struct net_device *dev, 211 struct sk_buff *skb, 212 const void *daddr) 213 { 214 struct neighbour *n; 215 216 daddr = choose_neigh_daddr(gw, skb, daddr); 217 n = __ipv6_neigh_lookup(dev, daddr); 218 if (n) 219 return n; 220 221 n = neigh_create(&nd_tbl, daddr, dev); 222 return IS_ERR(n) ? NULL : n; 223 } 224 225 static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst, 226 struct sk_buff *skb, 227 const void *daddr) 228 { 229 const struct rt6_info *rt = container_of(dst, struct rt6_info, dst); 230 231 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any), 232 dst->dev, skb, daddr); 233 } 234 235 static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr) 236 { 237 struct net_device *dev = dst->dev; 238 struct rt6_info *rt = (struct rt6_info *)dst; 239 240 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr); 241 if (!daddr) 242 return; 243 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) 244 return; 245 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr)) 246 return; 247 __ipv6_confirm_neigh(dev, daddr); 248 } 249 250 static struct dst_ops ip6_dst_ops_template = { 251 .family = AF_INET6, 252 .gc = ip6_dst_gc, 253 .gc_thresh = 1024, 254 .check = ip6_dst_check, 255 .default_advmss = ip6_default_advmss, 256 .mtu = ip6_mtu, 257 .cow_metrics = dst_cow_metrics_generic, 258 .destroy = ip6_dst_destroy, 259 .ifdown = ip6_dst_ifdown, 260 .negative_advice = ip6_negative_advice, 261 .link_failure = ip6_link_failure, 262 .update_pmtu = ip6_rt_update_pmtu, 263 .redirect = rt6_do_redirect, 264 .local_out = __ip6_local_out, 265 .neigh_lookup = ip6_dst_neigh_lookup, 266 .confirm_neigh = ip6_confirm_neigh, 267 }; 268 269 static struct dst_ops ip6_dst_blackhole_ops = { 270 .family = AF_INET6, 271 .default_advmss = ip6_default_advmss, 272 .neigh_lookup = ip6_dst_neigh_lookup, 273 .check = ip6_dst_check, 274 .destroy = ip6_dst_destroy, 275 .cow_metrics = dst_cow_metrics_generic, 276 .update_pmtu = dst_blackhole_update_pmtu, 277 .redirect = dst_blackhole_redirect, 278 .mtu = dst_blackhole_mtu, 279 }; 280 281 static const u32 ip6_template_metrics[RTAX_MAX] = { 282 [RTAX_HOPLIMIT - 1] = 0, 283 }; 284 285 static const struct fib6_info fib6_null_entry_template = { 286 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP), 287 .fib6_protocol = RTPROT_KERNEL, 288 .fib6_metric = ~(u32)0, 289 .fib6_ref = REFCOUNT_INIT(1), 290 .fib6_type = RTN_UNREACHABLE, 291 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics, 292 }; 293 294 static const struct rt6_info ip6_null_entry_template = { 295 .dst = { 296 .__refcnt = ATOMIC_INIT(1), 297 .__use = 1, 298 .obsolete = DST_OBSOLETE_FORCE_CHK, 299 .error = -ENETUNREACH, 300 .input = ip6_pkt_discard, 301 .output = ip6_pkt_discard_out, 302 }, 303 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 304 }; 305 306 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 307 308 static const struct rt6_info ip6_prohibit_entry_template = { 309 .dst = { 310 .__refcnt = ATOMIC_INIT(1), 311 .__use = 1, 312 .obsolete = DST_OBSOLETE_FORCE_CHK, 313 .error = -EACCES, 314 .input = ip6_pkt_prohibit, 315 .output = ip6_pkt_prohibit_out, 316 }, 317 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 318 }; 319 320 static const struct rt6_info ip6_blk_hole_entry_template = { 321 .dst = { 322 .__refcnt = ATOMIC_INIT(1), 323 .__use = 1, 324 .obsolete = DST_OBSOLETE_FORCE_CHK, 325 .error = -EINVAL, 326 .input = dst_discard, 327 .output = dst_discard_out, 328 }, 329 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP), 330 }; 331 332 #endif 333 334 static void rt6_info_init(struct rt6_info *rt) 335 { 336 memset_after(rt, 0, dst); 337 INIT_LIST_HEAD(&rt->rt6i_uncached); 338 } 339 340 /* allocate dst with ip6_dst_ops */ 341 struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev, 342 int flags) 343 { 344 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev, 345 1, DST_OBSOLETE_FORCE_CHK, flags); 346 347 if (rt) { 348 rt6_info_init(rt); 349 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 350 } 351 352 return rt; 353 } 354 EXPORT_SYMBOL(ip6_dst_alloc); 355 356 static void ip6_dst_destroy(struct dst_entry *dst) 357 { 358 struct rt6_info *rt = (struct rt6_info *)dst; 359 struct fib6_info *from; 360 struct inet6_dev *idev; 361 362 ip_dst_metrics_put(dst); 363 rt6_uncached_list_del(rt); 364 365 idev = rt->rt6i_idev; 366 if (idev) { 367 rt->rt6i_idev = NULL; 368 in6_dev_put(idev); 369 } 370 371 from = xchg((__force struct fib6_info **)&rt->from, NULL); 372 fib6_info_release(from); 373 } 374 375 static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev, 376 int how) 377 { 378 struct rt6_info *rt = (struct rt6_info *)dst; 379 struct inet6_dev *idev = rt->rt6i_idev; 380 381 if (idev && idev->dev != blackhole_netdev) { 382 struct inet6_dev *blackhole_idev = in6_dev_get(blackhole_netdev); 383 384 if (blackhole_idev) { 385 rt->rt6i_idev = blackhole_idev; 386 in6_dev_put(idev); 387 } 388 } 389 } 390 391 static bool __rt6_check_expired(const struct rt6_info *rt) 392 { 393 if (rt->rt6i_flags & RTF_EXPIRES) 394 return time_after(jiffies, rt->dst.expires); 395 else 396 return false; 397 } 398 399 static bool rt6_check_expired(const struct rt6_info *rt) 400 { 401 struct fib6_info *from; 402 403 from = rcu_dereference(rt->from); 404 405 if (rt->rt6i_flags & RTF_EXPIRES) { 406 if (time_after(jiffies, rt->dst.expires)) 407 return true; 408 } else if (from) { 409 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK || 410 fib6_check_expired(from); 411 } 412 return false; 413 } 414 415 void fib6_select_path(const struct net *net, struct fib6_result *res, 416 struct flowi6 *fl6, int oif, bool have_oif_match, 417 const struct sk_buff *skb, int strict) 418 { 419 struct fib6_info *sibling, *next_sibling; 420 struct fib6_info *match = res->f6i; 421 422 if (!match->nh && (!match->fib6_nsiblings || have_oif_match)) 423 goto out; 424 425 if (match->nh && have_oif_match && res->nh) 426 return; 427 428 /* We might have already computed the hash for ICMPv6 errors. In such 429 * case it will always be non-zero. Otherwise now is the time to do it. 430 */ 431 if (!fl6->mp_hash && 432 (!match->nh || nexthop_is_multipath(match->nh))) 433 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL); 434 435 if (unlikely(match->nh)) { 436 nexthop_path_fib6_result(res, fl6->mp_hash); 437 return; 438 } 439 440 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound)) 441 goto out; 442 443 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings, 444 fib6_siblings) { 445 const struct fib6_nh *nh = sibling->fib6_nh; 446 int nh_upper_bound; 447 448 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound); 449 if (fl6->mp_hash > nh_upper_bound) 450 continue; 451 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0) 452 break; 453 match = sibling; 454 break; 455 } 456 457 out: 458 res->f6i = match; 459 res->nh = match->fib6_nh; 460 } 461 462 /* 463 * Route lookup. rcu_read_lock() should be held. 464 */ 465 466 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh, 467 const struct in6_addr *saddr, int oif, int flags) 468 { 469 const struct net_device *dev; 470 471 if (nh->fib_nh_flags & RTNH_F_DEAD) 472 return false; 473 474 dev = nh->fib_nh_dev; 475 if (oif) { 476 if (dev->ifindex == oif) 477 return true; 478 } else { 479 if (ipv6_chk_addr(net, saddr, dev, 480 flags & RT6_LOOKUP_F_IFACE)) 481 return true; 482 } 483 484 return false; 485 } 486 487 struct fib6_nh_dm_arg { 488 struct net *net; 489 const struct in6_addr *saddr; 490 int oif; 491 int flags; 492 struct fib6_nh *nh; 493 }; 494 495 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg) 496 { 497 struct fib6_nh_dm_arg *arg = _arg; 498 499 arg->nh = nh; 500 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif, 501 arg->flags); 502 } 503 504 /* returns fib6_nh from nexthop or NULL */ 505 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh, 506 struct fib6_result *res, 507 const struct in6_addr *saddr, 508 int oif, int flags) 509 { 510 struct fib6_nh_dm_arg arg = { 511 .net = net, 512 .saddr = saddr, 513 .oif = oif, 514 .flags = flags, 515 }; 516 517 if (nexthop_is_blackhole(nh)) 518 return NULL; 519 520 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg)) 521 return arg.nh; 522 523 return NULL; 524 } 525 526 static void rt6_device_match(struct net *net, struct fib6_result *res, 527 const struct in6_addr *saddr, int oif, int flags) 528 { 529 struct fib6_info *f6i = res->f6i; 530 struct fib6_info *spf6i; 531 struct fib6_nh *nh; 532 533 if (!oif && ipv6_addr_any(saddr)) { 534 if (unlikely(f6i->nh)) { 535 nh = nexthop_fib6_nh(f6i->nh); 536 if (nexthop_is_blackhole(f6i->nh)) 537 goto out_blackhole; 538 } else { 539 nh = f6i->fib6_nh; 540 } 541 if (!(nh->fib_nh_flags & RTNH_F_DEAD)) 542 goto out; 543 } 544 545 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) { 546 bool matched = false; 547 548 if (unlikely(spf6i->nh)) { 549 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr, 550 oif, flags); 551 if (nh) 552 matched = true; 553 } else { 554 nh = spf6i->fib6_nh; 555 if (__rt6_device_match(net, nh, saddr, oif, flags)) 556 matched = true; 557 } 558 if (matched) { 559 res->f6i = spf6i; 560 goto out; 561 } 562 } 563 564 if (oif && flags & RT6_LOOKUP_F_IFACE) { 565 res->f6i = net->ipv6.fib6_null_entry; 566 nh = res->f6i->fib6_nh; 567 goto out; 568 } 569 570 if (unlikely(f6i->nh)) { 571 nh = nexthop_fib6_nh(f6i->nh); 572 if (nexthop_is_blackhole(f6i->nh)) 573 goto out_blackhole; 574 } else { 575 nh = f6i->fib6_nh; 576 } 577 578 if (nh->fib_nh_flags & RTNH_F_DEAD) { 579 res->f6i = net->ipv6.fib6_null_entry; 580 nh = res->f6i->fib6_nh; 581 } 582 out: 583 res->nh = nh; 584 res->fib6_type = res->f6i->fib6_type; 585 res->fib6_flags = res->f6i->fib6_flags; 586 return; 587 588 out_blackhole: 589 res->fib6_flags |= RTF_REJECT; 590 res->fib6_type = RTN_BLACKHOLE; 591 res->nh = nh; 592 } 593 594 #ifdef CONFIG_IPV6_ROUTER_PREF 595 struct __rt6_probe_work { 596 struct work_struct work; 597 struct in6_addr target; 598 struct net_device *dev; 599 netdevice_tracker dev_tracker; 600 }; 601 602 static void rt6_probe_deferred(struct work_struct *w) 603 { 604 struct in6_addr mcaddr; 605 struct __rt6_probe_work *work = 606 container_of(w, struct __rt6_probe_work, work); 607 608 addrconf_addr_solict_mult(&work->target, &mcaddr); 609 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); 610 netdev_put(work->dev, &work->dev_tracker); 611 kfree(work); 612 } 613 614 static void rt6_probe(struct fib6_nh *fib6_nh) 615 { 616 struct __rt6_probe_work *work = NULL; 617 const struct in6_addr *nh_gw; 618 unsigned long last_probe; 619 struct neighbour *neigh; 620 struct net_device *dev; 621 struct inet6_dev *idev; 622 623 /* 624 * Okay, this does not seem to be appropriate 625 * for now, however, we need to check if it 626 * is really so; aka Router Reachability Probing. 627 * 628 * Router Reachability Probe MUST be rate-limited 629 * to no more than one per minute. 630 */ 631 if (!fib6_nh->fib_nh_gw_family) 632 return; 633 634 nh_gw = &fib6_nh->fib_nh_gw6; 635 dev = fib6_nh->fib_nh_dev; 636 rcu_read_lock_bh(); 637 last_probe = READ_ONCE(fib6_nh->last_probe); 638 idev = __in6_dev_get(dev); 639 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw); 640 if (neigh) { 641 if (neigh->nud_state & NUD_VALID) 642 goto out; 643 644 write_lock(&neigh->lock); 645 if (!(neigh->nud_state & NUD_VALID) && 646 time_after(jiffies, 647 neigh->updated + idev->cnf.rtr_probe_interval)) { 648 work = kmalloc(sizeof(*work), GFP_ATOMIC); 649 if (work) 650 __neigh_set_probe_once(neigh); 651 } 652 write_unlock(&neigh->lock); 653 } else if (time_after(jiffies, last_probe + 654 idev->cnf.rtr_probe_interval)) { 655 work = kmalloc(sizeof(*work), GFP_ATOMIC); 656 } 657 658 if (!work || cmpxchg(&fib6_nh->last_probe, 659 last_probe, jiffies) != last_probe) { 660 kfree(work); 661 } else { 662 INIT_WORK(&work->work, rt6_probe_deferred); 663 work->target = *nh_gw; 664 netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC); 665 work->dev = dev; 666 schedule_work(&work->work); 667 } 668 669 out: 670 rcu_read_unlock_bh(); 671 } 672 #else 673 static inline void rt6_probe(struct fib6_nh *fib6_nh) 674 { 675 } 676 #endif 677 678 /* 679 * Default Router Selection (RFC 2461 6.3.6) 680 */ 681 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh) 682 { 683 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD; 684 struct neighbour *neigh; 685 686 rcu_read_lock_bh(); 687 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev, 688 &fib6_nh->fib_nh_gw6); 689 if (neigh) { 690 read_lock(&neigh->lock); 691 if (neigh->nud_state & NUD_VALID) 692 ret = RT6_NUD_SUCCEED; 693 #ifdef CONFIG_IPV6_ROUTER_PREF 694 else if (!(neigh->nud_state & NUD_FAILED)) 695 ret = RT6_NUD_SUCCEED; 696 else 697 ret = RT6_NUD_FAIL_PROBE; 698 #endif 699 read_unlock(&neigh->lock); 700 } else { 701 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ? 702 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR; 703 } 704 rcu_read_unlock_bh(); 705 706 return ret; 707 } 708 709 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif, 710 int strict) 711 { 712 int m = 0; 713 714 if (!oif || nh->fib_nh_dev->ifindex == oif) 715 m = 2; 716 717 if (!m && (strict & RT6_LOOKUP_F_IFACE)) 718 return RT6_NUD_FAIL_HARD; 719 #ifdef CONFIG_IPV6_ROUTER_PREF 720 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2; 721 #endif 722 if ((strict & RT6_LOOKUP_F_REACHABLE) && 723 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) { 724 int n = rt6_check_neigh(nh); 725 if (n < 0) 726 return n; 727 } 728 return m; 729 } 730 731 static bool find_match(struct fib6_nh *nh, u32 fib6_flags, 732 int oif, int strict, int *mpri, bool *do_rr) 733 { 734 bool match_do_rr = false; 735 bool rc = false; 736 int m; 737 738 if (nh->fib_nh_flags & RTNH_F_DEAD) 739 goto out; 740 741 if (ip6_ignore_linkdown(nh->fib_nh_dev) && 742 nh->fib_nh_flags & RTNH_F_LINKDOWN && 743 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE)) 744 goto out; 745 746 m = rt6_score_route(nh, fib6_flags, oif, strict); 747 if (m == RT6_NUD_FAIL_DO_RR) { 748 match_do_rr = true; 749 m = 0; /* lowest valid score */ 750 } else if (m == RT6_NUD_FAIL_HARD) { 751 goto out; 752 } 753 754 if (strict & RT6_LOOKUP_F_REACHABLE) 755 rt6_probe(nh); 756 757 /* note that m can be RT6_NUD_FAIL_PROBE at this point */ 758 if (m > *mpri) { 759 *do_rr = match_do_rr; 760 *mpri = m; 761 rc = true; 762 } 763 out: 764 return rc; 765 } 766 767 struct fib6_nh_frl_arg { 768 u32 flags; 769 int oif; 770 int strict; 771 int *mpri; 772 bool *do_rr; 773 struct fib6_nh *nh; 774 }; 775 776 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg) 777 { 778 struct fib6_nh_frl_arg *arg = _arg; 779 780 arg->nh = nh; 781 return find_match(nh, arg->flags, arg->oif, arg->strict, 782 arg->mpri, arg->do_rr); 783 } 784 785 static void __find_rr_leaf(struct fib6_info *f6i_start, 786 struct fib6_info *nomatch, u32 metric, 787 struct fib6_result *res, struct fib6_info **cont, 788 int oif, int strict, bool *do_rr, int *mpri) 789 { 790 struct fib6_info *f6i; 791 792 for (f6i = f6i_start; 793 f6i && f6i != nomatch; 794 f6i = rcu_dereference(f6i->fib6_next)) { 795 bool matched = false; 796 struct fib6_nh *nh; 797 798 if (cont && f6i->fib6_metric != metric) { 799 *cont = f6i; 800 return; 801 } 802 803 if (fib6_check_expired(f6i)) 804 continue; 805 806 if (unlikely(f6i->nh)) { 807 struct fib6_nh_frl_arg arg = { 808 .flags = f6i->fib6_flags, 809 .oif = oif, 810 .strict = strict, 811 .mpri = mpri, 812 .do_rr = do_rr 813 }; 814 815 if (nexthop_is_blackhole(f6i->nh)) { 816 res->fib6_flags = RTF_REJECT; 817 res->fib6_type = RTN_BLACKHOLE; 818 res->f6i = f6i; 819 res->nh = nexthop_fib6_nh(f6i->nh); 820 return; 821 } 822 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match, 823 &arg)) { 824 matched = true; 825 nh = arg.nh; 826 } 827 } else { 828 nh = f6i->fib6_nh; 829 if (find_match(nh, f6i->fib6_flags, oif, strict, 830 mpri, do_rr)) 831 matched = true; 832 } 833 if (matched) { 834 res->f6i = f6i; 835 res->nh = nh; 836 res->fib6_flags = f6i->fib6_flags; 837 res->fib6_type = f6i->fib6_type; 838 } 839 } 840 } 841 842 static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf, 843 struct fib6_info *rr_head, int oif, int strict, 844 bool *do_rr, struct fib6_result *res) 845 { 846 u32 metric = rr_head->fib6_metric; 847 struct fib6_info *cont = NULL; 848 int mpri = -1; 849 850 __find_rr_leaf(rr_head, NULL, metric, res, &cont, 851 oif, strict, do_rr, &mpri); 852 853 __find_rr_leaf(leaf, rr_head, metric, res, &cont, 854 oif, strict, do_rr, &mpri); 855 856 if (res->f6i || !cont) 857 return; 858 859 __find_rr_leaf(cont, NULL, metric, res, NULL, 860 oif, strict, do_rr, &mpri); 861 } 862 863 static void rt6_select(struct net *net, struct fib6_node *fn, int oif, 864 struct fib6_result *res, int strict) 865 { 866 struct fib6_info *leaf = rcu_dereference(fn->leaf); 867 struct fib6_info *rt0; 868 bool do_rr = false; 869 int key_plen; 870 871 /* make sure this function or its helpers sets f6i */ 872 res->f6i = NULL; 873 874 if (!leaf || leaf == net->ipv6.fib6_null_entry) 875 goto out; 876 877 rt0 = rcu_dereference(fn->rr_ptr); 878 if (!rt0) 879 rt0 = leaf; 880 881 /* Double check to make sure fn is not an intermediate node 882 * and fn->leaf does not points to its child's leaf 883 * (This might happen if all routes under fn are deleted from 884 * the tree and fib6_repair_tree() is called on the node.) 885 */ 886 key_plen = rt0->fib6_dst.plen; 887 #ifdef CONFIG_IPV6_SUBTREES 888 if (rt0->fib6_src.plen) 889 key_plen = rt0->fib6_src.plen; 890 #endif 891 if (fn->fn_bit != key_plen) 892 goto out; 893 894 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res); 895 if (do_rr) { 896 struct fib6_info *next = rcu_dereference(rt0->fib6_next); 897 898 /* no entries matched; do round-robin */ 899 if (!next || next->fib6_metric != rt0->fib6_metric) 900 next = leaf; 901 902 if (next != rt0) { 903 spin_lock_bh(&leaf->fib6_table->tb6_lock); 904 /* make sure next is not being deleted from the tree */ 905 if (next->fib6_node) 906 rcu_assign_pointer(fn->rr_ptr, next); 907 spin_unlock_bh(&leaf->fib6_table->tb6_lock); 908 } 909 } 910 911 out: 912 if (!res->f6i) { 913 res->f6i = net->ipv6.fib6_null_entry; 914 res->nh = res->f6i->fib6_nh; 915 res->fib6_flags = res->f6i->fib6_flags; 916 res->fib6_type = res->f6i->fib6_type; 917 } 918 } 919 920 static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res) 921 { 922 return (res->f6i->fib6_flags & RTF_NONEXTHOP) || 923 res->nh->fib_nh_gw_family; 924 } 925 926 #ifdef CONFIG_IPV6_ROUTE_INFO 927 int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, 928 const struct in6_addr *gwaddr) 929 { 930 struct net *net = dev_net(dev); 931 struct route_info *rinfo = (struct route_info *) opt; 932 struct in6_addr prefix_buf, *prefix; 933 unsigned int pref; 934 unsigned long lifetime; 935 struct fib6_info *rt; 936 937 if (len < sizeof(struct route_info)) { 938 return -EINVAL; 939 } 940 941 /* Sanity check for prefix_len and length */ 942 if (rinfo->length > 3) { 943 return -EINVAL; 944 } else if (rinfo->prefix_len > 128) { 945 return -EINVAL; 946 } else if (rinfo->prefix_len > 64) { 947 if (rinfo->length < 2) { 948 return -EINVAL; 949 } 950 } else if (rinfo->prefix_len > 0) { 951 if (rinfo->length < 1) { 952 return -EINVAL; 953 } 954 } 955 956 pref = rinfo->route_pref; 957 if (pref == ICMPV6_ROUTER_PREF_INVALID) 958 return -EINVAL; 959 960 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ); 961 962 if (rinfo->length == 3) 963 prefix = (struct in6_addr *)rinfo->prefix; 964 else { 965 /* this function is safe */ 966 ipv6_addr_prefix(&prefix_buf, 967 (struct in6_addr *)rinfo->prefix, 968 rinfo->prefix_len); 969 prefix = &prefix_buf; 970 } 971 972 if (rinfo->prefix_len == 0) 973 rt = rt6_get_dflt_router(net, gwaddr, dev); 974 else 975 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len, 976 gwaddr, dev); 977 978 if (rt && !lifetime) { 979 ip6_del_rt(net, rt, false); 980 rt = NULL; 981 } 982 983 if (!rt && lifetime) 984 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, 985 dev, pref); 986 else if (rt) 987 rt->fib6_flags = RTF_ROUTEINFO | 988 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref); 989 990 if (rt) { 991 if (!addrconf_finite_timeout(lifetime)) 992 fib6_clean_expires(rt); 993 else 994 fib6_set_expires(rt, jiffies + HZ * lifetime); 995 996 fib6_info_release(rt); 997 } 998 return 0; 999 } 1000 #endif 1001 1002 /* 1003 * Misc support functions 1004 */ 1005 1006 /* called with rcu_lock held */ 1007 static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) 1008 { 1009 struct net_device *dev = res->nh->fib_nh_dev; 1010 1011 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) { 1012 /* for copies of local routes, dst->dev needs to be the 1013 * device if it is a master device, the master device if 1014 * device is enslaved, and the loopback as the default 1015 */ 1016 if (netif_is_l3_slave(dev) && 1017 !rt6_need_strict(&res->f6i->fib6_dst.addr)) 1018 dev = l3mdev_master_dev_rcu(dev); 1019 else if (!netif_is_l3_master(dev)) 1020 dev = dev_net(dev)->loopback_dev; 1021 /* last case is netif_is_l3_master(dev) is true in which 1022 * case we want dev returned to be dev 1023 */ 1024 } 1025 1026 return dev; 1027 } 1028 1029 static const int fib6_prop[RTN_MAX + 1] = { 1030 [RTN_UNSPEC] = 0, 1031 [RTN_UNICAST] = 0, 1032 [RTN_LOCAL] = 0, 1033 [RTN_BROADCAST] = 0, 1034 [RTN_ANYCAST] = 0, 1035 [RTN_MULTICAST] = 0, 1036 [RTN_BLACKHOLE] = -EINVAL, 1037 [RTN_UNREACHABLE] = -EHOSTUNREACH, 1038 [RTN_PROHIBIT] = -EACCES, 1039 [RTN_THROW] = -EAGAIN, 1040 [RTN_NAT] = -EINVAL, 1041 [RTN_XRESOLVE] = -EINVAL, 1042 }; 1043 1044 static int ip6_rt_type_to_error(u8 fib6_type) 1045 { 1046 return fib6_prop[fib6_type]; 1047 } 1048 1049 static unsigned short fib6_info_dst_flags(struct fib6_info *rt) 1050 { 1051 unsigned short flags = 0; 1052 1053 if (rt->dst_nocount) 1054 flags |= DST_NOCOUNT; 1055 if (rt->dst_nopolicy) 1056 flags |= DST_NOPOLICY; 1057 1058 return flags; 1059 } 1060 1061 static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type) 1062 { 1063 rt->dst.error = ip6_rt_type_to_error(fib6_type); 1064 1065 switch (fib6_type) { 1066 case RTN_BLACKHOLE: 1067 rt->dst.output = dst_discard_out; 1068 rt->dst.input = dst_discard; 1069 break; 1070 case RTN_PROHIBIT: 1071 rt->dst.output = ip6_pkt_prohibit_out; 1072 rt->dst.input = ip6_pkt_prohibit; 1073 break; 1074 case RTN_THROW: 1075 case RTN_UNREACHABLE: 1076 default: 1077 rt->dst.output = ip6_pkt_discard_out; 1078 rt->dst.input = ip6_pkt_discard; 1079 break; 1080 } 1081 } 1082 1083 static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res) 1084 { 1085 struct fib6_info *f6i = res->f6i; 1086 1087 if (res->fib6_flags & RTF_REJECT) { 1088 ip6_rt_init_dst_reject(rt, res->fib6_type); 1089 return; 1090 } 1091 1092 rt->dst.error = 0; 1093 rt->dst.output = ip6_output; 1094 1095 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) { 1096 rt->dst.input = ip6_input; 1097 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) { 1098 rt->dst.input = ip6_mc_input; 1099 } else { 1100 rt->dst.input = ip6_forward; 1101 } 1102 1103 if (res->nh->fib_nh_lws) { 1104 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws); 1105 lwtunnel_set_redirect(&rt->dst); 1106 } 1107 1108 rt->dst.lastuse = jiffies; 1109 } 1110 1111 /* Caller must already hold reference to @from */ 1112 static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from) 1113 { 1114 rt->rt6i_flags &= ~RTF_EXPIRES; 1115 rcu_assign_pointer(rt->from, from); 1116 ip_dst_init_metrics(&rt->dst, from->fib6_metrics); 1117 } 1118 1119 /* Caller must already hold reference to f6i in result */ 1120 static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res) 1121 { 1122 const struct fib6_nh *nh = res->nh; 1123 const struct net_device *dev = nh->fib_nh_dev; 1124 struct fib6_info *f6i = res->f6i; 1125 1126 ip6_rt_init_dst(rt, res); 1127 1128 rt->rt6i_dst = f6i->fib6_dst; 1129 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL; 1130 rt->rt6i_flags = res->fib6_flags; 1131 if (nh->fib_nh_gw_family) { 1132 rt->rt6i_gateway = nh->fib_nh_gw6; 1133 rt->rt6i_flags |= RTF_GATEWAY; 1134 } 1135 rt6_set_from(rt, f6i); 1136 #ifdef CONFIG_IPV6_SUBTREES 1137 rt->rt6i_src = f6i->fib6_src; 1138 #endif 1139 } 1140 1141 static struct fib6_node* fib6_backtrack(struct fib6_node *fn, 1142 struct in6_addr *saddr) 1143 { 1144 struct fib6_node *pn, *sn; 1145 while (1) { 1146 if (fn->fn_flags & RTN_TL_ROOT) 1147 return NULL; 1148 pn = rcu_dereference(fn->parent); 1149 sn = FIB6_SUBTREE(pn); 1150 if (sn && sn != fn) 1151 fn = fib6_node_lookup(sn, NULL, saddr); 1152 else 1153 fn = pn; 1154 if (fn->fn_flags & RTN_RTINFO) 1155 return fn; 1156 } 1157 } 1158 1159 static bool ip6_hold_safe(struct net *net, struct rt6_info **prt) 1160 { 1161 struct rt6_info *rt = *prt; 1162 1163 if (dst_hold_safe(&rt->dst)) 1164 return true; 1165 if (net) { 1166 rt = net->ipv6.ip6_null_entry; 1167 dst_hold(&rt->dst); 1168 } else { 1169 rt = NULL; 1170 } 1171 *prt = rt; 1172 return false; 1173 } 1174 1175 /* called with rcu_lock held */ 1176 static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res) 1177 { 1178 struct net_device *dev = res->nh->fib_nh_dev; 1179 struct fib6_info *f6i = res->f6i; 1180 unsigned short flags; 1181 struct rt6_info *nrt; 1182 1183 if (!fib6_info_hold_safe(f6i)) 1184 goto fallback; 1185 1186 flags = fib6_info_dst_flags(f6i); 1187 nrt = ip6_dst_alloc(dev_net(dev), dev, flags); 1188 if (!nrt) { 1189 fib6_info_release(f6i); 1190 goto fallback; 1191 } 1192 1193 ip6_rt_copy_init(nrt, res); 1194 return nrt; 1195 1196 fallback: 1197 nrt = dev_net(dev)->ipv6.ip6_null_entry; 1198 dst_hold(&nrt->dst); 1199 return nrt; 1200 } 1201 1202 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, 1203 struct fib6_table *table, 1204 struct flowi6 *fl6, 1205 const struct sk_buff *skb, 1206 int flags) 1207 { 1208 struct fib6_result res = {}; 1209 struct fib6_node *fn; 1210 struct rt6_info *rt; 1211 1212 rcu_read_lock(); 1213 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 1214 restart: 1215 res.f6i = rcu_dereference(fn->leaf); 1216 if (!res.f6i) 1217 res.f6i = net->ipv6.fib6_null_entry; 1218 else 1219 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif, 1220 flags); 1221 1222 if (res.f6i == net->ipv6.fib6_null_entry) { 1223 fn = fib6_backtrack(fn, &fl6->saddr); 1224 if (fn) 1225 goto restart; 1226 1227 rt = net->ipv6.ip6_null_entry; 1228 dst_hold(&rt->dst); 1229 goto out; 1230 } else if (res.fib6_flags & RTF_REJECT) { 1231 goto do_create; 1232 } 1233 1234 fib6_select_path(net, &res, fl6, fl6->flowi6_oif, 1235 fl6->flowi6_oif != 0, skb, flags); 1236 1237 /* Search through exception table */ 1238 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 1239 if (rt) { 1240 if (ip6_hold_safe(net, &rt)) 1241 dst_use_noref(&rt->dst, jiffies); 1242 } else { 1243 do_create: 1244 rt = ip6_create_rt_rcu(&res); 1245 } 1246 1247 out: 1248 trace_fib6_table_lookup(net, &res, table, fl6); 1249 1250 rcu_read_unlock(); 1251 1252 return rt; 1253 } 1254 1255 struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6, 1256 const struct sk_buff *skb, int flags) 1257 { 1258 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup); 1259 } 1260 EXPORT_SYMBOL_GPL(ip6_route_lookup); 1261 1262 struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr, 1263 const struct in6_addr *saddr, int oif, 1264 const struct sk_buff *skb, int strict) 1265 { 1266 struct flowi6 fl6 = { 1267 .flowi6_oif = oif, 1268 .daddr = *daddr, 1269 }; 1270 struct dst_entry *dst; 1271 int flags = strict ? RT6_LOOKUP_F_IFACE : 0; 1272 1273 if (saddr) { 1274 memcpy(&fl6.saddr, saddr, sizeof(*saddr)); 1275 flags |= RT6_LOOKUP_F_HAS_SADDR; 1276 } 1277 1278 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup); 1279 if (dst->error == 0) 1280 return (struct rt6_info *) dst; 1281 1282 dst_release(dst); 1283 1284 return NULL; 1285 } 1286 EXPORT_SYMBOL(rt6_lookup); 1287 1288 /* ip6_ins_rt is called with FREE table->tb6_lock. 1289 * It takes new route entry, the addition fails by any reason the 1290 * route is released. 1291 * Caller must hold dst before calling it. 1292 */ 1293 1294 static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info, 1295 struct netlink_ext_ack *extack) 1296 { 1297 int err; 1298 struct fib6_table *table; 1299 1300 table = rt->fib6_table; 1301 spin_lock_bh(&table->tb6_lock); 1302 err = fib6_add(&table->tb6_root, rt, info, extack); 1303 spin_unlock_bh(&table->tb6_lock); 1304 1305 return err; 1306 } 1307 1308 int ip6_ins_rt(struct net *net, struct fib6_info *rt) 1309 { 1310 struct nl_info info = { .nl_net = net, }; 1311 1312 return __ip6_ins_rt(rt, &info, NULL); 1313 } 1314 1315 static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res, 1316 const struct in6_addr *daddr, 1317 const struct in6_addr *saddr) 1318 { 1319 struct fib6_info *f6i = res->f6i; 1320 struct net_device *dev; 1321 struct rt6_info *rt; 1322 1323 /* 1324 * Clone the route. 1325 */ 1326 1327 if (!fib6_info_hold_safe(f6i)) 1328 return NULL; 1329 1330 dev = ip6_rt_get_dev_rcu(res); 1331 rt = ip6_dst_alloc(dev_net(dev), dev, 0); 1332 if (!rt) { 1333 fib6_info_release(f6i); 1334 return NULL; 1335 } 1336 1337 ip6_rt_copy_init(rt, res); 1338 rt->rt6i_flags |= RTF_CACHE; 1339 rt->rt6i_dst.addr = *daddr; 1340 rt->rt6i_dst.plen = 128; 1341 1342 if (!rt6_is_gw_or_nonexthop(res)) { 1343 if (f6i->fib6_dst.plen != 128 && 1344 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr)) 1345 rt->rt6i_flags |= RTF_ANYCAST; 1346 #ifdef CONFIG_IPV6_SUBTREES 1347 if (rt->rt6i_src.plen && saddr) { 1348 rt->rt6i_src.addr = *saddr; 1349 rt->rt6i_src.plen = 128; 1350 } 1351 #endif 1352 } 1353 1354 return rt; 1355 } 1356 1357 static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res) 1358 { 1359 struct fib6_info *f6i = res->f6i; 1360 unsigned short flags = fib6_info_dst_flags(f6i); 1361 struct net_device *dev; 1362 struct rt6_info *pcpu_rt; 1363 1364 if (!fib6_info_hold_safe(f6i)) 1365 return NULL; 1366 1367 rcu_read_lock(); 1368 dev = ip6_rt_get_dev_rcu(res); 1369 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT); 1370 rcu_read_unlock(); 1371 if (!pcpu_rt) { 1372 fib6_info_release(f6i); 1373 return NULL; 1374 } 1375 ip6_rt_copy_init(pcpu_rt, res); 1376 pcpu_rt->rt6i_flags |= RTF_PCPU; 1377 1378 if (f6i->nh) 1379 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev)); 1380 1381 return pcpu_rt; 1382 } 1383 1384 static bool rt6_is_valid(const struct rt6_info *rt6) 1385 { 1386 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev)); 1387 } 1388 1389 /* It should be called with rcu_read_lock() acquired */ 1390 static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res) 1391 { 1392 struct rt6_info *pcpu_rt; 1393 1394 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu); 1395 1396 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) { 1397 struct rt6_info *prev, **p; 1398 1399 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1400 prev = xchg(p, NULL); 1401 if (prev) { 1402 dst_dev_put(&prev->dst); 1403 dst_release(&prev->dst); 1404 } 1405 1406 pcpu_rt = NULL; 1407 } 1408 1409 return pcpu_rt; 1410 } 1411 1412 static struct rt6_info *rt6_make_pcpu_route(struct net *net, 1413 const struct fib6_result *res) 1414 { 1415 struct rt6_info *pcpu_rt, *prev, **p; 1416 1417 pcpu_rt = ip6_rt_pcpu_alloc(res); 1418 if (!pcpu_rt) 1419 return NULL; 1420 1421 p = this_cpu_ptr(res->nh->rt6i_pcpu); 1422 prev = cmpxchg(p, NULL, pcpu_rt); 1423 BUG_ON(prev); 1424 1425 if (res->f6i->fib6_destroying) { 1426 struct fib6_info *from; 1427 1428 from = xchg((__force struct fib6_info **)&pcpu_rt->from, NULL); 1429 fib6_info_release(from); 1430 } 1431 1432 return pcpu_rt; 1433 } 1434 1435 /* exception hash table implementation 1436 */ 1437 static DEFINE_SPINLOCK(rt6_exception_lock); 1438 1439 /* Remove rt6_ex from hash table and free the memory 1440 * Caller must hold rt6_exception_lock 1441 */ 1442 static void rt6_remove_exception(struct rt6_exception_bucket *bucket, 1443 struct rt6_exception *rt6_ex) 1444 { 1445 struct fib6_info *from; 1446 struct net *net; 1447 1448 if (!bucket || !rt6_ex) 1449 return; 1450 1451 net = dev_net(rt6_ex->rt6i->dst.dev); 1452 net->ipv6.rt6_stats->fib_rt_cache--; 1453 1454 /* purge completely the exception to allow releasing the held resources: 1455 * some [sk] cache may keep the dst around for unlimited time 1456 */ 1457 from = xchg((__force struct fib6_info **)&rt6_ex->rt6i->from, NULL); 1458 fib6_info_release(from); 1459 dst_dev_put(&rt6_ex->rt6i->dst); 1460 1461 hlist_del_rcu(&rt6_ex->hlist); 1462 dst_release(&rt6_ex->rt6i->dst); 1463 kfree_rcu(rt6_ex, rcu); 1464 WARN_ON_ONCE(!bucket->depth); 1465 bucket->depth--; 1466 } 1467 1468 /* Remove oldest rt6_ex in bucket and free the memory 1469 * Caller must hold rt6_exception_lock 1470 */ 1471 static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) 1472 { 1473 struct rt6_exception *rt6_ex, *oldest = NULL; 1474 1475 if (!bucket) 1476 return; 1477 1478 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 1479 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp)) 1480 oldest = rt6_ex; 1481 } 1482 rt6_remove_exception(bucket, oldest); 1483 } 1484 1485 static u32 rt6_exception_hash(const struct in6_addr *dst, 1486 const struct in6_addr *src) 1487 { 1488 static siphash_aligned_key_t rt6_exception_key; 1489 struct { 1490 struct in6_addr dst; 1491 struct in6_addr src; 1492 } __aligned(SIPHASH_ALIGNMENT) combined = { 1493 .dst = *dst, 1494 }; 1495 u64 val; 1496 1497 net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key)); 1498 1499 #ifdef CONFIG_IPV6_SUBTREES 1500 if (src) 1501 combined.src = *src; 1502 #endif 1503 val = siphash(&combined, sizeof(combined), &rt6_exception_key); 1504 1505 return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT); 1506 } 1507 1508 /* Helper function to find the cached rt in the hash table 1509 * and update bucket pointer to point to the bucket for this 1510 * (daddr, saddr) pair 1511 * Caller must hold rt6_exception_lock 1512 */ 1513 static struct rt6_exception * 1514 __rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket, 1515 const struct in6_addr *daddr, 1516 const struct in6_addr *saddr) 1517 { 1518 struct rt6_exception *rt6_ex; 1519 u32 hval; 1520 1521 if (!(*bucket) || !daddr) 1522 return NULL; 1523 1524 hval = rt6_exception_hash(daddr, saddr); 1525 *bucket += hval; 1526 1527 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) { 1528 struct rt6_info *rt6 = rt6_ex->rt6i; 1529 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1530 1531 #ifdef CONFIG_IPV6_SUBTREES 1532 if (matched && saddr) 1533 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1534 #endif 1535 if (matched) 1536 return rt6_ex; 1537 } 1538 return NULL; 1539 } 1540 1541 /* Helper function to find the cached rt in the hash table 1542 * and update bucket pointer to point to the bucket for this 1543 * (daddr, saddr) pair 1544 * Caller must hold rcu_read_lock() 1545 */ 1546 static struct rt6_exception * 1547 __rt6_find_exception_rcu(struct rt6_exception_bucket **bucket, 1548 const struct in6_addr *daddr, 1549 const struct in6_addr *saddr) 1550 { 1551 struct rt6_exception *rt6_ex; 1552 u32 hval; 1553 1554 WARN_ON_ONCE(!rcu_read_lock_held()); 1555 1556 if (!(*bucket) || !daddr) 1557 return NULL; 1558 1559 hval = rt6_exception_hash(daddr, saddr); 1560 *bucket += hval; 1561 1562 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) { 1563 struct rt6_info *rt6 = rt6_ex->rt6i; 1564 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr); 1565 1566 #ifdef CONFIG_IPV6_SUBTREES 1567 if (matched && saddr) 1568 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr); 1569 #endif 1570 if (matched) 1571 return rt6_ex; 1572 } 1573 return NULL; 1574 } 1575 1576 static unsigned int fib6_mtu(const struct fib6_result *res) 1577 { 1578 const struct fib6_nh *nh = res->nh; 1579 unsigned int mtu; 1580 1581 if (res->f6i->fib6_pmtu) { 1582 mtu = res->f6i->fib6_pmtu; 1583 } else { 1584 struct net_device *dev = nh->fib_nh_dev; 1585 struct inet6_dev *idev; 1586 1587 rcu_read_lock(); 1588 idev = __in6_dev_get(dev); 1589 mtu = idev->cnf.mtu6; 1590 rcu_read_unlock(); 1591 } 1592 1593 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 1594 1595 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 1596 } 1597 1598 #define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL 1599 1600 /* used when the flushed bit is not relevant, only access to the bucket 1601 * (ie., all bucket users except rt6_insert_exception); 1602 * 1603 * called under rcu lock; sometimes called with rt6_exception_lock held 1604 */ 1605 static 1606 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh, 1607 spinlock_t *lock) 1608 { 1609 struct rt6_exception_bucket *bucket; 1610 1611 if (lock) 1612 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1613 lockdep_is_held(lock)); 1614 else 1615 bucket = rcu_dereference(nh->rt6i_exception_bucket); 1616 1617 /* remove bucket flushed bit if set */ 1618 if (bucket) { 1619 unsigned long p = (unsigned long)bucket; 1620 1621 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED; 1622 bucket = (struct rt6_exception_bucket *)p; 1623 } 1624 1625 return bucket; 1626 } 1627 1628 static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket) 1629 { 1630 unsigned long p = (unsigned long)bucket; 1631 1632 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED); 1633 } 1634 1635 /* called with rt6_exception_lock held */ 1636 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh, 1637 spinlock_t *lock) 1638 { 1639 struct rt6_exception_bucket *bucket; 1640 unsigned long p; 1641 1642 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1643 lockdep_is_held(lock)); 1644 1645 p = (unsigned long)bucket; 1646 p |= FIB6_EXCEPTION_BUCKET_FLUSHED; 1647 bucket = (struct rt6_exception_bucket *)p; 1648 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1649 } 1650 1651 static int rt6_insert_exception(struct rt6_info *nrt, 1652 const struct fib6_result *res) 1653 { 1654 struct net *net = dev_net(nrt->dst.dev); 1655 struct rt6_exception_bucket *bucket; 1656 struct fib6_info *f6i = res->f6i; 1657 struct in6_addr *src_key = NULL; 1658 struct rt6_exception *rt6_ex; 1659 struct fib6_nh *nh = res->nh; 1660 int max_depth; 1661 int err = 0; 1662 1663 spin_lock_bh(&rt6_exception_lock); 1664 1665 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, 1666 lockdep_is_held(&rt6_exception_lock)); 1667 if (!bucket) { 1668 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket), 1669 GFP_ATOMIC); 1670 if (!bucket) { 1671 err = -ENOMEM; 1672 goto out; 1673 } 1674 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket); 1675 } else if (fib6_nh_excptn_bucket_flushed(bucket)) { 1676 err = -EINVAL; 1677 goto out; 1678 } 1679 1680 #ifdef CONFIG_IPV6_SUBTREES 1681 /* fib6_src.plen != 0 indicates f6i is in subtree 1682 * and exception table is indexed by a hash of 1683 * both fib6_dst and fib6_src. 1684 * Otherwise, the exception table is indexed by 1685 * a hash of only fib6_dst. 1686 */ 1687 if (f6i->fib6_src.plen) 1688 src_key = &nrt->rt6i_src.addr; 1689 #endif 1690 /* rt6_mtu_change() might lower mtu on f6i. 1691 * Only insert this exception route if its mtu 1692 * is less than f6i's mtu value. 1693 */ 1694 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) { 1695 err = -EINVAL; 1696 goto out; 1697 } 1698 1699 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr, 1700 src_key); 1701 if (rt6_ex) 1702 rt6_remove_exception(bucket, rt6_ex); 1703 1704 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC); 1705 if (!rt6_ex) { 1706 err = -ENOMEM; 1707 goto out; 1708 } 1709 rt6_ex->rt6i = nrt; 1710 rt6_ex->stamp = jiffies; 1711 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain); 1712 bucket->depth++; 1713 net->ipv6.rt6_stats->fib_rt_cache++; 1714 1715 /* Randomize max depth to avoid some side channels attacks. */ 1716 max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH); 1717 while (bucket->depth > max_depth) 1718 rt6_exception_remove_oldest(bucket); 1719 1720 out: 1721 spin_unlock_bh(&rt6_exception_lock); 1722 1723 /* Update fn->fn_sernum to invalidate all cached dst */ 1724 if (!err) { 1725 spin_lock_bh(&f6i->fib6_table->tb6_lock); 1726 fib6_update_sernum(net, f6i); 1727 spin_unlock_bh(&f6i->fib6_table->tb6_lock); 1728 fib6_force_start_gc(net); 1729 } 1730 1731 return err; 1732 } 1733 1734 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from) 1735 { 1736 struct rt6_exception_bucket *bucket; 1737 struct rt6_exception *rt6_ex; 1738 struct hlist_node *tmp; 1739 int i; 1740 1741 spin_lock_bh(&rt6_exception_lock); 1742 1743 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1744 if (!bucket) 1745 goto out; 1746 1747 /* Prevent rt6_insert_exception() to recreate the bucket list */ 1748 if (!from) 1749 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock); 1750 1751 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 1752 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) { 1753 if (!from || 1754 rcu_access_pointer(rt6_ex->rt6i->from) == from) 1755 rt6_remove_exception(bucket, rt6_ex); 1756 } 1757 WARN_ON_ONCE(!from && bucket->depth); 1758 bucket++; 1759 } 1760 out: 1761 spin_unlock_bh(&rt6_exception_lock); 1762 } 1763 1764 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg) 1765 { 1766 struct fib6_info *f6i = arg; 1767 1768 fib6_nh_flush_exceptions(nh, f6i); 1769 1770 return 0; 1771 } 1772 1773 void rt6_flush_exceptions(struct fib6_info *f6i) 1774 { 1775 if (f6i->nh) 1776 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions, 1777 f6i); 1778 else 1779 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i); 1780 } 1781 1782 /* Find cached rt in the hash table inside passed in rt 1783 * Caller has to hold rcu_read_lock() 1784 */ 1785 static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res, 1786 const struct in6_addr *daddr, 1787 const struct in6_addr *saddr) 1788 { 1789 const struct in6_addr *src_key = NULL; 1790 struct rt6_exception_bucket *bucket; 1791 struct rt6_exception *rt6_ex; 1792 struct rt6_info *ret = NULL; 1793 1794 #ifdef CONFIG_IPV6_SUBTREES 1795 /* fib6i_src.plen != 0 indicates f6i is in subtree 1796 * and exception table is indexed by a hash of 1797 * both fib6_dst and fib6_src. 1798 * However, the src addr used to create the hash 1799 * might not be exactly the passed in saddr which 1800 * is a /128 addr from the flow. 1801 * So we need to use f6i->fib6_src to redo lookup 1802 * if the passed in saddr does not find anything. 1803 * (See the logic in ip6_rt_cache_alloc() on how 1804 * rt->rt6i_src is updated.) 1805 */ 1806 if (res->f6i->fib6_src.plen) 1807 src_key = saddr; 1808 find_ex: 1809 #endif 1810 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL); 1811 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key); 1812 1813 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i)) 1814 ret = rt6_ex->rt6i; 1815 1816 #ifdef CONFIG_IPV6_SUBTREES 1817 /* Use fib6_src as src_key and redo lookup */ 1818 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) { 1819 src_key = &res->f6i->fib6_src.addr; 1820 goto find_ex; 1821 } 1822 #endif 1823 1824 return ret; 1825 } 1826 1827 /* Remove the passed in cached rt from the hash table that contains it */ 1828 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen, 1829 const struct rt6_info *rt) 1830 { 1831 const struct in6_addr *src_key = NULL; 1832 struct rt6_exception_bucket *bucket; 1833 struct rt6_exception *rt6_ex; 1834 int err; 1835 1836 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 1837 return -ENOENT; 1838 1839 spin_lock_bh(&rt6_exception_lock); 1840 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 1841 1842 #ifdef CONFIG_IPV6_SUBTREES 1843 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1844 * and exception table is indexed by a hash of 1845 * both rt6i_dst and rt6i_src. 1846 * Otherwise, the exception table is indexed by 1847 * a hash of only rt6i_dst. 1848 */ 1849 if (plen) 1850 src_key = &rt->rt6i_src.addr; 1851 #endif 1852 rt6_ex = __rt6_find_exception_spinlock(&bucket, 1853 &rt->rt6i_dst.addr, 1854 src_key); 1855 if (rt6_ex) { 1856 rt6_remove_exception(bucket, rt6_ex); 1857 err = 0; 1858 } else { 1859 err = -ENOENT; 1860 } 1861 1862 spin_unlock_bh(&rt6_exception_lock); 1863 return err; 1864 } 1865 1866 struct fib6_nh_excptn_arg { 1867 struct rt6_info *rt; 1868 int plen; 1869 }; 1870 1871 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg) 1872 { 1873 struct fib6_nh_excptn_arg *arg = _arg; 1874 int err; 1875 1876 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt); 1877 if (err == 0) 1878 return 1; 1879 1880 return 0; 1881 } 1882 1883 static int rt6_remove_exception_rt(struct rt6_info *rt) 1884 { 1885 struct fib6_info *from; 1886 1887 from = rcu_dereference(rt->from); 1888 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1889 return -EINVAL; 1890 1891 if (from->nh) { 1892 struct fib6_nh_excptn_arg arg = { 1893 .rt = rt, 1894 .plen = from->fib6_src.plen 1895 }; 1896 int rc; 1897 1898 /* rc = 1 means an entry was found */ 1899 rc = nexthop_for_each_fib6_nh(from->nh, 1900 rt6_nh_remove_exception_rt, 1901 &arg); 1902 return rc ? 0 : -ENOENT; 1903 } 1904 1905 return fib6_nh_remove_exception(from->fib6_nh, 1906 from->fib6_src.plen, rt); 1907 } 1908 1909 /* Find rt6_ex which contains the passed in rt cache and 1910 * refresh its stamp 1911 */ 1912 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen, 1913 const struct rt6_info *rt) 1914 { 1915 const struct in6_addr *src_key = NULL; 1916 struct rt6_exception_bucket *bucket; 1917 struct rt6_exception *rt6_ex; 1918 1919 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 1920 #ifdef CONFIG_IPV6_SUBTREES 1921 /* rt6i_src.plen != 0 indicates 'from' is in subtree 1922 * and exception table is indexed by a hash of 1923 * both rt6i_dst and rt6i_src. 1924 * Otherwise, the exception table is indexed by 1925 * a hash of only rt6i_dst. 1926 */ 1927 if (plen) 1928 src_key = &rt->rt6i_src.addr; 1929 #endif 1930 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key); 1931 if (rt6_ex) 1932 rt6_ex->stamp = jiffies; 1933 } 1934 1935 struct fib6_nh_match_arg { 1936 const struct net_device *dev; 1937 const struct in6_addr *gw; 1938 struct fib6_nh *match; 1939 }; 1940 1941 /* determine if fib6_nh has given device and gateway */ 1942 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg) 1943 { 1944 struct fib6_nh_match_arg *arg = _arg; 1945 1946 if (arg->dev != nh->fib_nh_dev || 1947 (arg->gw && !nh->fib_nh_gw_family) || 1948 (!arg->gw && nh->fib_nh_gw_family) || 1949 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6))) 1950 return 0; 1951 1952 arg->match = nh; 1953 1954 /* found a match, break the loop */ 1955 return 1; 1956 } 1957 1958 static void rt6_update_exception_stamp_rt(struct rt6_info *rt) 1959 { 1960 struct fib6_info *from; 1961 struct fib6_nh *fib6_nh; 1962 1963 rcu_read_lock(); 1964 1965 from = rcu_dereference(rt->from); 1966 if (!from || !(rt->rt6i_flags & RTF_CACHE)) 1967 goto unlock; 1968 1969 if (from->nh) { 1970 struct fib6_nh_match_arg arg = { 1971 .dev = rt->dst.dev, 1972 .gw = &rt->rt6i_gateway, 1973 }; 1974 1975 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg); 1976 1977 if (!arg.match) 1978 goto unlock; 1979 fib6_nh = arg.match; 1980 } else { 1981 fib6_nh = from->fib6_nh; 1982 } 1983 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt); 1984 unlock: 1985 rcu_read_unlock(); 1986 } 1987 1988 static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, 1989 struct rt6_info *rt, int mtu) 1990 { 1991 /* If the new MTU is lower than the route PMTU, this new MTU will be the 1992 * lowest MTU in the path: always allow updating the route PMTU to 1993 * reflect PMTU decreases. 1994 * 1995 * If the new MTU is higher, and the route PMTU is equal to the local 1996 * MTU, this means the old MTU is the lowest in the path, so allow 1997 * updating it: if other nodes now have lower MTUs, PMTU discovery will 1998 * handle this. 1999 */ 2000 2001 if (dst_mtu(&rt->dst) >= mtu) 2002 return true; 2003 2004 if (dst_mtu(&rt->dst) == idev->cnf.mtu6) 2005 return true; 2006 2007 return false; 2008 } 2009 2010 static void rt6_exceptions_update_pmtu(struct inet6_dev *idev, 2011 const struct fib6_nh *nh, int mtu) 2012 { 2013 struct rt6_exception_bucket *bucket; 2014 struct rt6_exception *rt6_ex; 2015 int i; 2016 2017 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2018 if (!bucket) 2019 return; 2020 2021 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2022 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 2023 struct rt6_info *entry = rt6_ex->rt6i; 2024 2025 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected 2026 * route), the metrics of its rt->from have already 2027 * been updated. 2028 */ 2029 if (dst_metric_raw(&entry->dst, RTAX_MTU) && 2030 rt6_mtu_change_route_allowed(idev, entry, mtu)) 2031 dst_metric_set(&entry->dst, RTAX_MTU, mtu); 2032 } 2033 bucket++; 2034 } 2035 } 2036 2037 #define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE) 2038 2039 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh, 2040 const struct in6_addr *gateway) 2041 { 2042 struct rt6_exception_bucket *bucket; 2043 struct rt6_exception *rt6_ex; 2044 struct hlist_node *tmp; 2045 int i; 2046 2047 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2048 return; 2049 2050 spin_lock_bh(&rt6_exception_lock); 2051 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2052 if (bucket) { 2053 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2054 hlist_for_each_entry_safe(rt6_ex, tmp, 2055 &bucket->chain, hlist) { 2056 struct rt6_info *entry = rt6_ex->rt6i; 2057 2058 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) == 2059 RTF_CACHE_GATEWAY && 2060 ipv6_addr_equal(gateway, 2061 &entry->rt6i_gateway)) { 2062 rt6_remove_exception(bucket, rt6_ex); 2063 } 2064 } 2065 bucket++; 2066 } 2067 } 2068 2069 spin_unlock_bh(&rt6_exception_lock); 2070 } 2071 2072 static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket, 2073 struct rt6_exception *rt6_ex, 2074 struct fib6_gc_args *gc_args, 2075 unsigned long now) 2076 { 2077 struct rt6_info *rt = rt6_ex->rt6i; 2078 2079 /* we are pruning and obsoleting aged-out and non gateway exceptions 2080 * even if others have still references to them, so that on next 2081 * dst_check() such references can be dropped. 2082 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when 2083 * expired, independently from their aging, as per RFC 8201 section 4 2084 */ 2085 if (!(rt->rt6i_flags & RTF_EXPIRES)) { 2086 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) { 2087 RT6_TRACE("aging clone %p\n", rt); 2088 rt6_remove_exception(bucket, rt6_ex); 2089 return; 2090 } 2091 } else if (time_after(jiffies, rt->dst.expires)) { 2092 RT6_TRACE("purging expired route %p\n", rt); 2093 rt6_remove_exception(bucket, rt6_ex); 2094 return; 2095 } 2096 2097 if (rt->rt6i_flags & RTF_GATEWAY) { 2098 struct neighbour *neigh; 2099 2100 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway); 2101 2102 if (!(neigh && (neigh->flags & NTF_ROUTER))) { 2103 RT6_TRACE("purging route %p via non-router but gateway\n", 2104 rt); 2105 rt6_remove_exception(bucket, rt6_ex); 2106 return; 2107 } 2108 } 2109 2110 gc_args->more++; 2111 } 2112 2113 static void fib6_nh_age_exceptions(const struct fib6_nh *nh, 2114 struct fib6_gc_args *gc_args, 2115 unsigned long now) 2116 { 2117 struct rt6_exception_bucket *bucket; 2118 struct rt6_exception *rt6_ex; 2119 struct hlist_node *tmp; 2120 int i; 2121 2122 if (!rcu_access_pointer(nh->rt6i_exception_bucket)) 2123 return; 2124 2125 rcu_read_lock_bh(); 2126 spin_lock(&rt6_exception_lock); 2127 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock); 2128 if (bucket) { 2129 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 2130 hlist_for_each_entry_safe(rt6_ex, tmp, 2131 &bucket->chain, hlist) { 2132 rt6_age_examine_exception(bucket, rt6_ex, 2133 gc_args, now); 2134 } 2135 bucket++; 2136 } 2137 } 2138 spin_unlock(&rt6_exception_lock); 2139 rcu_read_unlock_bh(); 2140 } 2141 2142 struct fib6_nh_age_excptn_arg { 2143 struct fib6_gc_args *gc_args; 2144 unsigned long now; 2145 }; 2146 2147 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg) 2148 { 2149 struct fib6_nh_age_excptn_arg *arg = _arg; 2150 2151 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now); 2152 return 0; 2153 } 2154 2155 void rt6_age_exceptions(struct fib6_info *f6i, 2156 struct fib6_gc_args *gc_args, 2157 unsigned long now) 2158 { 2159 if (f6i->nh) { 2160 struct fib6_nh_age_excptn_arg arg = { 2161 .gc_args = gc_args, 2162 .now = now 2163 }; 2164 2165 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions, 2166 &arg); 2167 } else { 2168 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now); 2169 } 2170 } 2171 2172 /* must be called with rcu lock held */ 2173 int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif, 2174 struct flowi6 *fl6, struct fib6_result *res, int strict) 2175 { 2176 struct fib6_node *fn, *saved_fn; 2177 2178 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 2179 saved_fn = fn; 2180 2181 redo_rt6_select: 2182 rt6_select(net, fn, oif, res, strict); 2183 if (res->f6i == net->ipv6.fib6_null_entry) { 2184 fn = fib6_backtrack(fn, &fl6->saddr); 2185 if (fn) 2186 goto redo_rt6_select; 2187 else if (strict & RT6_LOOKUP_F_REACHABLE) { 2188 /* also consider unreachable route */ 2189 strict &= ~RT6_LOOKUP_F_REACHABLE; 2190 fn = saved_fn; 2191 goto redo_rt6_select; 2192 } 2193 } 2194 2195 trace_fib6_table_lookup(net, res, table, fl6); 2196 2197 return 0; 2198 } 2199 2200 struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, 2201 int oif, struct flowi6 *fl6, 2202 const struct sk_buff *skb, int flags) 2203 { 2204 struct fib6_result res = {}; 2205 struct rt6_info *rt = NULL; 2206 int strict = 0; 2207 2208 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) && 2209 !rcu_read_lock_held()); 2210 2211 strict |= flags & RT6_LOOKUP_F_IFACE; 2212 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE; 2213 if (net->ipv6.devconf_all->forwarding == 0) 2214 strict |= RT6_LOOKUP_F_REACHABLE; 2215 2216 rcu_read_lock(); 2217 2218 fib6_table_lookup(net, table, oif, fl6, &res, strict); 2219 if (res.f6i == net->ipv6.fib6_null_entry) 2220 goto out; 2221 2222 fib6_select_path(net, &res, fl6, oif, false, skb, strict); 2223 2224 /*Search through exception table */ 2225 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr); 2226 if (rt) { 2227 goto out; 2228 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) && 2229 !res.nh->fib_nh_gw_family)) { 2230 /* Create a RTF_CACHE clone which will not be 2231 * owned by the fib6 tree. It is for the special case where 2232 * the daddr in the skb during the neighbor look-up is different 2233 * from the fl6->daddr used to look-up route here. 2234 */ 2235 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL); 2236 2237 if (rt) { 2238 /* 1 refcnt is taken during ip6_rt_cache_alloc(). 2239 * As rt6_uncached_list_add() does not consume refcnt, 2240 * this refcnt is always returned to the caller even 2241 * if caller sets RT6_LOOKUP_F_DST_NOREF flag. 2242 */ 2243 rt6_uncached_list_add(rt); 2244 rcu_read_unlock(); 2245 2246 return rt; 2247 } 2248 } else { 2249 /* Get a percpu copy */ 2250 local_bh_disable(); 2251 rt = rt6_get_pcpu_route(&res); 2252 2253 if (!rt) 2254 rt = rt6_make_pcpu_route(net, &res); 2255 2256 local_bh_enable(); 2257 } 2258 out: 2259 if (!rt) 2260 rt = net->ipv6.ip6_null_entry; 2261 if (!(flags & RT6_LOOKUP_F_DST_NOREF)) 2262 ip6_hold_safe(net, &rt); 2263 rcu_read_unlock(); 2264 2265 return rt; 2266 } 2267 EXPORT_SYMBOL_GPL(ip6_pol_route); 2268 2269 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, 2270 struct fib6_table *table, 2271 struct flowi6 *fl6, 2272 const struct sk_buff *skb, 2273 int flags) 2274 { 2275 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags); 2276 } 2277 2278 struct dst_entry *ip6_route_input_lookup(struct net *net, 2279 struct net_device *dev, 2280 struct flowi6 *fl6, 2281 const struct sk_buff *skb, 2282 int flags) 2283 { 2284 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG) 2285 flags |= RT6_LOOKUP_F_IFACE; 2286 2287 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input); 2288 } 2289 EXPORT_SYMBOL_GPL(ip6_route_input_lookup); 2290 2291 static void ip6_multipath_l3_keys(const struct sk_buff *skb, 2292 struct flow_keys *keys, 2293 struct flow_keys *flkeys) 2294 { 2295 const struct ipv6hdr *outer_iph = ipv6_hdr(skb); 2296 const struct ipv6hdr *key_iph = outer_iph; 2297 struct flow_keys *_flkeys = flkeys; 2298 const struct ipv6hdr *inner_iph; 2299 const struct icmp6hdr *icmph; 2300 struct ipv6hdr _inner_iph; 2301 struct icmp6hdr _icmph; 2302 2303 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6)) 2304 goto out; 2305 2306 icmph = skb_header_pointer(skb, skb_transport_offset(skb), 2307 sizeof(_icmph), &_icmph); 2308 if (!icmph) 2309 goto out; 2310 2311 if (!icmpv6_is_err(icmph->icmp6_type)) 2312 goto out; 2313 2314 inner_iph = skb_header_pointer(skb, 2315 skb_transport_offset(skb) + sizeof(*icmph), 2316 sizeof(_inner_iph), &_inner_iph); 2317 if (!inner_iph) 2318 goto out; 2319 2320 key_iph = inner_iph; 2321 _flkeys = NULL; 2322 out: 2323 if (_flkeys) { 2324 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src; 2325 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst; 2326 keys->tags.flow_label = _flkeys->tags.flow_label; 2327 keys->basic.ip_proto = _flkeys->basic.ip_proto; 2328 } else { 2329 keys->addrs.v6addrs.src = key_iph->saddr; 2330 keys->addrs.v6addrs.dst = key_iph->daddr; 2331 keys->tags.flow_label = ip6_flowlabel(key_iph); 2332 keys->basic.ip_proto = key_iph->nexthdr; 2333 } 2334 } 2335 2336 static u32 rt6_multipath_custom_hash_outer(const struct net *net, 2337 const struct sk_buff *skb, 2338 bool *p_has_inner) 2339 { 2340 u32 hash_fields = ip6_multipath_hash_fields(net); 2341 struct flow_keys keys, hash_keys; 2342 2343 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2344 return 0; 2345 2346 memset(&hash_keys, 0, sizeof(hash_keys)); 2347 skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP); 2348 2349 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2350 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2351 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2352 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2353 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2354 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2355 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2356 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2357 hash_keys.tags.flow_label = keys.tags.flow_label; 2358 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2359 hash_keys.ports.src = keys.ports.src; 2360 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2361 hash_keys.ports.dst = keys.ports.dst; 2362 2363 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION); 2364 return flow_hash_from_keys(&hash_keys); 2365 } 2366 2367 static u32 rt6_multipath_custom_hash_inner(const struct net *net, 2368 const struct sk_buff *skb, 2369 bool has_inner) 2370 { 2371 u32 hash_fields = ip6_multipath_hash_fields(net); 2372 struct flow_keys keys, hash_keys; 2373 2374 /* We assume the packet carries an encapsulation, but if none was 2375 * encountered during dissection of the outer flow, then there is no 2376 * point in calling the flow dissector again. 2377 */ 2378 if (!has_inner) 2379 return 0; 2380 2381 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)) 2382 return 0; 2383 2384 memset(&hash_keys, 0, sizeof(hash_keys)); 2385 skb_flow_dissect_flow_keys(skb, &keys, 0); 2386 2387 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION)) 2388 return 0; 2389 2390 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2391 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2392 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2393 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src; 2394 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2395 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst; 2396 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2397 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2398 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 2399 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src; 2400 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 2401 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst; 2402 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 2403 hash_keys.tags.flow_label = keys.tags.flow_label; 2404 } 2405 2406 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 2407 hash_keys.basic.ip_proto = keys.basic.ip_proto; 2408 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 2409 hash_keys.ports.src = keys.ports.src; 2410 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 2411 hash_keys.ports.dst = keys.ports.dst; 2412 2413 return flow_hash_from_keys(&hash_keys); 2414 } 2415 2416 static u32 rt6_multipath_custom_hash_skb(const struct net *net, 2417 const struct sk_buff *skb) 2418 { 2419 u32 mhash, mhash_inner; 2420 bool has_inner = true; 2421 2422 mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner); 2423 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner); 2424 2425 return jhash_2words(mhash, mhash_inner, 0); 2426 } 2427 2428 static u32 rt6_multipath_custom_hash_fl6(const struct net *net, 2429 const struct flowi6 *fl6) 2430 { 2431 u32 hash_fields = ip6_multipath_hash_fields(net); 2432 struct flow_keys hash_keys; 2433 2434 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) 2435 return 0; 2436 2437 memset(&hash_keys, 0, sizeof(hash_keys)); 2438 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2439 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 2440 hash_keys.addrs.v6addrs.src = fl6->saddr; 2441 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 2442 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2443 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 2444 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2445 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 2446 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2447 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 2448 hash_keys.ports.src = fl6->fl6_sport; 2449 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 2450 hash_keys.ports.dst = fl6->fl6_dport; 2451 2452 return flow_hash_from_keys(&hash_keys); 2453 } 2454 2455 /* if skb is set it will be used and fl6 can be NULL */ 2456 u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6, 2457 const struct sk_buff *skb, struct flow_keys *flkeys) 2458 { 2459 struct flow_keys hash_keys; 2460 u32 mhash = 0; 2461 2462 switch (ip6_multipath_hash_policy(net)) { 2463 case 0: 2464 memset(&hash_keys, 0, sizeof(hash_keys)); 2465 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2466 if (skb) { 2467 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2468 } else { 2469 hash_keys.addrs.v6addrs.src = fl6->saddr; 2470 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2471 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2472 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2473 } 2474 mhash = flow_hash_from_keys(&hash_keys); 2475 break; 2476 case 1: 2477 if (skb) { 2478 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP; 2479 struct flow_keys keys; 2480 2481 /* short-circuit if we already have L4 hash present */ 2482 if (skb->l4_hash) 2483 return skb_get_hash_raw(skb) >> 1; 2484 2485 memset(&hash_keys, 0, sizeof(hash_keys)); 2486 2487 if (!flkeys) { 2488 skb_flow_dissect_flow_keys(skb, &keys, flag); 2489 flkeys = &keys; 2490 } 2491 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2492 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2493 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2494 hash_keys.ports.src = flkeys->ports.src; 2495 hash_keys.ports.dst = flkeys->ports.dst; 2496 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2497 } else { 2498 memset(&hash_keys, 0, sizeof(hash_keys)); 2499 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2500 hash_keys.addrs.v6addrs.src = fl6->saddr; 2501 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2502 hash_keys.ports.src = fl6->fl6_sport; 2503 hash_keys.ports.dst = fl6->fl6_dport; 2504 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2505 } 2506 mhash = flow_hash_from_keys(&hash_keys); 2507 break; 2508 case 2: 2509 memset(&hash_keys, 0, sizeof(hash_keys)); 2510 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2511 if (skb) { 2512 struct flow_keys keys; 2513 2514 if (!flkeys) { 2515 skb_flow_dissect_flow_keys(skb, &keys, 0); 2516 flkeys = &keys; 2517 } 2518 2519 /* Inner can be v4 or v6 */ 2520 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 2521 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 2522 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src; 2523 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst; 2524 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 2525 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2526 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src; 2527 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst; 2528 hash_keys.tags.flow_label = flkeys->tags.flow_label; 2529 hash_keys.basic.ip_proto = flkeys->basic.ip_proto; 2530 } else { 2531 /* Same as case 0 */ 2532 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2533 ip6_multipath_l3_keys(skb, &hash_keys, flkeys); 2534 } 2535 } else { 2536 /* Same as case 0 */ 2537 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 2538 hash_keys.addrs.v6addrs.src = fl6->saddr; 2539 hash_keys.addrs.v6addrs.dst = fl6->daddr; 2540 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6); 2541 hash_keys.basic.ip_proto = fl6->flowi6_proto; 2542 } 2543 mhash = flow_hash_from_keys(&hash_keys); 2544 break; 2545 case 3: 2546 if (skb) 2547 mhash = rt6_multipath_custom_hash_skb(net, skb); 2548 else 2549 mhash = rt6_multipath_custom_hash_fl6(net, fl6); 2550 break; 2551 } 2552 2553 return mhash >> 1; 2554 } 2555 2556 /* Called with rcu held */ 2557 void ip6_route_input(struct sk_buff *skb) 2558 { 2559 const struct ipv6hdr *iph = ipv6_hdr(skb); 2560 struct net *net = dev_net(skb->dev); 2561 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF; 2562 struct ip_tunnel_info *tun_info; 2563 struct flowi6 fl6 = { 2564 .flowi6_iif = skb->dev->ifindex, 2565 .daddr = iph->daddr, 2566 .saddr = iph->saddr, 2567 .flowlabel = ip6_flowinfo(iph), 2568 .flowi6_mark = skb->mark, 2569 .flowi6_proto = iph->nexthdr, 2570 }; 2571 struct flow_keys *flkeys = NULL, _flkeys; 2572 2573 tun_info = skb_tunnel_info(skb); 2574 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX)) 2575 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id; 2576 2577 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys)) 2578 flkeys = &_flkeys; 2579 2580 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6)) 2581 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys); 2582 skb_dst_drop(skb); 2583 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev, 2584 &fl6, skb, flags)); 2585 } 2586 2587 INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, 2588 struct fib6_table *table, 2589 struct flowi6 *fl6, 2590 const struct sk_buff *skb, 2591 int flags) 2592 { 2593 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags); 2594 } 2595 2596 static struct dst_entry *ip6_route_output_flags_noref(struct net *net, 2597 const struct sock *sk, 2598 struct flowi6 *fl6, 2599 int flags) 2600 { 2601 bool any_src; 2602 2603 if (ipv6_addr_type(&fl6->daddr) & 2604 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) { 2605 struct dst_entry *dst; 2606 2607 /* This function does not take refcnt on the dst */ 2608 dst = l3mdev_link_scope_lookup(net, fl6); 2609 if (dst) 2610 return dst; 2611 } 2612 2613 fl6->flowi6_iif = LOOPBACK_IFINDEX; 2614 2615 flags |= RT6_LOOKUP_F_DST_NOREF; 2616 any_src = ipv6_addr_any(&fl6->saddr); 2617 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) || 2618 (fl6->flowi6_oif && any_src)) 2619 flags |= RT6_LOOKUP_F_IFACE; 2620 2621 if (!any_src) 2622 flags |= RT6_LOOKUP_F_HAS_SADDR; 2623 else if (sk) 2624 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs); 2625 2626 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output); 2627 } 2628 2629 struct dst_entry *ip6_route_output_flags(struct net *net, 2630 const struct sock *sk, 2631 struct flowi6 *fl6, 2632 int flags) 2633 { 2634 struct dst_entry *dst; 2635 struct rt6_info *rt6; 2636 2637 rcu_read_lock(); 2638 dst = ip6_route_output_flags_noref(net, sk, fl6, flags); 2639 rt6 = (struct rt6_info *)dst; 2640 /* For dst cached in uncached_list, refcnt is already taken. */ 2641 if (list_empty(&rt6->rt6i_uncached) && !dst_hold_safe(dst)) { 2642 dst = &net->ipv6.ip6_null_entry->dst; 2643 dst_hold(dst); 2644 } 2645 rcu_read_unlock(); 2646 2647 return dst; 2648 } 2649 EXPORT_SYMBOL_GPL(ip6_route_output_flags); 2650 2651 struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig) 2652 { 2653 struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig; 2654 struct net_device *loopback_dev = net->loopback_dev; 2655 struct dst_entry *new = NULL; 2656 2657 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1, 2658 DST_OBSOLETE_DEAD, 0); 2659 if (rt) { 2660 rt6_info_init(rt); 2661 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc); 2662 2663 new = &rt->dst; 2664 new->__use = 1; 2665 new->input = dst_discard; 2666 new->output = dst_discard_out; 2667 2668 dst_copy_metrics(new, &ort->dst); 2669 2670 rt->rt6i_idev = in6_dev_get(loopback_dev); 2671 rt->rt6i_gateway = ort->rt6i_gateway; 2672 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU; 2673 2674 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key)); 2675 #ifdef CONFIG_IPV6_SUBTREES 2676 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key)); 2677 #endif 2678 } 2679 2680 dst_release(dst_orig); 2681 return new ? new : ERR_PTR(-ENOMEM); 2682 } 2683 2684 /* 2685 * Destination cache support functions 2686 */ 2687 2688 static bool fib6_check(struct fib6_info *f6i, u32 cookie) 2689 { 2690 u32 rt_cookie = 0; 2691 2692 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie) 2693 return false; 2694 2695 if (fib6_check_expired(f6i)) 2696 return false; 2697 2698 return true; 2699 } 2700 2701 static struct dst_entry *rt6_check(struct rt6_info *rt, 2702 struct fib6_info *from, 2703 u32 cookie) 2704 { 2705 u32 rt_cookie = 0; 2706 2707 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) || 2708 rt_cookie != cookie) 2709 return NULL; 2710 2711 if (rt6_check_expired(rt)) 2712 return NULL; 2713 2714 return &rt->dst; 2715 } 2716 2717 static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt, 2718 struct fib6_info *from, 2719 u32 cookie) 2720 { 2721 if (!__rt6_check_expired(rt) && 2722 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK && 2723 fib6_check(from, cookie)) 2724 return &rt->dst; 2725 else 2726 return NULL; 2727 } 2728 2729 INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, 2730 u32 cookie) 2731 { 2732 struct dst_entry *dst_ret; 2733 struct fib6_info *from; 2734 struct rt6_info *rt; 2735 2736 rt = container_of(dst, struct rt6_info, dst); 2737 2738 if (rt->sernum) 2739 return rt6_is_valid(rt) ? dst : NULL; 2740 2741 rcu_read_lock(); 2742 2743 /* All IPV6 dsts are created with ->obsolete set to the value 2744 * DST_OBSOLETE_FORCE_CHK which forces validation calls down 2745 * into this function always. 2746 */ 2747 2748 from = rcu_dereference(rt->from); 2749 2750 if (from && (rt->rt6i_flags & RTF_PCPU || 2751 unlikely(!list_empty(&rt->rt6i_uncached)))) 2752 dst_ret = rt6_dst_from_check(rt, from, cookie); 2753 else 2754 dst_ret = rt6_check(rt, from, cookie); 2755 2756 rcu_read_unlock(); 2757 2758 return dst_ret; 2759 } 2760 EXPORT_INDIRECT_CALLABLE(ip6_dst_check); 2761 2762 static struct dst_entry *ip6_negative_advice(struct dst_entry *dst) 2763 { 2764 struct rt6_info *rt = (struct rt6_info *) dst; 2765 2766 if (rt) { 2767 if (rt->rt6i_flags & RTF_CACHE) { 2768 rcu_read_lock(); 2769 if (rt6_check_expired(rt)) { 2770 rt6_remove_exception_rt(rt); 2771 dst = NULL; 2772 } 2773 rcu_read_unlock(); 2774 } else { 2775 dst_release(dst); 2776 dst = NULL; 2777 } 2778 } 2779 return dst; 2780 } 2781 2782 static void ip6_link_failure(struct sk_buff *skb) 2783 { 2784 struct rt6_info *rt; 2785 2786 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); 2787 2788 rt = (struct rt6_info *) skb_dst(skb); 2789 if (rt) { 2790 rcu_read_lock(); 2791 if (rt->rt6i_flags & RTF_CACHE) { 2792 rt6_remove_exception_rt(rt); 2793 } else { 2794 struct fib6_info *from; 2795 struct fib6_node *fn; 2796 2797 from = rcu_dereference(rt->from); 2798 if (from) { 2799 fn = rcu_dereference(from->fib6_node); 2800 if (fn && (rt->rt6i_flags & RTF_DEFAULT)) 2801 WRITE_ONCE(fn->fn_sernum, -1); 2802 } 2803 } 2804 rcu_read_unlock(); 2805 } 2806 } 2807 2808 static void rt6_update_expires(struct rt6_info *rt0, int timeout) 2809 { 2810 if (!(rt0->rt6i_flags & RTF_EXPIRES)) { 2811 struct fib6_info *from; 2812 2813 rcu_read_lock(); 2814 from = rcu_dereference(rt0->from); 2815 if (from) 2816 rt0->dst.expires = from->expires; 2817 rcu_read_unlock(); 2818 } 2819 2820 dst_set_expires(&rt0->dst, timeout); 2821 rt0->rt6i_flags |= RTF_EXPIRES; 2822 } 2823 2824 static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) 2825 { 2826 struct net *net = dev_net(rt->dst.dev); 2827 2828 dst_metric_set(&rt->dst, RTAX_MTU, mtu); 2829 rt->rt6i_flags |= RTF_MODIFIED; 2830 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); 2831 } 2832 2833 static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) 2834 { 2835 return !(rt->rt6i_flags & RTF_CACHE) && 2836 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from)); 2837 } 2838 2839 static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk, 2840 const struct ipv6hdr *iph, u32 mtu, 2841 bool confirm_neigh) 2842 { 2843 const struct in6_addr *daddr, *saddr; 2844 struct rt6_info *rt6 = (struct rt6_info *)dst; 2845 2846 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU) 2847 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it. 2848 * [see also comment in rt6_mtu_change_route()] 2849 */ 2850 2851 if (iph) { 2852 daddr = &iph->daddr; 2853 saddr = &iph->saddr; 2854 } else if (sk) { 2855 daddr = &sk->sk_v6_daddr; 2856 saddr = &inet6_sk(sk)->saddr; 2857 } else { 2858 daddr = NULL; 2859 saddr = NULL; 2860 } 2861 2862 if (confirm_neigh) 2863 dst_confirm_neigh(dst, daddr); 2864 2865 if (mtu < IPV6_MIN_MTU) 2866 return; 2867 if (mtu >= dst_mtu(dst)) 2868 return; 2869 2870 if (!rt6_cache_allowed_for_pmtu(rt6)) { 2871 rt6_do_update_pmtu(rt6, mtu); 2872 /* update rt6_ex->stamp for cache */ 2873 if (rt6->rt6i_flags & RTF_CACHE) 2874 rt6_update_exception_stamp_rt(rt6); 2875 } else if (daddr) { 2876 struct fib6_result res = {}; 2877 struct rt6_info *nrt6; 2878 2879 rcu_read_lock(); 2880 res.f6i = rcu_dereference(rt6->from); 2881 if (!res.f6i) 2882 goto out_unlock; 2883 2884 res.fib6_flags = res.f6i->fib6_flags; 2885 res.fib6_type = res.f6i->fib6_type; 2886 2887 if (res.f6i->nh) { 2888 struct fib6_nh_match_arg arg = { 2889 .dev = dst->dev, 2890 .gw = &rt6->rt6i_gateway, 2891 }; 2892 2893 nexthop_for_each_fib6_nh(res.f6i->nh, 2894 fib6_nh_find_match, &arg); 2895 2896 /* fib6_info uses a nexthop that does not have fib6_nh 2897 * using the dst->dev + gw. Should be impossible. 2898 */ 2899 if (!arg.match) 2900 goto out_unlock; 2901 2902 res.nh = arg.match; 2903 } else { 2904 res.nh = res.f6i->fib6_nh; 2905 } 2906 2907 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr); 2908 if (nrt6) { 2909 rt6_do_update_pmtu(nrt6, mtu); 2910 if (rt6_insert_exception(nrt6, &res)) 2911 dst_release_immediate(&nrt6->dst); 2912 } 2913 out_unlock: 2914 rcu_read_unlock(); 2915 } 2916 } 2917 2918 static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, 2919 struct sk_buff *skb, u32 mtu, 2920 bool confirm_neigh) 2921 { 2922 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu, 2923 confirm_neigh); 2924 } 2925 2926 void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, 2927 int oif, u32 mark, kuid_t uid) 2928 { 2929 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 2930 struct dst_entry *dst; 2931 struct flowi6 fl6 = { 2932 .flowi6_oif = oif, 2933 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark), 2934 .daddr = iph->daddr, 2935 .saddr = iph->saddr, 2936 .flowlabel = ip6_flowinfo(iph), 2937 .flowi6_uid = uid, 2938 }; 2939 2940 dst = ip6_route_output(net, NULL, &fl6); 2941 if (!dst->error) 2942 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true); 2943 dst_release(dst); 2944 } 2945 EXPORT_SYMBOL_GPL(ip6_update_pmtu); 2946 2947 void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu) 2948 { 2949 int oif = sk->sk_bound_dev_if; 2950 struct dst_entry *dst; 2951 2952 if (!oif && skb->dev) 2953 oif = l3mdev_master_ifindex(skb->dev); 2954 2955 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, sk->sk_mark, sk->sk_uid); 2956 2957 dst = __sk_dst_get(sk); 2958 if (!dst || !dst->obsolete || 2959 dst->ops->check(dst, inet6_sk(sk)->dst_cookie)) 2960 return; 2961 2962 bh_lock_sock(sk); 2963 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) 2964 ip6_datagram_dst_update(sk, false); 2965 bh_unlock_sock(sk); 2966 } 2967 EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu); 2968 2969 void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst, 2970 const struct flowi6 *fl6) 2971 { 2972 #ifdef CONFIG_IPV6_SUBTREES 2973 struct ipv6_pinfo *np = inet6_sk(sk); 2974 #endif 2975 2976 ip6_dst_store(sk, dst, 2977 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ? 2978 &sk->sk_v6_daddr : NULL, 2979 #ifdef CONFIG_IPV6_SUBTREES 2980 ipv6_addr_equal(&fl6->saddr, &np->saddr) ? 2981 &np->saddr : 2982 #endif 2983 NULL); 2984 } 2985 2986 static bool ip6_redirect_nh_match(const struct fib6_result *res, 2987 struct flowi6 *fl6, 2988 const struct in6_addr *gw, 2989 struct rt6_info **ret) 2990 { 2991 const struct fib6_nh *nh = res->nh; 2992 2993 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family || 2994 fl6->flowi6_oif != nh->fib_nh_dev->ifindex) 2995 return false; 2996 2997 /* rt_cache's gateway might be different from its 'parent' 2998 * in the case of an ip redirect. 2999 * So we keep searching in the exception table if the gateway 3000 * is different. 3001 */ 3002 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) { 3003 struct rt6_info *rt_cache; 3004 3005 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr); 3006 if (rt_cache && 3007 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) { 3008 *ret = rt_cache; 3009 return true; 3010 } 3011 return false; 3012 } 3013 return true; 3014 } 3015 3016 struct fib6_nh_rd_arg { 3017 struct fib6_result *res; 3018 struct flowi6 *fl6; 3019 const struct in6_addr *gw; 3020 struct rt6_info **ret; 3021 }; 3022 3023 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg) 3024 { 3025 struct fib6_nh_rd_arg *arg = _arg; 3026 3027 arg->res->nh = nh; 3028 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret); 3029 } 3030 3031 /* Handle redirects */ 3032 struct ip6rd_flowi { 3033 struct flowi6 fl6; 3034 struct in6_addr gateway; 3035 }; 3036 3037 INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, 3038 struct fib6_table *table, 3039 struct flowi6 *fl6, 3040 const struct sk_buff *skb, 3041 int flags) 3042 { 3043 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6; 3044 struct rt6_info *ret = NULL; 3045 struct fib6_result res = {}; 3046 struct fib6_nh_rd_arg arg = { 3047 .res = &res, 3048 .fl6 = fl6, 3049 .gw = &rdfl->gateway, 3050 .ret = &ret 3051 }; 3052 struct fib6_info *rt; 3053 struct fib6_node *fn; 3054 3055 /* Get the "current" route for this destination and 3056 * check if the redirect has come from appropriate router. 3057 * 3058 * RFC 4861 specifies that redirects should only be 3059 * accepted if they come from the nexthop to the target. 3060 * Due to the way the routes are chosen, this notion 3061 * is a bit fuzzy and one might need to check all possible 3062 * routes. 3063 */ 3064 3065 rcu_read_lock(); 3066 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr); 3067 restart: 3068 for_each_fib6_node_rt_rcu(fn) { 3069 res.f6i = rt; 3070 if (fib6_check_expired(rt)) 3071 continue; 3072 if (rt->fib6_flags & RTF_REJECT) 3073 break; 3074 if (unlikely(rt->nh)) { 3075 if (nexthop_is_blackhole(rt->nh)) 3076 continue; 3077 /* on match, res->nh is filled in and potentially ret */ 3078 if (nexthop_for_each_fib6_nh(rt->nh, 3079 fib6_nh_redirect_match, 3080 &arg)) 3081 goto out; 3082 } else { 3083 res.nh = rt->fib6_nh; 3084 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway, 3085 &ret)) 3086 goto out; 3087 } 3088 } 3089 3090 if (!rt) 3091 rt = net->ipv6.fib6_null_entry; 3092 else if (rt->fib6_flags & RTF_REJECT) { 3093 ret = net->ipv6.ip6_null_entry; 3094 goto out; 3095 } 3096 3097 if (rt == net->ipv6.fib6_null_entry) { 3098 fn = fib6_backtrack(fn, &fl6->saddr); 3099 if (fn) 3100 goto restart; 3101 } 3102 3103 res.f6i = rt; 3104 res.nh = rt->fib6_nh; 3105 out: 3106 if (ret) { 3107 ip6_hold_safe(net, &ret); 3108 } else { 3109 res.fib6_flags = res.f6i->fib6_flags; 3110 res.fib6_type = res.f6i->fib6_type; 3111 ret = ip6_create_rt_rcu(&res); 3112 } 3113 3114 rcu_read_unlock(); 3115 3116 trace_fib6_table_lookup(net, &res, table, fl6); 3117 return ret; 3118 }; 3119 3120 static struct dst_entry *ip6_route_redirect(struct net *net, 3121 const struct flowi6 *fl6, 3122 const struct sk_buff *skb, 3123 const struct in6_addr *gateway) 3124 { 3125 int flags = RT6_LOOKUP_F_HAS_SADDR; 3126 struct ip6rd_flowi rdfl; 3127 3128 rdfl.fl6 = *fl6; 3129 rdfl.gateway = *gateway; 3130 3131 return fib6_rule_lookup(net, &rdfl.fl6, skb, 3132 flags, __ip6_route_redirect); 3133 } 3134 3135 void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, 3136 kuid_t uid) 3137 { 3138 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; 3139 struct dst_entry *dst; 3140 struct flowi6 fl6 = { 3141 .flowi6_iif = LOOPBACK_IFINDEX, 3142 .flowi6_oif = oif, 3143 .flowi6_mark = mark, 3144 .daddr = iph->daddr, 3145 .saddr = iph->saddr, 3146 .flowlabel = ip6_flowinfo(iph), 3147 .flowi6_uid = uid, 3148 }; 3149 3150 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr); 3151 rt6_do_redirect(dst, NULL, skb); 3152 dst_release(dst); 3153 } 3154 EXPORT_SYMBOL_GPL(ip6_redirect); 3155 3156 void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif) 3157 { 3158 const struct ipv6hdr *iph = ipv6_hdr(skb); 3159 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb); 3160 struct dst_entry *dst; 3161 struct flowi6 fl6 = { 3162 .flowi6_iif = LOOPBACK_IFINDEX, 3163 .flowi6_oif = oif, 3164 .daddr = msg->dest, 3165 .saddr = iph->daddr, 3166 .flowi6_uid = sock_net_uid(net, NULL), 3167 }; 3168 3169 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr); 3170 rt6_do_redirect(dst, NULL, skb); 3171 dst_release(dst); 3172 } 3173 3174 void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) 3175 { 3176 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark, 3177 sk->sk_uid); 3178 } 3179 EXPORT_SYMBOL_GPL(ip6_sk_redirect); 3180 3181 static unsigned int ip6_default_advmss(const struct dst_entry *dst) 3182 { 3183 struct net_device *dev = dst->dev; 3184 unsigned int mtu = dst_mtu(dst); 3185 struct net *net = dev_net(dev); 3186 3187 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 3188 3189 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) 3190 mtu = net->ipv6.sysctl.ip6_rt_min_advmss; 3191 3192 /* 3193 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and 3194 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size. 3195 * IPV6_MAXPLEN is also valid and means: "any MSS, 3196 * rely only on pmtu discovery" 3197 */ 3198 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr)) 3199 mtu = IPV6_MAXPLEN; 3200 return mtu; 3201 } 3202 3203 INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst) 3204 { 3205 return ip6_dst_mtu_maybe_forward(dst, false); 3206 } 3207 EXPORT_INDIRECT_CALLABLE(ip6_mtu); 3208 3209 /* MTU selection: 3210 * 1. mtu on route is locked - use it 3211 * 2. mtu from nexthop exception 3212 * 3. mtu from egress device 3213 * 3214 * based on ip6_dst_mtu_forward and exception logic of 3215 * rt6_find_cached_rt; called with rcu_read_lock 3216 */ 3217 u32 ip6_mtu_from_fib6(const struct fib6_result *res, 3218 const struct in6_addr *daddr, 3219 const struct in6_addr *saddr) 3220 { 3221 const struct fib6_nh *nh = res->nh; 3222 struct fib6_info *f6i = res->f6i; 3223 struct inet6_dev *idev; 3224 struct rt6_info *rt; 3225 u32 mtu = 0; 3226 3227 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) { 3228 mtu = f6i->fib6_pmtu; 3229 if (mtu) 3230 goto out; 3231 } 3232 3233 rt = rt6_find_cached_rt(res, daddr, saddr); 3234 if (unlikely(rt)) { 3235 mtu = dst_metric_raw(&rt->dst, RTAX_MTU); 3236 } else { 3237 struct net_device *dev = nh->fib_nh_dev; 3238 3239 mtu = IPV6_MIN_MTU; 3240 idev = __in6_dev_get(dev); 3241 if (idev && idev->cnf.mtu6 > mtu) 3242 mtu = idev->cnf.mtu6; 3243 } 3244 3245 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); 3246 out: 3247 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu); 3248 } 3249 3250 struct dst_entry *icmp6_dst_alloc(struct net_device *dev, 3251 struct flowi6 *fl6) 3252 { 3253 struct dst_entry *dst; 3254 struct rt6_info *rt; 3255 struct inet6_dev *idev = in6_dev_get(dev); 3256 struct net *net = dev_net(dev); 3257 3258 if (unlikely(!idev)) 3259 return ERR_PTR(-ENODEV); 3260 3261 rt = ip6_dst_alloc(net, dev, 0); 3262 if (unlikely(!rt)) { 3263 in6_dev_put(idev); 3264 dst = ERR_PTR(-ENOMEM); 3265 goto out; 3266 } 3267 3268 rt->dst.input = ip6_input; 3269 rt->dst.output = ip6_output; 3270 rt->rt6i_gateway = fl6->daddr; 3271 rt->rt6i_dst.addr = fl6->daddr; 3272 rt->rt6i_dst.plen = 128; 3273 rt->rt6i_idev = idev; 3274 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0); 3275 3276 /* Add this dst into uncached_list so that rt6_disable_ip() can 3277 * do proper release of the net_device 3278 */ 3279 rt6_uncached_list_add(rt); 3280 3281 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0); 3282 3283 out: 3284 return dst; 3285 } 3286 3287 static void ip6_dst_gc(struct dst_ops *ops) 3288 { 3289 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); 3290 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; 3291 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; 3292 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; 3293 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; 3294 unsigned int val; 3295 int entries; 3296 3297 if (time_after(rt_last_gc + rt_min_interval, jiffies)) 3298 goto out; 3299 3300 fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true); 3301 entries = dst_entries_get_slow(ops); 3302 if (entries < ops->gc_thresh) 3303 atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1); 3304 out: 3305 val = atomic_read(&net->ipv6.ip6_rt_gc_expire); 3306 atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); 3307 } 3308 3309 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg, 3310 const struct in6_addr *gw_addr, u32 tbid, 3311 int flags, struct fib6_result *res) 3312 { 3313 struct flowi6 fl6 = { 3314 .flowi6_oif = cfg->fc_ifindex, 3315 .daddr = *gw_addr, 3316 .saddr = cfg->fc_prefsrc, 3317 }; 3318 struct fib6_table *table; 3319 int err; 3320 3321 table = fib6_get_table(net, tbid); 3322 if (!table) 3323 return -EINVAL; 3324 3325 if (!ipv6_addr_any(&cfg->fc_prefsrc)) 3326 flags |= RT6_LOOKUP_F_HAS_SADDR; 3327 3328 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE; 3329 3330 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags); 3331 if (!err && res->f6i != net->ipv6.fib6_null_entry) 3332 fib6_select_path(net, res, &fl6, cfg->fc_ifindex, 3333 cfg->fc_ifindex != 0, NULL, flags); 3334 3335 return err; 3336 } 3337 3338 static int ip6_route_check_nh_onlink(struct net *net, 3339 struct fib6_config *cfg, 3340 const struct net_device *dev, 3341 struct netlink_ext_ack *extack) 3342 { 3343 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; 3344 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3345 struct fib6_result res = {}; 3346 int err; 3347 3348 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res); 3349 if (!err && !(res.fib6_flags & RTF_REJECT) && 3350 /* ignore match if it is the default route */ 3351 !ipv6_addr_any(&res.f6i->fib6_dst.addr) && 3352 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) { 3353 NL_SET_ERR_MSG(extack, 3354 "Nexthop has invalid gateway or device mismatch"); 3355 err = -EINVAL; 3356 } 3357 3358 return err; 3359 } 3360 3361 static int ip6_route_check_nh(struct net *net, 3362 struct fib6_config *cfg, 3363 struct net_device **_dev, 3364 struct inet6_dev **idev) 3365 { 3366 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3367 struct net_device *dev = _dev ? *_dev : NULL; 3368 int flags = RT6_LOOKUP_F_IFACE; 3369 struct fib6_result res = {}; 3370 int err = -EHOSTUNREACH; 3371 3372 if (cfg->fc_table) { 3373 err = ip6_nh_lookup_table(net, cfg, gw_addr, 3374 cfg->fc_table, flags, &res); 3375 /* gw_addr can not require a gateway or resolve to a reject 3376 * route. If a device is given, it must match the result. 3377 */ 3378 if (err || res.fib6_flags & RTF_REJECT || 3379 res.nh->fib_nh_gw_family || 3380 (dev && dev != res.nh->fib_nh_dev)) 3381 err = -EHOSTUNREACH; 3382 } 3383 3384 if (err < 0) { 3385 struct flowi6 fl6 = { 3386 .flowi6_oif = cfg->fc_ifindex, 3387 .daddr = *gw_addr, 3388 }; 3389 3390 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags); 3391 if (err || res.fib6_flags & RTF_REJECT || 3392 res.nh->fib_nh_gw_family) 3393 err = -EHOSTUNREACH; 3394 3395 if (err) 3396 return err; 3397 3398 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex, 3399 cfg->fc_ifindex != 0, NULL, flags); 3400 } 3401 3402 err = 0; 3403 if (dev) { 3404 if (dev != res.nh->fib_nh_dev) 3405 err = -EHOSTUNREACH; 3406 } else { 3407 *_dev = dev = res.nh->fib_nh_dev; 3408 dev_hold(dev); 3409 *idev = in6_dev_get(dev); 3410 } 3411 3412 return err; 3413 } 3414 3415 static int ip6_validate_gw(struct net *net, struct fib6_config *cfg, 3416 struct net_device **_dev, struct inet6_dev **idev, 3417 struct netlink_ext_ack *extack) 3418 { 3419 const struct in6_addr *gw_addr = &cfg->fc_gateway; 3420 int gwa_type = ipv6_addr_type(gw_addr); 3421 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true; 3422 const struct net_device *dev = *_dev; 3423 bool need_addr_check = !dev; 3424 int err = -EINVAL; 3425 3426 /* if gw_addr is local we will fail to detect this in case 3427 * address is still TENTATIVE (DAD in progress). rt6_lookup() 3428 * will return already-added prefix route via interface that 3429 * prefix route was assigned to, which might be non-loopback. 3430 */ 3431 if (dev && 3432 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3433 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3434 goto out; 3435 } 3436 3437 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) { 3438 /* IPv6 strictly inhibits using not link-local 3439 * addresses as nexthop address. 3440 * Otherwise, router will not able to send redirects. 3441 * It is very good, but in some (rare!) circumstances 3442 * (SIT, PtP, NBMA NOARP links) it is handy to allow 3443 * some exceptions. --ANK 3444 * We allow IPv4-mapped nexthops to support RFC4798-type 3445 * addressing 3446 */ 3447 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) { 3448 NL_SET_ERR_MSG(extack, "Invalid gateway address"); 3449 goto out; 3450 } 3451 3452 rcu_read_lock(); 3453 3454 if (cfg->fc_flags & RTNH_F_ONLINK) 3455 err = ip6_route_check_nh_onlink(net, cfg, dev, extack); 3456 else 3457 err = ip6_route_check_nh(net, cfg, _dev, idev); 3458 3459 rcu_read_unlock(); 3460 3461 if (err) 3462 goto out; 3463 } 3464 3465 /* reload in case device was changed */ 3466 dev = *_dev; 3467 3468 err = -EINVAL; 3469 if (!dev) { 3470 NL_SET_ERR_MSG(extack, "Egress device not specified"); 3471 goto out; 3472 } else if (dev->flags & IFF_LOOPBACK) { 3473 NL_SET_ERR_MSG(extack, 3474 "Egress device can not be loopback device for this route"); 3475 goto out; 3476 } 3477 3478 /* if we did not check gw_addr above, do so now that the 3479 * egress device has been resolved. 3480 */ 3481 if (need_addr_check && 3482 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) { 3483 NL_SET_ERR_MSG(extack, "Gateway can not be a local address"); 3484 goto out; 3485 } 3486 3487 err = 0; 3488 out: 3489 return err; 3490 } 3491 3492 static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type) 3493 { 3494 if ((flags & RTF_REJECT) || 3495 (dev && (dev->flags & IFF_LOOPBACK) && 3496 !(addr_type & IPV6_ADDR_LOOPBACK) && 3497 !(flags & (RTF_ANYCAST | RTF_LOCAL)))) 3498 return true; 3499 3500 return false; 3501 } 3502 3503 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, 3504 struct fib6_config *cfg, gfp_t gfp_flags, 3505 struct netlink_ext_ack *extack) 3506 { 3507 struct net_device *dev = NULL; 3508 struct inet6_dev *idev = NULL; 3509 int addr_type; 3510 int err; 3511 3512 fib6_nh->fib_nh_family = AF_INET6; 3513 #ifdef CONFIG_IPV6_ROUTER_PREF 3514 fib6_nh->last_probe = jiffies; 3515 #endif 3516 if (cfg->fc_is_fdb) { 3517 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3518 fib6_nh->fib_nh_gw_family = AF_INET6; 3519 return 0; 3520 } 3521 3522 err = -ENODEV; 3523 if (cfg->fc_ifindex) { 3524 dev = dev_get_by_index(net, cfg->fc_ifindex); 3525 if (!dev) 3526 goto out; 3527 idev = in6_dev_get(dev); 3528 if (!idev) 3529 goto out; 3530 } 3531 3532 if (cfg->fc_flags & RTNH_F_ONLINK) { 3533 if (!dev) { 3534 NL_SET_ERR_MSG(extack, 3535 "Nexthop device required for onlink"); 3536 goto out; 3537 } 3538 3539 if (!(dev->flags & IFF_UP)) { 3540 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3541 err = -ENETDOWN; 3542 goto out; 3543 } 3544 3545 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK; 3546 } 3547 3548 fib6_nh->fib_nh_weight = 1; 3549 3550 /* We cannot add true routes via loopback here, 3551 * they would result in kernel looping; promote them to reject routes 3552 */ 3553 addr_type = ipv6_addr_type(&cfg->fc_dst); 3554 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { 3555 /* hold loopback dev/idev if we haven't done so. */ 3556 if (dev != net->loopback_dev) { 3557 if (dev) { 3558 dev_put(dev); 3559 in6_dev_put(idev); 3560 } 3561 dev = net->loopback_dev; 3562 dev_hold(dev); 3563 idev = in6_dev_get(dev); 3564 if (!idev) { 3565 err = -ENODEV; 3566 goto out; 3567 } 3568 } 3569 goto pcpu_alloc; 3570 } 3571 3572 if (cfg->fc_flags & RTF_GATEWAY) { 3573 err = ip6_validate_gw(net, cfg, &dev, &idev, extack); 3574 if (err) 3575 goto out; 3576 3577 fib6_nh->fib_nh_gw6 = cfg->fc_gateway; 3578 fib6_nh->fib_nh_gw_family = AF_INET6; 3579 } 3580 3581 err = -ENODEV; 3582 if (!dev) 3583 goto out; 3584 3585 if (idev->cnf.disable_ipv6) { 3586 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device"); 3587 err = -EACCES; 3588 goto out; 3589 } 3590 3591 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) { 3592 NL_SET_ERR_MSG(extack, "Nexthop device is not up"); 3593 err = -ENETDOWN; 3594 goto out; 3595 } 3596 3597 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) && 3598 !netif_carrier_ok(dev)) 3599 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 3600 3601 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap, 3602 cfg->fc_encap_type, cfg, gfp_flags, extack); 3603 if (err) 3604 goto out; 3605 3606 pcpu_alloc: 3607 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags); 3608 if (!fib6_nh->rt6i_pcpu) { 3609 err = -ENOMEM; 3610 goto out; 3611 } 3612 3613 fib6_nh->fib_nh_dev = dev; 3614 netdev_tracker_alloc(dev, &fib6_nh->fib_nh_dev_tracker, gfp_flags); 3615 3616 fib6_nh->fib_nh_oif = dev->ifindex; 3617 err = 0; 3618 out: 3619 if (idev) 3620 in6_dev_put(idev); 3621 3622 if (err) { 3623 lwtstate_put(fib6_nh->fib_nh_lws); 3624 fib6_nh->fib_nh_lws = NULL; 3625 dev_put(dev); 3626 } 3627 3628 return err; 3629 } 3630 3631 void fib6_nh_release(struct fib6_nh *fib6_nh) 3632 { 3633 struct rt6_exception_bucket *bucket; 3634 3635 rcu_read_lock(); 3636 3637 fib6_nh_flush_exceptions(fib6_nh, NULL); 3638 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL); 3639 if (bucket) { 3640 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL); 3641 kfree(bucket); 3642 } 3643 3644 rcu_read_unlock(); 3645 3646 fib6_nh_release_dsts(fib6_nh); 3647 free_percpu(fib6_nh->rt6i_pcpu); 3648 3649 fib_nh_common_release(&fib6_nh->nh_common); 3650 } 3651 3652 void fib6_nh_release_dsts(struct fib6_nh *fib6_nh) 3653 { 3654 int cpu; 3655 3656 if (!fib6_nh->rt6i_pcpu) 3657 return; 3658 3659 for_each_possible_cpu(cpu) { 3660 struct rt6_info *pcpu_rt, **ppcpu_rt; 3661 3662 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); 3663 pcpu_rt = xchg(ppcpu_rt, NULL); 3664 if (pcpu_rt) { 3665 dst_dev_put(&pcpu_rt->dst); 3666 dst_release(&pcpu_rt->dst); 3667 } 3668 } 3669 } 3670 3671 static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg, 3672 gfp_t gfp_flags, 3673 struct netlink_ext_ack *extack) 3674 { 3675 struct net *net = cfg->fc_nlinfo.nl_net; 3676 struct fib6_info *rt = NULL; 3677 struct nexthop *nh = NULL; 3678 struct fib6_table *table; 3679 struct fib6_nh *fib6_nh; 3680 int err = -EINVAL; 3681 int addr_type; 3682 3683 /* RTF_PCPU is an internal flag; can not be set by userspace */ 3684 if (cfg->fc_flags & RTF_PCPU) { 3685 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU"); 3686 goto out; 3687 } 3688 3689 /* RTF_CACHE is an internal flag; can not be set by userspace */ 3690 if (cfg->fc_flags & RTF_CACHE) { 3691 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE"); 3692 goto out; 3693 } 3694 3695 if (cfg->fc_type > RTN_MAX) { 3696 NL_SET_ERR_MSG(extack, "Invalid route type"); 3697 goto out; 3698 } 3699 3700 if (cfg->fc_dst_len > 128) { 3701 NL_SET_ERR_MSG(extack, "Invalid prefix length"); 3702 goto out; 3703 } 3704 if (cfg->fc_src_len > 128) { 3705 NL_SET_ERR_MSG(extack, "Invalid source address length"); 3706 goto out; 3707 } 3708 #ifndef CONFIG_IPV6_SUBTREES 3709 if (cfg->fc_src_len) { 3710 NL_SET_ERR_MSG(extack, 3711 "Specifying source address requires IPV6_SUBTREES to be enabled"); 3712 goto out; 3713 } 3714 #endif 3715 if (cfg->fc_nh_id) { 3716 nh = nexthop_find_by_id(net, cfg->fc_nh_id); 3717 if (!nh) { 3718 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 3719 goto out; 3720 } 3721 err = fib6_check_nexthop(nh, cfg, extack); 3722 if (err) 3723 goto out; 3724 } 3725 3726 err = -ENOBUFS; 3727 if (cfg->fc_nlinfo.nlh && 3728 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) { 3729 table = fib6_get_table(net, cfg->fc_table); 3730 if (!table) { 3731 pr_warn("NLM_F_CREATE should be specified when creating new route\n"); 3732 table = fib6_new_table(net, cfg->fc_table); 3733 } 3734 } else { 3735 table = fib6_new_table(net, cfg->fc_table); 3736 } 3737 3738 if (!table) 3739 goto out; 3740 3741 err = -ENOMEM; 3742 rt = fib6_info_alloc(gfp_flags, !nh); 3743 if (!rt) 3744 goto out; 3745 3746 rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len, 3747 extack); 3748 if (IS_ERR(rt->fib6_metrics)) { 3749 err = PTR_ERR(rt->fib6_metrics); 3750 /* Do not leave garbage there. */ 3751 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics; 3752 goto out_free; 3753 } 3754 3755 if (cfg->fc_flags & RTF_ADDRCONF) 3756 rt->dst_nocount = true; 3757 3758 if (cfg->fc_flags & RTF_EXPIRES) 3759 fib6_set_expires(rt, jiffies + 3760 clock_t_to_jiffies(cfg->fc_expires)); 3761 else 3762 fib6_clean_expires(rt); 3763 3764 if (cfg->fc_protocol == RTPROT_UNSPEC) 3765 cfg->fc_protocol = RTPROT_BOOT; 3766 rt->fib6_protocol = cfg->fc_protocol; 3767 3768 rt->fib6_table = table; 3769 rt->fib6_metric = cfg->fc_metric; 3770 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST; 3771 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY; 3772 3773 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len); 3774 rt->fib6_dst.plen = cfg->fc_dst_len; 3775 3776 #ifdef CONFIG_IPV6_SUBTREES 3777 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len); 3778 rt->fib6_src.plen = cfg->fc_src_len; 3779 #endif 3780 if (nh) { 3781 if (rt->fib6_src.plen) { 3782 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing"); 3783 goto out_free; 3784 } 3785 if (!nexthop_get(nh)) { 3786 NL_SET_ERR_MSG(extack, "Nexthop has been deleted"); 3787 goto out_free; 3788 } 3789 rt->nh = nh; 3790 fib6_nh = nexthop_fib6_nh(rt->nh); 3791 } else { 3792 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack); 3793 if (err) 3794 goto out; 3795 3796 fib6_nh = rt->fib6_nh; 3797 3798 /* We cannot add true routes via loopback here, they would 3799 * result in kernel looping; promote them to reject routes 3800 */ 3801 addr_type = ipv6_addr_type(&cfg->fc_dst); 3802 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev, 3803 addr_type)) 3804 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP; 3805 } 3806 3807 if (!ipv6_addr_any(&cfg->fc_prefsrc)) { 3808 struct net_device *dev = fib6_nh->fib_nh_dev; 3809 3810 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) { 3811 NL_SET_ERR_MSG(extack, "Invalid source address"); 3812 err = -EINVAL; 3813 goto out; 3814 } 3815 rt->fib6_prefsrc.addr = cfg->fc_prefsrc; 3816 rt->fib6_prefsrc.plen = 128; 3817 } else 3818 rt->fib6_prefsrc.plen = 0; 3819 3820 return rt; 3821 out: 3822 fib6_info_release(rt); 3823 return ERR_PTR(err); 3824 out_free: 3825 ip_fib_metrics_put(rt->fib6_metrics); 3826 kfree(rt); 3827 return ERR_PTR(err); 3828 } 3829 3830 int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags, 3831 struct netlink_ext_ack *extack) 3832 { 3833 struct fib6_info *rt; 3834 int err; 3835 3836 rt = ip6_route_info_create(cfg, gfp_flags, extack); 3837 if (IS_ERR(rt)) 3838 return PTR_ERR(rt); 3839 3840 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack); 3841 fib6_info_release(rt); 3842 3843 return err; 3844 } 3845 3846 static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info) 3847 { 3848 struct net *net = info->nl_net; 3849 struct fib6_table *table; 3850 int err; 3851 3852 if (rt == net->ipv6.fib6_null_entry) { 3853 err = -ENOENT; 3854 goto out; 3855 } 3856 3857 table = rt->fib6_table; 3858 spin_lock_bh(&table->tb6_lock); 3859 err = fib6_del(rt, info); 3860 spin_unlock_bh(&table->tb6_lock); 3861 3862 out: 3863 fib6_info_release(rt); 3864 return err; 3865 } 3866 3867 int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify) 3868 { 3869 struct nl_info info = { 3870 .nl_net = net, 3871 .skip_notify = skip_notify 3872 }; 3873 3874 return __ip6_del_rt(rt, &info); 3875 } 3876 3877 static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) 3878 { 3879 struct nl_info *info = &cfg->fc_nlinfo; 3880 struct net *net = info->nl_net; 3881 struct sk_buff *skb = NULL; 3882 struct fib6_table *table; 3883 int err = -ENOENT; 3884 3885 if (rt == net->ipv6.fib6_null_entry) 3886 goto out_put; 3887 table = rt->fib6_table; 3888 spin_lock_bh(&table->tb6_lock); 3889 3890 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { 3891 struct fib6_info *sibling, *next_sibling; 3892 struct fib6_node *fn; 3893 3894 /* prefer to send a single notification with all hops */ 3895 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 3896 if (skb) { 3897 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 3898 3899 if (rt6_fill_node(net, skb, rt, NULL, 3900 NULL, NULL, 0, RTM_DELROUTE, 3901 info->portid, seq, 0) < 0) { 3902 kfree_skb(skb); 3903 skb = NULL; 3904 } else 3905 info->skip_notify = 1; 3906 } 3907 3908 /* 'rt' points to the first sibling route. If it is not the 3909 * leaf, then we do not need to send a notification. Otherwise, 3910 * we need to check if the last sibling has a next route or not 3911 * and emit a replace or delete notification, respectively. 3912 */ 3913 info->skip_notify_kernel = 1; 3914 fn = rcu_dereference_protected(rt->fib6_node, 3915 lockdep_is_held(&table->tb6_lock)); 3916 if (rcu_access_pointer(fn->leaf) == rt) { 3917 struct fib6_info *last_sibling, *replace_rt; 3918 3919 last_sibling = list_last_entry(&rt->fib6_siblings, 3920 struct fib6_info, 3921 fib6_siblings); 3922 replace_rt = rcu_dereference_protected( 3923 last_sibling->fib6_next, 3924 lockdep_is_held(&table->tb6_lock)); 3925 if (replace_rt) 3926 call_fib6_entry_notifiers_replace(net, 3927 replace_rt); 3928 else 3929 call_fib6_multipath_entry_notifiers(net, 3930 FIB_EVENT_ENTRY_DEL, 3931 rt, rt->fib6_nsiblings, 3932 NULL); 3933 } 3934 list_for_each_entry_safe(sibling, next_sibling, 3935 &rt->fib6_siblings, 3936 fib6_siblings) { 3937 err = fib6_del(sibling, info); 3938 if (err) 3939 goto out_unlock; 3940 } 3941 } 3942 3943 err = fib6_del(rt, info); 3944 out_unlock: 3945 spin_unlock_bh(&table->tb6_lock); 3946 out_put: 3947 fib6_info_release(rt); 3948 3949 if (skb) { 3950 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 3951 info->nlh, gfp_any()); 3952 } 3953 return err; 3954 } 3955 3956 static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg) 3957 { 3958 int rc = -ESRCH; 3959 3960 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex) 3961 goto out; 3962 3963 if (cfg->fc_flags & RTF_GATEWAY && 3964 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway)) 3965 goto out; 3966 3967 rc = rt6_remove_exception_rt(rt); 3968 out: 3969 return rc; 3970 } 3971 3972 static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt, 3973 struct fib6_nh *nh) 3974 { 3975 struct fib6_result res = { 3976 .f6i = rt, 3977 .nh = nh, 3978 }; 3979 struct rt6_info *rt_cache; 3980 3981 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src); 3982 if (rt_cache) 3983 return __ip6_del_cached_rt(rt_cache, cfg); 3984 3985 return 0; 3986 } 3987 3988 struct fib6_nh_del_cached_rt_arg { 3989 struct fib6_config *cfg; 3990 struct fib6_info *f6i; 3991 }; 3992 3993 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg) 3994 { 3995 struct fib6_nh_del_cached_rt_arg *arg = _arg; 3996 int rc; 3997 3998 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh); 3999 return rc != -ESRCH ? rc : 0; 4000 } 4001 4002 static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i) 4003 { 4004 struct fib6_nh_del_cached_rt_arg arg = { 4005 .cfg = cfg, 4006 .f6i = f6i 4007 }; 4008 4009 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg); 4010 } 4011 4012 static int ip6_route_del(struct fib6_config *cfg, 4013 struct netlink_ext_ack *extack) 4014 { 4015 struct fib6_table *table; 4016 struct fib6_info *rt; 4017 struct fib6_node *fn; 4018 int err = -ESRCH; 4019 4020 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table); 4021 if (!table) { 4022 NL_SET_ERR_MSG(extack, "FIB table does not exist"); 4023 return err; 4024 } 4025 4026 rcu_read_lock(); 4027 4028 fn = fib6_locate(&table->tb6_root, 4029 &cfg->fc_dst, cfg->fc_dst_len, 4030 &cfg->fc_src, cfg->fc_src_len, 4031 !(cfg->fc_flags & RTF_CACHE)); 4032 4033 if (fn) { 4034 for_each_fib6_node_rt_rcu(fn) { 4035 struct fib6_nh *nh; 4036 4037 if (rt->nh && cfg->fc_nh_id && 4038 rt->nh->id != cfg->fc_nh_id) 4039 continue; 4040 4041 if (cfg->fc_flags & RTF_CACHE) { 4042 int rc = 0; 4043 4044 if (rt->nh) { 4045 rc = ip6_del_cached_rt_nh(cfg, rt); 4046 } else if (cfg->fc_nh_id) { 4047 continue; 4048 } else { 4049 nh = rt->fib6_nh; 4050 rc = ip6_del_cached_rt(cfg, rt, nh); 4051 } 4052 if (rc != -ESRCH) { 4053 rcu_read_unlock(); 4054 return rc; 4055 } 4056 continue; 4057 } 4058 4059 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric) 4060 continue; 4061 if (cfg->fc_protocol && 4062 cfg->fc_protocol != rt->fib6_protocol) 4063 continue; 4064 4065 if (rt->nh) { 4066 if (!fib6_info_hold_safe(rt)) 4067 continue; 4068 rcu_read_unlock(); 4069 4070 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4071 } 4072 if (cfg->fc_nh_id) 4073 continue; 4074 4075 nh = rt->fib6_nh; 4076 if (cfg->fc_ifindex && 4077 (!nh->fib_nh_dev || 4078 nh->fib_nh_dev->ifindex != cfg->fc_ifindex)) 4079 continue; 4080 if (cfg->fc_flags & RTF_GATEWAY && 4081 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6)) 4082 continue; 4083 if (!fib6_info_hold_safe(rt)) 4084 continue; 4085 rcu_read_unlock(); 4086 4087 /* if gateway was specified only delete the one hop */ 4088 if (cfg->fc_flags & RTF_GATEWAY) 4089 return __ip6_del_rt(rt, &cfg->fc_nlinfo); 4090 4091 return __ip6_del_rt_siblings(rt, cfg); 4092 } 4093 } 4094 rcu_read_unlock(); 4095 4096 return err; 4097 } 4098 4099 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb) 4100 { 4101 struct netevent_redirect netevent; 4102 struct rt6_info *rt, *nrt = NULL; 4103 struct fib6_result res = {}; 4104 struct ndisc_options ndopts; 4105 struct inet6_dev *in6_dev; 4106 struct neighbour *neigh; 4107 struct rd_msg *msg; 4108 int optlen, on_link; 4109 u8 *lladdr; 4110 4111 optlen = skb_tail_pointer(skb) - skb_transport_header(skb); 4112 optlen -= sizeof(*msg); 4113 4114 if (optlen < 0) { 4115 net_dbg_ratelimited("rt6_do_redirect: packet too short\n"); 4116 return; 4117 } 4118 4119 msg = (struct rd_msg *)icmp6_hdr(skb); 4120 4121 if (ipv6_addr_is_multicast(&msg->dest)) { 4122 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n"); 4123 return; 4124 } 4125 4126 on_link = 0; 4127 if (ipv6_addr_equal(&msg->dest, &msg->target)) { 4128 on_link = 1; 4129 } else if (ipv6_addr_type(&msg->target) != 4130 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) { 4131 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n"); 4132 return; 4133 } 4134 4135 in6_dev = __in6_dev_get(skb->dev); 4136 if (!in6_dev) 4137 return; 4138 if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects) 4139 return; 4140 4141 /* RFC2461 8.1: 4142 * The IP source address of the Redirect MUST be the same as the current 4143 * first-hop router for the specified ICMP Destination Address. 4144 */ 4145 4146 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) { 4147 net_dbg_ratelimited("rt6_redirect: invalid ND options\n"); 4148 return; 4149 } 4150 4151 lladdr = NULL; 4152 if (ndopts.nd_opts_tgt_lladdr) { 4153 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, 4154 skb->dev); 4155 if (!lladdr) { 4156 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n"); 4157 return; 4158 } 4159 } 4160 4161 rt = (struct rt6_info *) dst; 4162 if (rt->rt6i_flags & RTF_REJECT) { 4163 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n"); 4164 return; 4165 } 4166 4167 /* Redirect received -> path was valid. 4168 * Look, redirects are sent only in response to data packets, 4169 * so that this nexthop apparently is reachable. --ANK 4170 */ 4171 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr); 4172 4173 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1); 4174 if (!neigh) 4175 return; 4176 4177 /* 4178 * We have finally decided to accept it. 4179 */ 4180 4181 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE, 4182 NEIGH_UPDATE_F_WEAK_OVERRIDE| 4183 NEIGH_UPDATE_F_OVERRIDE| 4184 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER| 4185 NEIGH_UPDATE_F_ISROUTER)), 4186 NDISC_REDIRECT, &ndopts); 4187 4188 rcu_read_lock(); 4189 res.f6i = rcu_dereference(rt->from); 4190 if (!res.f6i) 4191 goto out; 4192 4193 if (res.f6i->nh) { 4194 struct fib6_nh_match_arg arg = { 4195 .dev = dst->dev, 4196 .gw = &rt->rt6i_gateway, 4197 }; 4198 4199 nexthop_for_each_fib6_nh(res.f6i->nh, 4200 fib6_nh_find_match, &arg); 4201 4202 /* fib6_info uses a nexthop that does not have fib6_nh 4203 * using the dst->dev. Should be impossible 4204 */ 4205 if (!arg.match) 4206 goto out; 4207 res.nh = arg.match; 4208 } else { 4209 res.nh = res.f6i->fib6_nh; 4210 } 4211 4212 res.fib6_flags = res.f6i->fib6_flags; 4213 res.fib6_type = res.f6i->fib6_type; 4214 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL); 4215 if (!nrt) 4216 goto out; 4217 4218 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE; 4219 if (on_link) 4220 nrt->rt6i_flags &= ~RTF_GATEWAY; 4221 4222 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key; 4223 4224 /* rt6_insert_exception() will take care of duplicated exceptions */ 4225 if (rt6_insert_exception(nrt, &res)) { 4226 dst_release_immediate(&nrt->dst); 4227 goto out; 4228 } 4229 4230 netevent.old = &rt->dst; 4231 netevent.new = &nrt->dst; 4232 netevent.daddr = &msg->dest; 4233 netevent.neigh = neigh; 4234 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent); 4235 4236 out: 4237 rcu_read_unlock(); 4238 neigh_release(neigh); 4239 } 4240 4241 #ifdef CONFIG_IPV6_ROUTE_INFO 4242 static struct fib6_info *rt6_get_route_info(struct net *net, 4243 const struct in6_addr *prefix, int prefixlen, 4244 const struct in6_addr *gwaddr, 4245 struct net_device *dev) 4246 { 4247 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4248 int ifindex = dev->ifindex; 4249 struct fib6_node *fn; 4250 struct fib6_info *rt = NULL; 4251 struct fib6_table *table; 4252 4253 table = fib6_get_table(net, tb_id); 4254 if (!table) 4255 return NULL; 4256 4257 rcu_read_lock(); 4258 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true); 4259 if (!fn) 4260 goto out; 4261 4262 for_each_fib6_node_rt_rcu(fn) { 4263 /* these routes do not use nexthops */ 4264 if (rt->nh) 4265 continue; 4266 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex) 4267 continue; 4268 if (!(rt->fib6_flags & RTF_ROUTEINFO) || 4269 !rt->fib6_nh->fib_nh_gw_family) 4270 continue; 4271 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr)) 4272 continue; 4273 if (!fib6_info_hold_safe(rt)) 4274 continue; 4275 break; 4276 } 4277 out: 4278 rcu_read_unlock(); 4279 return rt; 4280 } 4281 4282 static struct fib6_info *rt6_add_route_info(struct net *net, 4283 const struct in6_addr *prefix, int prefixlen, 4284 const struct in6_addr *gwaddr, 4285 struct net_device *dev, 4286 unsigned int pref) 4287 { 4288 struct fib6_config cfg = { 4289 .fc_metric = IP6_RT_PRIO_USER, 4290 .fc_ifindex = dev->ifindex, 4291 .fc_dst_len = prefixlen, 4292 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO | 4293 RTF_UP | RTF_PREF(pref), 4294 .fc_protocol = RTPROT_RA, 4295 .fc_type = RTN_UNICAST, 4296 .fc_nlinfo.portid = 0, 4297 .fc_nlinfo.nlh = NULL, 4298 .fc_nlinfo.nl_net = net, 4299 }; 4300 4301 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; 4302 cfg.fc_dst = *prefix; 4303 cfg.fc_gateway = *gwaddr; 4304 4305 /* We should treat it as a default route if prefix length is 0. */ 4306 if (!prefixlen) 4307 cfg.fc_flags |= RTF_DEFAULT; 4308 4309 ip6_route_add(&cfg, GFP_ATOMIC, NULL); 4310 4311 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev); 4312 } 4313 #endif 4314 4315 struct fib6_info *rt6_get_dflt_router(struct net *net, 4316 const struct in6_addr *addr, 4317 struct net_device *dev) 4318 { 4319 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; 4320 struct fib6_info *rt; 4321 struct fib6_table *table; 4322 4323 table = fib6_get_table(net, tb_id); 4324 if (!table) 4325 return NULL; 4326 4327 rcu_read_lock(); 4328 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4329 struct fib6_nh *nh; 4330 4331 /* RA routes do not use nexthops */ 4332 if (rt->nh) 4333 continue; 4334 4335 nh = rt->fib6_nh; 4336 if (dev == nh->fib_nh_dev && 4337 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) && 4338 ipv6_addr_equal(&nh->fib_nh_gw6, addr)) 4339 break; 4340 } 4341 if (rt && !fib6_info_hold_safe(rt)) 4342 rt = NULL; 4343 rcu_read_unlock(); 4344 return rt; 4345 } 4346 4347 struct fib6_info *rt6_add_dflt_router(struct net *net, 4348 const struct in6_addr *gwaddr, 4349 struct net_device *dev, 4350 unsigned int pref, 4351 u32 defrtr_usr_metric) 4352 { 4353 struct fib6_config cfg = { 4354 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, 4355 .fc_metric = defrtr_usr_metric, 4356 .fc_ifindex = dev->ifindex, 4357 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | 4358 RTF_UP | RTF_EXPIRES | RTF_PREF(pref), 4359 .fc_protocol = RTPROT_RA, 4360 .fc_type = RTN_UNICAST, 4361 .fc_nlinfo.portid = 0, 4362 .fc_nlinfo.nlh = NULL, 4363 .fc_nlinfo.nl_net = net, 4364 }; 4365 4366 cfg.fc_gateway = *gwaddr; 4367 4368 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) { 4369 struct fib6_table *table; 4370 4371 table = fib6_get_table(dev_net(dev), cfg.fc_table); 4372 if (table) 4373 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER; 4374 } 4375 4376 return rt6_get_dflt_router(net, gwaddr, dev); 4377 } 4378 4379 static void __rt6_purge_dflt_routers(struct net *net, 4380 struct fib6_table *table) 4381 { 4382 struct fib6_info *rt; 4383 4384 restart: 4385 rcu_read_lock(); 4386 for_each_fib6_node_rt_rcu(&table->tb6_root) { 4387 struct net_device *dev = fib6_info_nh_dev(rt); 4388 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL; 4389 4390 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) && 4391 (!idev || idev->cnf.accept_ra != 2) && 4392 fib6_info_hold_safe(rt)) { 4393 rcu_read_unlock(); 4394 ip6_del_rt(net, rt, false); 4395 goto restart; 4396 } 4397 } 4398 rcu_read_unlock(); 4399 4400 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; 4401 } 4402 4403 void rt6_purge_dflt_routers(struct net *net) 4404 { 4405 struct fib6_table *table; 4406 struct hlist_head *head; 4407 unsigned int h; 4408 4409 rcu_read_lock(); 4410 4411 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { 4412 head = &net->ipv6.fib_table_hash[h]; 4413 hlist_for_each_entry_rcu(table, head, tb6_hlist) { 4414 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) 4415 __rt6_purge_dflt_routers(net, table); 4416 } 4417 } 4418 4419 rcu_read_unlock(); 4420 } 4421 4422 static void rtmsg_to_fib6_config(struct net *net, 4423 struct in6_rtmsg *rtmsg, 4424 struct fib6_config *cfg) 4425 { 4426 *cfg = (struct fib6_config){ 4427 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ? 4428 : RT6_TABLE_MAIN, 4429 .fc_ifindex = rtmsg->rtmsg_ifindex, 4430 .fc_metric = rtmsg->rtmsg_metric ? : IP6_RT_PRIO_USER, 4431 .fc_expires = rtmsg->rtmsg_info, 4432 .fc_dst_len = rtmsg->rtmsg_dst_len, 4433 .fc_src_len = rtmsg->rtmsg_src_len, 4434 .fc_flags = rtmsg->rtmsg_flags, 4435 .fc_type = rtmsg->rtmsg_type, 4436 4437 .fc_nlinfo.nl_net = net, 4438 4439 .fc_dst = rtmsg->rtmsg_dst, 4440 .fc_src = rtmsg->rtmsg_src, 4441 .fc_gateway = rtmsg->rtmsg_gateway, 4442 }; 4443 } 4444 4445 int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg) 4446 { 4447 struct fib6_config cfg; 4448 int err; 4449 4450 if (cmd != SIOCADDRT && cmd != SIOCDELRT) 4451 return -EINVAL; 4452 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 4453 return -EPERM; 4454 4455 rtmsg_to_fib6_config(net, rtmsg, &cfg); 4456 4457 rtnl_lock(); 4458 switch (cmd) { 4459 case SIOCADDRT: 4460 err = ip6_route_add(&cfg, GFP_KERNEL, NULL); 4461 break; 4462 case SIOCDELRT: 4463 err = ip6_route_del(&cfg, NULL); 4464 break; 4465 } 4466 rtnl_unlock(); 4467 return err; 4468 } 4469 4470 /* 4471 * Drop the packet on the floor 4472 */ 4473 4474 static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes) 4475 { 4476 struct dst_entry *dst = skb_dst(skb); 4477 struct net *net = dev_net(dst->dev); 4478 struct inet6_dev *idev; 4479 SKB_DR(reason); 4480 int type; 4481 4482 if (netif_is_l3_master(skb->dev) || 4483 dst->dev == net->loopback_dev) 4484 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); 4485 else 4486 idev = ip6_dst_idev(dst); 4487 4488 switch (ipstats_mib_noroutes) { 4489 case IPSTATS_MIB_INNOROUTES: 4490 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr); 4491 if (type == IPV6_ADDR_ANY) { 4492 SKB_DR_SET(reason, IP_INADDRERRORS); 4493 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS); 4494 break; 4495 } 4496 SKB_DR_SET(reason, IP_INNOROUTES); 4497 fallthrough; 4498 case IPSTATS_MIB_OUTNOROUTES: 4499 SKB_DR_OR(reason, IP_OUTNOROUTES); 4500 IP6_INC_STATS(net, idev, ipstats_mib_noroutes); 4501 break; 4502 } 4503 4504 /* Start over by dropping the dst for l3mdev case */ 4505 if (netif_is_l3_master(skb->dev)) 4506 skb_dst_drop(skb); 4507 4508 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0); 4509 kfree_skb_reason(skb, reason); 4510 return 0; 4511 } 4512 4513 static int ip6_pkt_discard(struct sk_buff *skb) 4514 { 4515 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES); 4516 } 4517 4518 static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4519 { 4520 skb->dev = skb_dst(skb)->dev; 4521 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES); 4522 } 4523 4524 static int ip6_pkt_prohibit(struct sk_buff *skb) 4525 { 4526 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES); 4527 } 4528 4529 static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb) 4530 { 4531 skb->dev = skb_dst(skb)->dev; 4532 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES); 4533 } 4534 4535 /* 4536 * Allocate a dst for local (unicast / anycast) address. 4537 */ 4538 4539 struct fib6_info *addrconf_f6i_alloc(struct net *net, 4540 struct inet6_dev *idev, 4541 const struct in6_addr *addr, 4542 bool anycast, gfp_t gfp_flags) 4543 { 4544 struct fib6_config cfg = { 4545 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL, 4546 .fc_ifindex = idev->dev->ifindex, 4547 .fc_flags = RTF_UP | RTF_NONEXTHOP, 4548 .fc_dst = *addr, 4549 .fc_dst_len = 128, 4550 .fc_protocol = RTPROT_KERNEL, 4551 .fc_nlinfo.nl_net = net, 4552 .fc_ignore_dev_down = true, 4553 }; 4554 struct fib6_info *f6i; 4555 4556 if (anycast) { 4557 cfg.fc_type = RTN_ANYCAST; 4558 cfg.fc_flags |= RTF_ANYCAST; 4559 } else { 4560 cfg.fc_type = RTN_LOCAL; 4561 cfg.fc_flags |= RTF_LOCAL; 4562 } 4563 4564 f6i = ip6_route_info_create(&cfg, gfp_flags, NULL); 4565 if (!IS_ERR(f6i)) { 4566 f6i->dst_nocount = true; 4567 4568 if (!anycast && 4569 (net->ipv6.devconf_all->disable_policy || 4570 idev->cnf.disable_policy)) 4571 f6i->dst_nopolicy = true; 4572 } 4573 4574 return f6i; 4575 } 4576 4577 /* remove deleted ip from prefsrc entries */ 4578 struct arg_dev_net_ip { 4579 struct net_device *dev; 4580 struct net *net; 4581 struct in6_addr *addr; 4582 }; 4583 4584 static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg) 4585 { 4586 struct net_device *dev = ((struct arg_dev_net_ip *)arg)->dev; 4587 struct net *net = ((struct arg_dev_net_ip *)arg)->net; 4588 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr; 4589 4590 if (!rt->nh && 4591 ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) && 4592 rt != net->ipv6.fib6_null_entry && 4593 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr)) { 4594 spin_lock_bh(&rt6_exception_lock); 4595 /* remove prefsrc entry */ 4596 rt->fib6_prefsrc.plen = 0; 4597 spin_unlock_bh(&rt6_exception_lock); 4598 } 4599 return 0; 4600 } 4601 4602 void rt6_remove_prefsrc(struct inet6_ifaddr *ifp) 4603 { 4604 struct net *net = dev_net(ifp->idev->dev); 4605 struct arg_dev_net_ip adni = { 4606 .dev = ifp->idev->dev, 4607 .net = net, 4608 .addr = &ifp->addr, 4609 }; 4610 fib6_clean_all(net, fib6_remove_prefsrc, &adni); 4611 } 4612 4613 #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT) 4614 4615 /* Remove routers and update dst entries when gateway turn into host. */ 4616 static int fib6_clean_tohost(struct fib6_info *rt, void *arg) 4617 { 4618 struct in6_addr *gateway = (struct in6_addr *)arg; 4619 struct fib6_nh *nh; 4620 4621 /* RA routes do not use nexthops */ 4622 if (rt->nh) 4623 return 0; 4624 4625 nh = rt->fib6_nh; 4626 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) && 4627 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6)) 4628 return -1; 4629 4630 /* Further clean up cached routes in exception table. 4631 * This is needed because cached route may have a different 4632 * gateway than its 'parent' in the case of an ip redirect. 4633 */ 4634 fib6_nh_exceptions_clean_tohost(nh, gateway); 4635 4636 return 0; 4637 } 4638 4639 void rt6_clean_tohost(struct net *net, struct in6_addr *gateway) 4640 { 4641 fib6_clean_all(net, fib6_clean_tohost, gateway); 4642 } 4643 4644 struct arg_netdev_event { 4645 const struct net_device *dev; 4646 union { 4647 unsigned char nh_flags; 4648 unsigned long event; 4649 }; 4650 }; 4651 4652 static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt) 4653 { 4654 struct fib6_info *iter; 4655 struct fib6_node *fn; 4656 4657 fn = rcu_dereference_protected(rt->fib6_node, 4658 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4659 iter = rcu_dereference_protected(fn->leaf, 4660 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4661 while (iter) { 4662 if (iter->fib6_metric == rt->fib6_metric && 4663 rt6_qualify_for_ecmp(iter)) 4664 return iter; 4665 iter = rcu_dereference_protected(iter->fib6_next, 4666 lockdep_is_held(&rt->fib6_table->tb6_lock)); 4667 } 4668 4669 return NULL; 4670 } 4671 4672 /* only called for fib entries with builtin fib6_nh */ 4673 static bool rt6_is_dead(const struct fib6_info *rt) 4674 { 4675 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD || 4676 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN && 4677 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev))) 4678 return true; 4679 4680 return false; 4681 } 4682 4683 static int rt6_multipath_total_weight(const struct fib6_info *rt) 4684 { 4685 struct fib6_info *iter; 4686 int total = 0; 4687 4688 if (!rt6_is_dead(rt)) 4689 total += rt->fib6_nh->fib_nh_weight; 4690 4691 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 4692 if (!rt6_is_dead(iter)) 4693 total += iter->fib6_nh->fib_nh_weight; 4694 } 4695 4696 return total; 4697 } 4698 4699 static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) 4700 { 4701 int upper_bound = -1; 4702 4703 if (!rt6_is_dead(rt)) { 4704 *weight += rt->fib6_nh->fib_nh_weight; 4705 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, 4706 total) - 1; 4707 } 4708 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound); 4709 } 4710 4711 static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total) 4712 { 4713 struct fib6_info *iter; 4714 int weight = 0; 4715 4716 rt6_upper_bound_set(rt, &weight, total); 4717 4718 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4719 rt6_upper_bound_set(iter, &weight, total); 4720 } 4721 4722 void rt6_multipath_rebalance(struct fib6_info *rt) 4723 { 4724 struct fib6_info *first; 4725 int total; 4726 4727 /* In case the entire multipath route was marked for flushing, 4728 * then there is no need to rebalance upon the removal of every 4729 * sibling route. 4730 */ 4731 if (!rt->fib6_nsiblings || rt->should_flush) 4732 return; 4733 4734 /* During lookup routes are evaluated in order, so we need to 4735 * make sure upper bounds are assigned from the first sibling 4736 * onwards. 4737 */ 4738 first = rt6_multipath_first_sibling(rt); 4739 if (WARN_ON_ONCE(!first)) 4740 return; 4741 4742 total = rt6_multipath_total_weight(first); 4743 rt6_multipath_upper_bound_set(first, total); 4744 } 4745 4746 static int fib6_ifup(struct fib6_info *rt, void *p_arg) 4747 { 4748 const struct arg_netdev_event *arg = p_arg; 4749 struct net *net = dev_net(arg->dev); 4750 4751 if (rt != net->ipv6.fib6_null_entry && !rt->nh && 4752 rt->fib6_nh->fib_nh_dev == arg->dev) { 4753 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags; 4754 fib6_update_sernum_upto_root(net, rt); 4755 rt6_multipath_rebalance(rt); 4756 } 4757 4758 return 0; 4759 } 4760 4761 void rt6_sync_up(struct net_device *dev, unsigned char nh_flags) 4762 { 4763 struct arg_netdev_event arg = { 4764 .dev = dev, 4765 { 4766 .nh_flags = nh_flags, 4767 }, 4768 }; 4769 4770 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev)) 4771 arg.nh_flags |= RTNH_F_LINKDOWN; 4772 4773 fib6_clean_all(dev_net(dev), fib6_ifup, &arg); 4774 } 4775 4776 /* only called for fib entries with inline fib6_nh */ 4777 static bool rt6_multipath_uses_dev(const struct fib6_info *rt, 4778 const struct net_device *dev) 4779 { 4780 struct fib6_info *iter; 4781 4782 if (rt->fib6_nh->fib_nh_dev == dev) 4783 return true; 4784 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4785 if (iter->fib6_nh->fib_nh_dev == dev) 4786 return true; 4787 4788 return false; 4789 } 4790 4791 static void rt6_multipath_flush(struct fib6_info *rt) 4792 { 4793 struct fib6_info *iter; 4794 4795 rt->should_flush = 1; 4796 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4797 iter->should_flush = 1; 4798 } 4799 4800 static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt, 4801 const struct net_device *down_dev) 4802 { 4803 struct fib6_info *iter; 4804 unsigned int dead = 0; 4805 4806 if (rt->fib6_nh->fib_nh_dev == down_dev || 4807 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4808 dead++; 4809 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4810 if (iter->fib6_nh->fib_nh_dev == down_dev || 4811 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD) 4812 dead++; 4813 4814 return dead; 4815 } 4816 4817 static void rt6_multipath_nh_flags_set(struct fib6_info *rt, 4818 const struct net_device *dev, 4819 unsigned char nh_flags) 4820 { 4821 struct fib6_info *iter; 4822 4823 if (rt->fib6_nh->fib_nh_dev == dev) 4824 rt->fib6_nh->fib_nh_flags |= nh_flags; 4825 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) 4826 if (iter->fib6_nh->fib_nh_dev == dev) 4827 iter->fib6_nh->fib_nh_flags |= nh_flags; 4828 } 4829 4830 /* called with write lock held for table with rt */ 4831 static int fib6_ifdown(struct fib6_info *rt, void *p_arg) 4832 { 4833 const struct arg_netdev_event *arg = p_arg; 4834 const struct net_device *dev = arg->dev; 4835 struct net *net = dev_net(dev); 4836 4837 if (rt == net->ipv6.fib6_null_entry || rt->nh) 4838 return 0; 4839 4840 switch (arg->event) { 4841 case NETDEV_UNREGISTER: 4842 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4843 case NETDEV_DOWN: 4844 if (rt->should_flush) 4845 return -1; 4846 if (!rt->fib6_nsiblings) 4847 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0; 4848 if (rt6_multipath_uses_dev(rt, dev)) { 4849 unsigned int count; 4850 4851 count = rt6_multipath_dead_count(rt, dev); 4852 if (rt->fib6_nsiblings + 1 == count) { 4853 rt6_multipath_flush(rt); 4854 return -1; 4855 } 4856 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD | 4857 RTNH_F_LINKDOWN); 4858 fib6_update_sernum(net, rt); 4859 rt6_multipath_rebalance(rt); 4860 } 4861 return -2; 4862 case NETDEV_CHANGE: 4863 if (rt->fib6_nh->fib_nh_dev != dev || 4864 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 4865 break; 4866 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN; 4867 rt6_multipath_rebalance(rt); 4868 break; 4869 } 4870 4871 return 0; 4872 } 4873 4874 void rt6_sync_down_dev(struct net_device *dev, unsigned long event) 4875 { 4876 struct arg_netdev_event arg = { 4877 .dev = dev, 4878 { 4879 .event = event, 4880 }, 4881 }; 4882 struct net *net = dev_net(dev); 4883 4884 if (net->ipv6.sysctl.skip_notify_on_dev_down) 4885 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); 4886 else 4887 fib6_clean_all(net, fib6_ifdown, &arg); 4888 } 4889 4890 void rt6_disable_ip(struct net_device *dev, unsigned long event) 4891 { 4892 rt6_sync_down_dev(dev, event); 4893 rt6_uncached_list_flush_dev(dev); 4894 neigh_ifdown(&nd_tbl, dev); 4895 } 4896 4897 struct rt6_mtu_change_arg { 4898 struct net_device *dev; 4899 unsigned int mtu; 4900 struct fib6_info *f6i; 4901 }; 4902 4903 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) 4904 { 4905 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg; 4906 struct fib6_info *f6i = arg->f6i; 4907 4908 /* For administrative MTU increase, there is no way to discover 4909 * IPv6 PMTU increase, so PMTU increase should be updated here. 4910 * Since RFC 1981 doesn't include administrative MTU increase 4911 * update PMTU increase is a MUST. (i.e. jumbo frame) 4912 */ 4913 if (nh->fib_nh_dev == arg->dev) { 4914 struct inet6_dev *idev = __in6_dev_get(arg->dev); 4915 u32 mtu = f6i->fib6_pmtu; 4916 4917 if (mtu >= arg->mtu || 4918 (mtu < arg->mtu && mtu == idev->cnf.mtu6)) 4919 fib6_metric_set(f6i, RTAX_MTU, arg->mtu); 4920 4921 spin_lock_bh(&rt6_exception_lock); 4922 rt6_exceptions_update_pmtu(idev, nh, arg->mtu); 4923 spin_unlock_bh(&rt6_exception_lock); 4924 } 4925 4926 return 0; 4927 } 4928 4929 static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg) 4930 { 4931 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg; 4932 struct inet6_dev *idev; 4933 4934 /* In IPv6 pmtu discovery is not optional, 4935 so that RTAX_MTU lock cannot disable it. 4936 We still use this lock to block changes 4937 caused by addrconf/ndisc. 4938 */ 4939 4940 idev = __in6_dev_get(arg->dev); 4941 if (!idev) 4942 return 0; 4943 4944 if (fib6_metric_locked(f6i, RTAX_MTU)) 4945 return 0; 4946 4947 arg->f6i = f6i; 4948 if (f6i->nh) { 4949 /* fib6_nh_mtu_change only returns 0, so this is safe */ 4950 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change, 4951 arg); 4952 } 4953 4954 return fib6_nh_mtu_change(f6i->fib6_nh, arg); 4955 } 4956 4957 void rt6_mtu_change(struct net_device *dev, unsigned int mtu) 4958 { 4959 struct rt6_mtu_change_arg arg = { 4960 .dev = dev, 4961 .mtu = mtu, 4962 }; 4963 4964 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg); 4965 } 4966 4967 static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = { 4968 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 }, 4969 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) }, 4970 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) }, 4971 [RTA_OIF] = { .type = NLA_U32 }, 4972 [RTA_IIF] = { .type = NLA_U32 }, 4973 [RTA_PRIORITY] = { .type = NLA_U32 }, 4974 [RTA_METRICS] = { .type = NLA_NESTED }, 4975 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) }, 4976 [RTA_PREF] = { .type = NLA_U8 }, 4977 [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, 4978 [RTA_ENCAP] = { .type = NLA_NESTED }, 4979 [RTA_EXPIRES] = { .type = NLA_U32 }, 4980 [RTA_UID] = { .type = NLA_U32 }, 4981 [RTA_MARK] = { .type = NLA_U32 }, 4982 [RTA_TABLE] = { .type = NLA_U32 }, 4983 [RTA_IP_PROTO] = { .type = NLA_U8 }, 4984 [RTA_SPORT] = { .type = NLA_U16 }, 4985 [RTA_DPORT] = { .type = NLA_U16 }, 4986 [RTA_NH_ID] = { .type = NLA_U32 }, 4987 }; 4988 4989 static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, 4990 struct fib6_config *cfg, 4991 struct netlink_ext_ack *extack) 4992 { 4993 struct rtmsg *rtm; 4994 struct nlattr *tb[RTA_MAX+1]; 4995 unsigned int pref; 4996 int err; 4997 4998 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 4999 rtm_ipv6_policy, extack); 5000 if (err < 0) 5001 goto errout; 5002 5003 err = -EINVAL; 5004 rtm = nlmsg_data(nlh); 5005 5006 if (rtm->rtm_tos) { 5007 NL_SET_ERR_MSG(extack, 5008 "Invalid dsfield (tos): option not available for IPv6"); 5009 goto errout; 5010 } 5011 5012 *cfg = (struct fib6_config){ 5013 .fc_table = rtm->rtm_table, 5014 .fc_dst_len = rtm->rtm_dst_len, 5015 .fc_src_len = rtm->rtm_src_len, 5016 .fc_flags = RTF_UP, 5017 .fc_protocol = rtm->rtm_protocol, 5018 .fc_type = rtm->rtm_type, 5019 5020 .fc_nlinfo.portid = NETLINK_CB(skb).portid, 5021 .fc_nlinfo.nlh = nlh, 5022 .fc_nlinfo.nl_net = sock_net(skb->sk), 5023 }; 5024 5025 if (rtm->rtm_type == RTN_UNREACHABLE || 5026 rtm->rtm_type == RTN_BLACKHOLE || 5027 rtm->rtm_type == RTN_PROHIBIT || 5028 rtm->rtm_type == RTN_THROW) 5029 cfg->fc_flags |= RTF_REJECT; 5030 5031 if (rtm->rtm_type == RTN_LOCAL) 5032 cfg->fc_flags |= RTF_LOCAL; 5033 5034 if (rtm->rtm_flags & RTM_F_CLONED) 5035 cfg->fc_flags |= RTF_CACHE; 5036 5037 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK); 5038 5039 if (tb[RTA_NH_ID]) { 5040 if (tb[RTA_GATEWAY] || tb[RTA_OIF] || 5041 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) { 5042 NL_SET_ERR_MSG(extack, 5043 "Nexthop specification and nexthop id are mutually exclusive"); 5044 goto errout; 5045 } 5046 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]); 5047 } 5048 5049 if (tb[RTA_GATEWAY]) { 5050 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]); 5051 cfg->fc_flags |= RTF_GATEWAY; 5052 } 5053 if (tb[RTA_VIA]) { 5054 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute"); 5055 goto errout; 5056 } 5057 5058 if (tb[RTA_DST]) { 5059 int plen = (rtm->rtm_dst_len + 7) >> 3; 5060 5061 if (nla_len(tb[RTA_DST]) < plen) 5062 goto errout; 5063 5064 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen); 5065 } 5066 5067 if (tb[RTA_SRC]) { 5068 int plen = (rtm->rtm_src_len + 7) >> 3; 5069 5070 if (nla_len(tb[RTA_SRC]) < plen) 5071 goto errout; 5072 5073 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen); 5074 } 5075 5076 if (tb[RTA_PREFSRC]) 5077 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]); 5078 5079 if (tb[RTA_OIF]) 5080 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]); 5081 5082 if (tb[RTA_PRIORITY]) 5083 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]); 5084 5085 if (tb[RTA_METRICS]) { 5086 cfg->fc_mx = nla_data(tb[RTA_METRICS]); 5087 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]); 5088 } 5089 5090 if (tb[RTA_TABLE]) 5091 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]); 5092 5093 if (tb[RTA_MULTIPATH]) { 5094 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]); 5095 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]); 5096 5097 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp, 5098 cfg->fc_mp_len, extack); 5099 if (err < 0) 5100 goto errout; 5101 } 5102 5103 if (tb[RTA_PREF]) { 5104 pref = nla_get_u8(tb[RTA_PREF]); 5105 if (pref != ICMPV6_ROUTER_PREF_LOW && 5106 pref != ICMPV6_ROUTER_PREF_HIGH) 5107 pref = ICMPV6_ROUTER_PREF_MEDIUM; 5108 cfg->fc_flags |= RTF_PREF(pref); 5109 } 5110 5111 if (tb[RTA_ENCAP]) 5112 cfg->fc_encap = tb[RTA_ENCAP]; 5113 5114 if (tb[RTA_ENCAP_TYPE]) { 5115 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]); 5116 5117 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack); 5118 if (err < 0) 5119 goto errout; 5120 } 5121 5122 if (tb[RTA_EXPIRES]) { 5123 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ); 5124 5125 if (addrconf_finite_timeout(timeout)) { 5126 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ); 5127 cfg->fc_flags |= RTF_EXPIRES; 5128 } 5129 } 5130 5131 err = 0; 5132 errout: 5133 return err; 5134 } 5135 5136 struct rt6_nh { 5137 struct fib6_info *fib6_info; 5138 struct fib6_config r_cfg; 5139 struct list_head next; 5140 }; 5141 5142 static int ip6_route_info_append(struct net *net, 5143 struct list_head *rt6_nh_list, 5144 struct fib6_info *rt, 5145 struct fib6_config *r_cfg) 5146 { 5147 struct rt6_nh *nh; 5148 int err = -EEXIST; 5149 5150 list_for_each_entry(nh, rt6_nh_list, next) { 5151 /* check if fib6_info already exists */ 5152 if (rt6_duplicate_nexthop(nh->fib6_info, rt)) 5153 return err; 5154 } 5155 5156 nh = kzalloc(sizeof(*nh), GFP_KERNEL); 5157 if (!nh) 5158 return -ENOMEM; 5159 nh->fib6_info = rt; 5160 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg)); 5161 list_add_tail(&nh->next, rt6_nh_list); 5162 5163 return 0; 5164 } 5165 5166 static void ip6_route_mpath_notify(struct fib6_info *rt, 5167 struct fib6_info *rt_last, 5168 struct nl_info *info, 5169 __u16 nlflags) 5170 { 5171 /* if this is an APPEND route, then rt points to the first route 5172 * inserted and rt_last points to last route inserted. Userspace 5173 * wants a consistent dump of the route which starts at the first 5174 * nexthop. Since sibling routes are always added at the end of 5175 * the list, find the first sibling of the last route appended 5176 */ 5177 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) { 5178 rt = list_first_entry(&rt_last->fib6_siblings, 5179 struct fib6_info, 5180 fib6_siblings); 5181 } 5182 5183 if (rt) 5184 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags); 5185 } 5186 5187 static bool ip6_route_mpath_should_notify(const struct fib6_info *rt) 5188 { 5189 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt); 5190 bool should_notify = false; 5191 struct fib6_info *leaf; 5192 struct fib6_node *fn; 5193 5194 rcu_read_lock(); 5195 fn = rcu_dereference(rt->fib6_node); 5196 if (!fn) 5197 goto out; 5198 5199 leaf = rcu_dereference(fn->leaf); 5200 if (!leaf) 5201 goto out; 5202 5203 if (rt == leaf || 5204 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric && 5205 rt6_qualify_for_ecmp(leaf))) 5206 should_notify = true; 5207 out: 5208 rcu_read_unlock(); 5209 5210 return should_notify; 5211 } 5212 5213 static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla, 5214 struct netlink_ext_ack *extack) 5215 { 5216 if (nla_len(nla) < sizeof(*gw)) { 5217 NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY"); 5218 return -EINVAL; 5219 } 5220 5221 *gw = nla_get_in6_addr(nla); 5222 5223 return 0; 5224 } 5225 5226 static int ip6_route_multipath_add(struct fib6_config *cfg, 5227 struct netlink_ext_ack *extack) 5228 { 5229 struct fib6_info *rt_notif = NULL, *rt_last = NULL; 5230 struct nl_info *info = &cfg->fc_nlinfo; 5231 struct fib6_config r_cfg; 5232 struct rtnexthop *rtnh; 5233 struct fib6_info *rt; 5234 struct rt6_nh *err_nh; 5235 struct rt6_nh *nh, *nh_safe; 5236 __u16 nlflags; 5237 int remaining; 5238 int attrlen; 5239 int err = 1; 5240 int nhn = 0; 5241 int replace = (cfg->fc_nlinfo.nlh && 5242 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE)); 5243 LIST_HEAD(rt6_nh_list); 5244 5245 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE; 5246 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND) 5247 nlflags |= NLM_F_APPEND; 5248 5249 remaining = cfg->fc_mp_len; 5250 rtnh = (struct rtnexthop *)cfg->fc_mp; 5251 5252 /* Parse a Multipath Entry and build a list (rt6_nh_list) of 5253 * fib6_info structs per nexthop 5254 */ 5255 while (rtnh_ok(rtnh, remaining)) { 5256 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5257 if (rtnh->rtnh_ifindex) 5258 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5259 5260 attrlen = rtnh_attrlen(rtnh); 5261 if (attrlen > 0) { 5262 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5263 5264 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5265 if (nla) { 5266 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla, 5267 extack); 5268 if (err) 5269 goto cleanup; 5270 5271 r_cfg.fc_flags |= RTF_GATEWAY; 5272 } 5273 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP); 5274 5275 /* RTA_ENCAP_TYPE length checked in 5276 * lwtunnel_valid_encap_type_attr 5277 */ 5278 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); 5279 if (nla) 5280 r_cfg.fc_encap_type = nla_get_u16(nla); 5281 } 5282 5283 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK); 5284 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack); 5285 if (IS_ERR(rt)) { 5286 err = PTR_ERR(rt); 5287 rt = NULL; 5288 goto cleanup; 5289 } 5290 if (!rt6_qualify_for_ecmp(rt)) { 5291 err = -EINVAL; 5292 NL_SET_ERR_MSG(extack, 5293 "Device only routes can not be added for IPv6 using the multipath API."); 5294 fib6_info_release(rt); 5295 goto cleanup; 5296 } 5297 5298 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1; 5299 5300 err = ip6_route_info_append(info->nl_net, &rt6_nh_list, 5301 rt, &r_cfg); 5302 if (err) { 5303 fib6_info_release(rt); 5304 goto cleanup; 5305 } 5306 5307 rtnh = rtnh_next(rtnh, &remaining); 5308 } 5309 5310 if (list_empty(&rt6_nh_list)) { 5311 NL_SET_ERR_MSG(extack, 5312 "Invalid nexthop configuration - no valid nexthops"); 5313 return -EINVAL; 5314 } 5315 5316 /* for add and replace send one notification with all nexthops. 5317 * Skip the notification in fib6_add_rt2node and send one with 5318 * the full route when done 5319 */ 5320 info->skip_notify = 1; 5321 5322 /* For add and replace, send one notification with all nexthops. For 5323 * append, send one notification with all appended nexthops. 5324 */ 5325 info->skip_notify_kernel = 1; 5326 5327 err_nh = NULL; 5328 list_for_each_entry(nh, &rt6_nh_list, next) { 5329 err = __ip6_ins_rt(nh->fib6_info, info, extack); 5330 fib6_info_release(nh->fib6_info); 5331 5332 if (!err) { 5333 /* save reference to last route successfully inserted */ 5334 rt_last = nh->fib6_info; 5335 5336 /* save reference to first route for notification */ 5337 if (!rt_notif) 5338 rt_notif = nh->fib6_info; 5339 } 5340 5341 /* nh->fib6_info is used or freed at this point, reset to NULL*/ 5342 nh->fib6_info = NULL; 5343 if (err) { 5344 if (replace && nhn) 5345 NL_SET_ERR_MSG_MOD(extack, 5346 "multipath route replace failed (check consistency of installed routes)"); 5347 err_nh = nh; 5348 goto add_errout; 5349 } 5350 5351 /* Because each route is added like a single route we remove 5352 * these flags after the first nexthop: if there is a collision, 5353 * we have already failed to add the first nexthop: 5354 * fib6_add_rt2node() has rejected it; when replacing, old 5355 * nexthops have been replaced by first new, the rest should 5356 * be added to it. 5357 */ 5358 if (cfg->fc_nlinfo.nlh) { 5359 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL | 5360 NLM_F_REPLACE); 5361 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE; 5362 } 5363 nhn++; 5364 } 5365 5366 /* An in-kernel notification should only be sent in case the new 5367 * multipath route is added as the first route in the node, or if 5368 * it was appended to it. We pass 'rt_notif' since it is the first 5369 * sibling and might allow us to skip some checks in the replace case. 5370 */ 5371 if (ip6_route_mpath_should_notify(rt_notif)) { 5372 enum fib_event_type fib_event; 5373 5374 if (rt_notif->fib6_nsiblings != nhn - 1) 5375 fib_event = FIB_EVENT_ENTRY_APPEND; 5376 else 5377 fib_event = FIB_EVENT_ENTRY_REPLACE; 5378 5379 err = call_fib6_multipath_entry_notifiers(info->nl_net, 5380 fib_event, rt_notif, 5381 nhn - 1, extack); 5382 if (err) { 5383 /* Delete all the siblings that were just added */ 5384 err_nh = NULL; 5385 goto add_errout; 5386 } 5387 } 5388 5389 /* success ... tell user about new route */ 5390 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5391 goto cleanup; 5392 5393 add_errout: 5394 /* send notification for routes that were added so that 5395 * the delete notifications sent by ip6_route_del are 5396 * coherent 5397 */ 5398 if (rt_notif) 5399 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags); 5400 5401 /* Delete routes that were already added */ 5402 list_for_each_entry(nh, &rt6_nh_list, next) { 5403 if (err_nh == nh) 5404 break; 5405 ip6_route_del(&nh->r_cfg, extack); 5406 } 5407 5408 cleanup: 5409 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) { 5410 if (nh->fib6_info) 5411 fib6_info_release(nh->fib6_info); 5412 list_del(&nh->next); 5413 kfree(nh); 5414 } 5415 5416 return err; 5417 } 5418 5419 static int ip6_route_multipath_del(struct fib6_config *cfg, 5420 struct netlink_ext_ack *extack) 5421 { 5422 struct fib6_config r_cfg; 5423 struct rtnexthop *rtnh; 5424 int last_err = 0; 5425 int remaining; 5426 int attrlen; 5427 int err; 5428 5429 remaining = cfg->fc_mp_len; 5430 rtnh = (struct rtnexthop *)cfg->fc_mp; 5431 5432 /* Parse a Multipath Entry */ 5433 while (rtnh_ok(rtnh, remaining)) { 5434 memcpy(&r_cfg, cfg, sizeof(*cfg)); 5435 if (rtnh->rtnh_ifindex) 5436 r_cfg.fc_ifindex = rtnh->rtnh_ifindex; 5437 5438 attrlen = rtnh_attrlen(rtnh); 5439 if (attrlen > 0) { 5440 struct nlattr *nla, *attrs = rtnh_attrs(rtnh); 5441 5442 nla = nla_find(attrs, attrlen, RTA_GATEWAY); 5443 if (nla) { 5444 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla, 5445 extack); 5446 if (err) { 5447 last_err = err; 5448 goto next_rtnh; 5449 } 5450 5451 r_cfg.fc_flags |= RTF_GATEWAY; 5452 } 5453 } 5454 err = ip6_route_del(&r_cfg, extack); 5455 if (err) 5456 last_err = err; 5457 5458 next_rtnh: 5459 rtnh = rtnh_next(rtnh, &remaining); 5460 } 5461 5462 return last_err; 5463 } 5464 5465 static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5466 struct netlink_ext_ack *extack) 5467 { 5468 struct fib6_config cfg; 5469 int err; 5470 5471 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5472 if (err < 0) 5473 return err; 5474 5475 if (cfg.fc_nh_id && 5476 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) { 5477 NL_SET_ERR_MSG(extack, "Nexthop id does not exist"); 5478 return -EINVAL; 5479 } 5480 5481 if (cfg.fc_mp) 5482 return ip6_route_multipath_del(&cfg, extack); 5483 else { 5484 cfg.fc_delete_all_nh = 1; 5485 return ip6_route_del(&cfg, extack); 5486 } 5487 } 5488 5489 static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh, 5490 struct netlink_ext_ack *extack) 5491 { 5492 struct fib6_config cfg; 5493 int err; 5494 5495 err = rtm_to_fib6_config(skb, nlh, &cfg, extack); 5496 if (err < 0) 5497 return err; 5498 5499 if (cfg.fc_metric == 0) 5500 cfg.fc_metric = IP6_RT_PRIO_USER; 5501 5502 if (cfg.fc_mp) 5503 return ip6_route_multipath_add(&cfg, extack); 5504 else 5505 return ip6_route_add(&cfg, GFP_KERNEL, extack); 5506 } 5507 5508 /* add the overhead of this fib6_nh to nexthop_len */ 5509 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg) 5510 { 5511 int *nexthop_len = arg; 5512 5513 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */ 5514 + NLA_ALIGN(sizeof(struct rtnexthop)) 5515 + nla_total_size(16); /* RTA_GATEWAY */ 5516 5517 if (nh->fib_nh_lws) { 5518 /* RTA_ENCAP_TYPE */ 5519 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5520 /* RTA_ENCAP */ 5521 *nexthop_len += nla_total_size(2); 5522 } 5523 5524 return 0; 5525 } 5526 5527 static size_t rt6_nlmsg_size(struct fib6_info *f6i) 5528 { 5529 int nexthop_len; 5530 5531 if (f6i->nh) { 5532 nexthop_len = nla_total_size(4); /* RTA_NH_ID */ 5533 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size, 5534 &nexthop_len); 5535 } else { 5536 struct fib6_info *sibling, *next_sibling; 5537 struct fib6_nh *nh = f6i->fib6_nh; 5538 5539 nexthop_len = 0; 5540 if (f6i->fib6_nsiblings) { 5541 rt6_nh_nlmsg_size(nh, &nexthop_len); 5542 5543 list_for_each_entry_safe(sibling, next_sibling, 5544 &f6i->fib6_siblings, fib6_siblings) { 5545 rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len); 5546 } 5547 } 5548 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws); 5549 } 5550 5551 return NLMSG_ALIGN(sizeof(struct rtmsg)) 5552 + nla_total_size(16) /* RTA_SRC */ 5553 + nla_total_size(16) /* RTA_DST */ 5554 + nla_total_size(16) /* RTA_GATEWAY */ 5555 + nla_total_size(16) /* RTA_PREFSRC */ 5556 + nla_total_size(4) /* RTA_TABLE */ 5557 + nla_total_size(4) /* RTA_IIF */ 5558 + nla_total_size(4) /* RTA_OIF */ 5559 + nla_total_size(4) /* RTA_PRIORITY */ 5560 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */ 5561 + nla_total_size(sizeof(struct rta_cacheinfo)) 5562 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */ 5563 + nla_total_size(1) /* RTA_PREF */ 5564 + nexthop_len; 5565 } 5566 5567 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh, 5568 unsigned char *flags) 5569 { 5570 if (nexthop_is_multipath(nh)) { 5571 struct nlattr *mp; 5572 5573 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5574 if (!mp) 5575 goto nla_put_failure; 5576 5577 if (nexthop_mpath_fill_node(skb, nh, AF_INET6)) 5578 goto nla_put_failure; 5579 5580 nla_nest_end(skb, mp); 5581 } else { 5582 struct fib6_nh *fib6_nh; 5583 5584 fib6_nh = nexthop_fib6_nh(nh); 5585 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6, 5586 flags, false) < 0) 5587 goto nla_put_failure; 5588 } 5589 5590 return 0; 5591 5592 nla_put_failure: 5593 return -EMSGSIZE; 5594 } 5595 5596 static int rt6_fill_node(struct net *net, struct sk_buff *skb, 5597 struct fib6_info *rt, struct dst_entry *dst, 5598 struct in6_addr *dest, struct in6_addr *src, 5599 int iif, int type, u32 portid, u32 seq, 5600 unsigned int flags) 5601 { 5602 struct rt6_info *rt6 = (struct rt6_info *)dst; 5603 struct rt6key *rt6_dst, *rt6_src; 5604 u32 *pmetrics, table, rt6_flags; 5605 unsigned char nh_flags = 0; 5606 struct nlmsghdr *nlh; 5607 struct rtmsg *rtm; 5608 long expires = 0; 5609 5610 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags); 5611 if (!nlh) 5612 return -EMSGSIZE; 5613 5614 if (rt6) { 5615 rt6_dst = &rt6->rt6i_dst; 5616 rt6_src = &rt6->rt6i_src; 5617 rt6_flags = rt6->rt6i_flags; 5618 } else { 5619 rt6_dst = &rt->fib6_dst; 5620 rt6_src = &rt->fib6_src; 5621 rt6_flags = rt->fib6_flags; 5622 } 5623 5624 rtm = nlmsg_data(nlh); 5625 rtm->rtm_family = AF_INET6; 5626 rtm->rtm_dst_len = rt6_dst->plen; 5627 rtm->rtm_src_len = rt6_src->plen; 5628 rtm->rtm_tos = 0; 5629 if (rt->fib6_table) 5630 table = rt->fib6_table->tb6_id; 5631 else 5632 table = RT6_TABLE_UNSPEC; 5633 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT; 5634 if (nla_put_u32(skb, RTA_TABLE, table)) 5635 goto nla_put_failure; 5636 5637 rtm->rtm_type = rt->fib6_type; 5638 rtm->rtm_flags = 0; 5639 rtm->rtm_scope = RT_SCOPE_UNIVERSE; 5640 rtm->rtm_protocol = rt->fib6_protocol; 5641 5642 if (rt6_flags & RTF_CACHE) 5643 rtm->rtm_flags |= RTM_F_CLONED; 5644 5645 if (dest) { 5646 if (nla_put_in6_addr(skb, RTA_DST, dest)) 5647 goto nla_put_failure; 5648 rtm->rtm_dst_len = 128; 5649 } else if (rtm->rtm_dst_len) 5650 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr)) 5651 goto nla_put_failure; 5652 #ifdef CONFIG_IPV6_SUBTREES 5653 if (src) { 5654 if (nla_put_in6_addr(skb, RTA_SRC, src)) 5655 goto nla_put_failure; 5656 rtm->rtm_src_len = 128; 5657 } else if (rtm->rtm_src_len && 5658 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr)) 5659 goto nla_put_failure; 5660 #endif 5661 if (iif) { 5662 #ifdef CONFIG_IPV6_MROUTE 5663 if (ipv6_addr_is_multicast(&rt6_dst->addr)) { 5664 int err = ip6mr_get_route(net, skb, rtm, portid); 5665 5666 if (err == 0) 5667 return 0; 5668 if (err < 0) 5669 goto nla_put_failure; 5670 } else 5671 #endif 5672 if (nla_put_u32(skb, RTA_IIF, iif)) 5673 goto nla_put_failure; 5674 } else if (dest) { 5675 struct in6_addr saddr_buf; 5676 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 && 5677 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5678 goto nla_put_failure; 5679 } 5680 5681 if (rt->fib6_prefsrc.plen) { 5682 struct in6_addr saddr_buf; 5683 saddr_buf = rt->fib6_prefsrc.addr; 5684 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf)) 5685 goto nla_put_failure; 5686 } 5687 5688 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics; 5689 if (rtnetlink_put_metrics(skb, pmetrics) < 0) 5690 goto nla_put_failure; 5691 5692 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric)) 5693 goto nla_put_failure; 5694 5695 /* For multipath routes, walk the siblings list and add 5696 * each as a nexthop within RTA_MULTIPATH. 5697 */ 5698 if (rt6) { 5699 if (rt6_flags & RTF_GATEWAY && 5700 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway)) 5701 goto nla_put_failure; 5702 5703 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex)) 5704 goto nla_put_failure; 5705 5706 if (dst->lwtstate && 5707 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0) 5708 goto nla_put_failure; 5709 } else if (rt->fib6_nsiblings) { 5710 struct fib6_info *sibling, *next_sibling; 5711 struct nlattr *mp; 5712 5713 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH); 5714 if (!mp) 5715 goto nla_put_failure; 5716 5717 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common, 5718 rt->fib6_nh->fib_nh_weight, AF_INET6, 5719 0) < 0) 5720 goto nla_put_failure; 5721 5722 list_for_each_entry_safe(sibling, next_sibling, 5723 &rt->fib6_siblings, fib6_siblings) { 5724 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common, 5725 sibling->fib6_nh->fib_nh_weight, 5726 AF_INET6, 0) < 0) 5727 goto nla_put_failure; 5728 } 5729 5730 nla_nest_end(skb, mp); 5731 } else if (rt->nh) { 5732 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id)) 5733 goto nla_put_failure; 5734 5735 if (nexthop_is_blackhole(rt->nh)) 5736 rtm->rtm_type = RTN_BLACKHOLE; 5737 5738 if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) && 5739 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0) 5740 goto nla_put_failure; 5741 5742 rtm->rtm_flags |= nh_flags; 5743 } else { 5744 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6, 5745 &nh_flags, false) < 0) 5746 goto nla_put_failure; 5747 5748 rtm->rtm_flags |= nh_flags; 5749 } 5750 5751 if (rt6_flags & RTF_EXPIRES) { 5752 expires = dst ? dst->expires : rt->expires; 5753 expires -= jiffies; 5754 } 5755 5756 if (!dst) { 5757 if (READ_ONCE(rt->offload)) 5758 rtm->rtm_flags |= RTM_F_OFFLOAD; 5759 if (READ_ONCE(rt->trap)) 5760 rtm->rtm_flags |= RTM_F_TRAP; 5761 if (READ_ONCE(rt->offload_failed)) 5762 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED; 5763 } 5764 5765 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0) 5766 goto nla_put_failure; 5767 5768 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags))) 5769 goto nla_put_failure; 5770 5771 5772 nlmsg_end(skb, nlh); 5773 return 0; 5774 5775 nla_put_failure: 5776 nlmsg_cancel(skb, nlh); 5777 return -EMSGSIZE; 5778 } 5779 5780 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg) 5781 { 5782 const struct net_device *dev = arg; 5783 5784 if (nh->fib_nh_dev == dev) 5785 return 1; 5786 5787 return 0; 5788 } 5789 5790 static bool fib6_info_uses_dev(const struct fib6_info *f6i, 5791 const struct net_device *dev) 5792 { 5793 if (f6i->nh) { 5794 struct net_device *_dev = (struct net_device *)dev; 5795 5796 return !!nexthop_for_each_fib6_nh(f6i->nh, 5797 fib6_info_nh_uses_dev, 5798 _dev); 5799 } 5800 5801 if (f6i->fib6_nh->fib_nh_dev == dev) 5802 return true; 5803 5804 if (f6i->fib6_nsiblings) { 5805 struct fib6_info *sibling, *next_sibling; 5806 5807 list_for_each_entry_safe(sibling, next_sibling, 5808 &f6i->fib6_siblings, fib6_siblings) { 5809 if (sibling->fib6_nh->fib_nh_dev == dev) 5810 return true; 5811 } 5812 } 5813 5814 return false; 5815 } 5816 5817 struct fib6_nh_exception_dump_walker { 5818 struct rt6_rtnl_dump_arg *dump; 5819 struct fib6_info *rt; 5820 unsigned int flags; 5821 unsigned int skip; 5822 unsigned int count; 5823 }; 5824 5825 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) 5826 { 5827 struct fib6_nh_exception_dump_walker *w = arg; 5828 struct rt6_rtnl_dump_arg *dump = w->dump; 5829 struct rt6_exception_bucket *bucket; 5830 struct rt6_exception *rt6_ex; 5831 int i, err; 5832 5833 bucket = fib6_nh_get_excptn_bucket(nh, NULL); 5834 if (!bucket) 5835 return 0; 5836 5837 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { 5838 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { 5839 if (w->skip) { 5840 w->skip--; 5841 continue; 5842 } 5843 5844 /* Expiration of entries doesn't bump sernum, insertion 5845 * does. Removal is triggered by insertion, so we can 5846 * rely on the fact that if entries change between two 5847 * partial dumps, this node is scanned again completely, 5848 * see rt6_insert_exception() and fib6_dump_table(). 5849 * 5850 * Count expired entries we go through as handled 5851 * entries that we'll skip next time, in case of partial 5852 * node dump. Otherwise, if entries expire meanwhile, 5853 * we'll skip the wrong amount. 5854 */ 5855 if (rt6_check_expired(rt6_ex->rt6i)) { 5856 w->count++; 5857 continue; 5858 } 5859 5860 err = rt6_fill_node(dump->net, dump->skb, w->rt, 5861 &rt6_ex->rt6i->dst, NULL, NULL, 0, 5862 RTM_NEWROUTE, 5863 NETLINK_CB(dump->cb->skb).portid, 5864 dump->cb->nlh->nlmsg_seq, w->flags); 5865 if (err) 5866 return err; 5867 5868 w->count++; 5869 } 5870 bucket++; 5871 } 5872 5873 return 0; 5874 } 5875 5876 /* Return -1 if done with node, number of handled routes on partial dump */ 5877 int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip) 5878 { 5879 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg; 5880 struct fib_dump_filter *filter = &arg->filter; 5881 unsigned int flags = NLM_F_MULTI; 5882 struct net *net = arg->net; 5883 int count = 0; 5884 5885 if (rt == net->ipv6.fib6_null_entry) 5886 return -1; 5887 5888 if ((filter->flags & RTM_F_PREFIX) && 5889 !(rt->fib6_flags & RTF_PREFIX_RT)) { 5890 /* success since this is not a prefix route */ 5891 return -1; 5892 } 5893 if (filter->filter_set && 5894 ((filter->rt_type && rt->fib6_type != filter->rt_type) || 5895 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) || 5896 (filter->protocol && rt->fib6_protocol != filter->protocol))) { 5897 return -1; 5898 } 5899 5900 if (filter->filter_set || 5901 !filter->dump_routes || !filter->dump_exceptions) { 5902 flags |= NLM_F_DUMP_FILTERED; 5903 } 5904 5905 if (filter->dump_routes) { 5906 if (skip) { 5907 skip--; 5908 } else { 5909 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 5910 0, RTM_NEWROUTE, 5911 NETLINK_CB(arg->cb->skb).portid, 5912 arg->cb->nlh->nlmsg_seq, flags)) { 5913 return 0; 5914 } 5915 count++; 5916 } 5917 } 5918 5919 if (filter->dump_exceptions) { 5920 struct fib6_nh_exception_dump_walker w = { .dump = arg, 5921 .rt = rt, 5922 .flags = flags, 5923 .skip = skip, 5924 .count = 0 }; 5925 int err; 5926 5927 rcu_read_lock(); 5928 if (rt->nh) { 5929 err = nexthop_for_each_fib6_nh(rt->nh, 5930 rt6_nh_dump_exceptions, 5931 &w); 5932 } else { 5933 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w); 5934 } 5935 rcu_read_unlock(); 5936 5937 if (err) 5938 return count + w.count; 5939 } 5940 5941 return -1; 5942 } 5943 5944 static int inet6_rtm_valid_getroute_req(struct sk_buff *skb, 5945 const struct nlmsghdr *nlh, 5946 struct nlattr **tb, 5947 struct netlink_ext_ack *extack) 5948 { 5949 struct rtmsg *rtm; 5950 int i, err; 5951 5952 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) { 5953 NL_SET_ERR_MSG_MOD(extack, 5954 "Invalid header for get route request"); 5955 return -EINVAL; 5956 } 5957 5958 if (!netlink_strict_get_check(skb)) 5959 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, 5960 rtm_ipv6_policy, extack); 5961 5962 rtm = nlmsg_data(nlh); 5963 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) || 5964 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) || 5965 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope || 5966 rtm->rtm_type) { 5967 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request"); 5968 return -EINVAL; 5969 } 5970 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) { 5971 NL_SET_ERR_MSG_MOD(extack, 5972 "Invalid flags for get route request"); 5973 return -EINVAL; 5974 } 5975 5976 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX, 5977 rtm_ipv6_policy, extack); 5978 if (err) 5979 return err; 5980 5981 if ((tb[RTA_SRC] && !rtm->rtm_src_len) || 5982 (tb[RTA_DST] && !rtm->rtm_dst_len)) { 5983 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6"); 5984 return -EINVAL; 5985 } 5986 5987 for (i = 0; i <= RTA_MAX; i++) { 5988 if (!tb[i]) 5989 continue; 5990 5991 switch (i) { 5992 case RTA_SRC: 5993 case RTA_DST: 5994 case RTA_IIF: 5995 case RTA_OIF: 5996 case RTA_MARK: 5997 case RTA_UID: 5998 case RTA_SPORT: 5999 case RTA_DPORT: 6000 case RTA_IP_PROTO: 6001 break; 6002 default: 6003 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request"); 6004 return -EINVAL; 6005 } 6006 } 6007 6008 return 0; 6009 } 6010 6011 static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, 6012 struct netlink_ext_ack *extack) 6013 { 6014 struct net *net = sock_net(in_skb->sk); 6015 struct nlattr *tb[RTA_MAX+1]; 6016 int err, iif = 0, oif = 0; 6017 struct fib6_info *from; 6018 struct dst_entry *dst; 6019 struct rt6_info *rt; 6020 struct sk_buff *skb; 6021 struct rtmsg *rtm; 6022 struct flowi6 fl6 = {}; 6023 bool fibmatch; 6024 6025 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack); 6026 if (err < 0) 6027 goto errout; 6028 6029 err = -EINVAL; 6030 rtm = nlmsg_data(nlh); 6031 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0); 6032 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH); 6033 6034 if (tb[RTA_SRC]) { 6035 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr)) 6036 goto errout; 6037 6038 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]); 6039 } 6040 6041 if (tb[RTA_DST]) { 6042 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr)) 6043 goto errout; 6044 6045 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]); 6046 } 6047 6048 if (tb[RTA_IIF]) 6049 iif = nla_get_u32(tb[RTA_IIF]); 6050 6051 if (tb[RTA_OIF]) 6052 oif = nla_get_u32(tb[RTA_OIF]); 6053 6054 if (tb[RTA_MARK]) 6055 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]); 6056 6057 if (tb[RTA_UID]) 6058 fl6.flowi6_uid = make_kuid(current_user_ns(), 6059 nla_get_u32(tb[RTA_UID])); 6060 else 6061 fl6.flowi6_uid = iif ? INVALID_UID : current_uid(); 6062 6063 if (tb[RTA_SPORT]) 6064 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]); 6065 6066 if (tb[RTA_DPORT]) 6067 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]); 6068 6069 if (tb[RTA_IP_PROTO]) { 6070 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], 6071 &fl6.flowi6_proto, AF_INET6, 6072 extack); 6073 if (err) 6074 goto errout; 6075 } 6076 6077 if (iif) { 6078 struct net_device *dev; 6079 int flags = 0; 6080 6081 rcu_read_lock(); 6082 6083 dev = dev_get_by_index_rcu(net, iif); 6084 if (!dev) { 6085 rcu_read_unlock(); 6086 err = -ENODEV; 6087 goto errout; 6088 } 6089 6090 fl6.flowi6_iif = iif; 6091 6092 if (!ipv6_addr_any(&fl6.saddr)) 6093 flags |= RT6_LOOKUP_F_HAS_SADDR; 6094 6095 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags); 6096 6097 rcu_read_unlock(); 6098 } else { 6099 fl6.flowi6_oif = oif; 6100 6101 dst = ip6_route_output(net, NULL, &fl6); 6102 } 6103 6104 6105 rt = container_of(dst, struct rt6_info, dst); 6106 if (rt->dst.error) { 6107 err = rt->dst.error; 6108 ip6_rt_put(rt); 6109 goto errout; 6110 } 6111 6112 if (rt == net->ipv6.ip6_null_entry) { 6113 err = rt->dst.error; 6114 ip6_rt_put(rt); 6115 goto errout; 6116 } 6117 6118 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 6119 if (!skb) { 6120 ip6_rt_put(rt); 6121 err = -ENOBUFS; 6122 goto errout; 6123 } 6124 6125 skb_dst_set(skb, &rt->dst); 6126 6127 rcu_read_lock(); 6128 from = rcu_dereference(rt->from); 6129 if (from) { 6130 if (fibmatch) 6131 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, 6132 iif, RTM_NEWROUTE, 6133 NETLINK_CB(in_skb).portid, 6134 nlh->nlmsg_seq, 0); 6135 else 6136 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr, 6137 &fl6.saddr, iif, RTM_NEWROUTE, 6138 NETLINK_CB(in_skb).portid, 6139 nlh->nlmsg_seq, 0); 6140 } else { 6141 err = -ENETUNREACH; 6142 } 6143 rcu_read_unlock(); 6144 6145 if (err < 0) { 6146 kfree_skb(skb); 6147 goto errout; 6148 } 6149 6150 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid); 6151 errout: 6152 return err; 6153 } 6154 6155 void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, 6156 unsigned int nlm_flags) 6157 { 6158 struct sk_buff *skb; 6159 struct net *net = info->nl_net; 6160 u32 seq; 6161 int err; 6162 6163 err = -ENOBUFS; 6164 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6165 6166 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6167 if (!skb) 6168 goto errout; 6169 6170 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6171 event, info->portid, seq, nlm_flags); 6172 if (err < 0) { 6173 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6174 WARN_ON(err == -EMSGSIZE); 6175 kfree_skb(skb); 6176 goto errout; 6177 } 6178 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6179 info->nlh, gfp_any()); 6180 return; 6181 errout: 6182 if (err < 0) 6183 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6184 } 6185 6186 void fib6_rt_update(struct net *net, struct fib6_info *rt, 6187 struct nl_info *info) 6188 { 6189 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; 6190 struct sk_buff *skb; 6191 int err = -ENOBUFS; 6192 6193 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); 6194 if (!skb) 6195 goto errout; 6196 6197 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0, 6198 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE); 6199 if (err < 0) { 6200 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6201 WARN_ON(err == -EMSGSIZE); 6202 kfree_skb(skb); 6203 goto errout; 6204 } 6205 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, 6206 info->nlh, gfp_any()); 6207 return; 6208 errout: 6209 if (err < 0) 6210 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6211 } 6212 6213 void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, 6214 bool offload, bool trap, bool offload_failed) 6215 { 6216 struct sk_buff *skb; 6217 int err; 6218 6219 if (READ_ONCE(f6i->offload) == offload && 6220 READ_ONCE(f6i->trap) == trap && 6221 READ_ONCE(f6i->offload_failed) == offload_failed) 6222 return; 6223 6224 WRITE_ONCE(f6i->offload, offload); 6225 WRITE_ONCE(f6i->trap, trap); 6226 6227 /* 2 means send notifications only if offload_failed was changed. */ 6228 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && 6229 READ_ONCE(f6i->offload_failed) == offload_failed) 6230 return; 6231 6232 WRITE_ONCE(f6i->offload_failed, offload_failed); 6233 6234 if (!rcu_access_pointer(f6i->fib6_node)) 6235 /* The route was removed from the tree, do not send 6236 * notification. 6237 */ 6238 return; 6239 6240 if (!net->ipv6.sysctl.fib_notify_on_flag_change) 6241 return; 6242 6243 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); 6244 if (!skb) { 6245 err = -ENOBUFS; 6246 goto errout; 6247 } 6248 6249 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0, 6250 0, 0); 6251 if (err < 0) { 6252 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ 6253 WARN_ON(err == -EMSGSIZE); 6254 kfree_skb(skb); 6255 goto errout; 6256 } 6257 6258 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL); 6259 return; 6260 6261 errout: 6262 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err); 6263 } 6264 EXPORT_SYMBOL(fib6_info_hw_flags_set); 6265 6266 static int ip6_route_dev_notify(struct notifier_block *this, 6267 unsigned long event, void *ptr) 6268 { 6269 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6270 struct net *net = dev_net(dev); 6271 6272 if (!(dev->flags & IFF_LOOPBACK)) 6273 return NOTIFY_OK; 6274 6275 if (event == NETDEV_REGISTER) { 6276 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev; 6277 net->ipv6.ip6_null_entry->dst.dev = dev; 6278 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev); 6279 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6280 net->ipv6.ip6_prohibit_entry->dst.dev = dev; 6281 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev); 6282 net->ipv6.ip6_blk_hole_entry->dst.dev = dev; 6283 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev); 6284 #endif 6285 } else if (event == NETDEV_UNREGISTER && 6286 dev->reg_state != NETREG_UNREGISTERED) { 6287 /* NETDEV_UNREGISTER could be fired for multiple times by 6288 * netdev_wait_allrefs(). Make sure we only call this once. 6289 */ 6290 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev); 6291 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6292 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev); 6293 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev); 6294 #endif 6295 } 6296 6297 return NOTIFY_OK; 6298 } 6299 6300 /* 6301 * /proc 6302 */ 6303 6304 #ifdef CONFIG_PROC_FS 6305 static int rt6_stats_seq_show(struct seq_file *seq, void *v) 6306 { 6307 struct net *net = (struct net *)seq->private; 6308 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n", 6309 net->ipv6.rt6_stats->fib_nodes, 6310 net->ipv6.rt6_stats->fib_route_nodes, 6311 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc), 6312 net->ipv6.rt6_stats->fib_rt_entries, 6313 net->ipv6.rt6_stats->fib_rt_cache, 6314 dst_entries_get_slow(&net->ipv6.ip6_dst_ops), 6315 net->ipv6.rt6_stats->fib_discarded_routes); 6316 6317 return 0; 6318 } 6319 #endif /* CONFIG_PROC_FS */ 6320 6321 #ifdef CONFIG_SYSCTL 6322 6323 static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write, 6324 void *buffer, size_t *lenp, loff_t *ppos) 6325 { 6326 struct net *net; 6327 int delay; 6328 int ret; 6329 if (!write) 6330 return -EINVAL; 6331 6332 net = (struct net *)ctl->extra1; 6333 delay = net->ipv6.sysctl.flush_delay; 6334 ret = proc_dointvec(ctl, write, buffer, lenp, ppos); 6335 if (ret) 6336 return ret; 6337 6338 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); 6339 return 0; 6340 } 6341 6342 static struct ctl_table ipv6_route_table_template[] = { 6343 { 6344 .procname = "max_size", 6345 .data = &init_net.ipv6.sysctl.ip6_rt_max_size, 6346 .maxlen = sizeof(int), 6347 .mode = 0644, 6348 .proc_handler = proc_dointvec, 6349 }, 6350 { 6351 .procname = "gc_thresh", 6352 .data = &ip6_dst_ops_template.gc_thresh, 6353 .maxlen = sizeof(int), 6354 .mode = 0644, 6355 .proc_handler = proc_dointvec, 6356 }, 6357 { 6358 .procname = "flush", 6359 .data = &init_net.ipv6.sysctl.flush_delay, 6360 .maxlen = sizeof(int), 6361 .mode = 0200, 6362 .proc_handler = ipv6_sysctl_rtcache_flush 6363 }, 6364 { 6365 .procname = "gc_min_interval", 6366 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6367 .maxlen = sizeof(int), 6368 .mode = 0644, 6369 .proc_handler = proc_dointvec_jiffies, 6370 }, 6371 { 6372 .procname = "gc_timeout", 6373 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout, 6374 .maxlen = sizeof(int), 6375 .mode = 0644, 6376 .proc_handler = proc_dointvec_jiffies, 6377 }, 6378 { 6379 .procname = "gc_interval", 6380 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval, 6381 .maxlen = sizeof(int), 6382 .mode = 0644, 6383 .proc_handler = proc_dointvec_jiffies, 6384 }, 6385 { 6386 .procname = "gc_elasticity", 6387 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity, 6388 .maxlen = sizeof(int), 6389 .mode = 0644, 6390 .proc_handler = proc_dointvec, 6391 }, 6392 { 6393 .procname = "mtu_expires", 6394 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires, 6395 .maxlen = sizeof(int), 6396 .mode = 0644, 6397 .proc_handler = proc_dointvec_jiffies, 6398 }, 6399 { 6400 .procname = "min_adv_mss", 6401 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss, 6402 .maxlen = sizeof(int), 6403 .mode = 0644, 6404 .proc_handler = proc_dointvec, 6405 }, 6406 { 6407 .procname = "gc_min_interval_ms", 6408 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval, 6409 .maxlen = sizeof(int), 6410 .mode = 0644, 6411 .proc_handler = proc_dointvec_ms_jiffies, 6412 }, 6413 { 6414 .procname = "skip_notify_on_dev_down", 6415 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down, 6416 .maxlen = sizeof(int), 6417 .mode = 0644, 6418 .proc_handler = proc_dointvec_minmax, 6419 .extra1 = SYSCTL_ZERO, 6420 .extra2 = SYSCTL_ONE, 6421 }, 6422 { } 6423 }; 6424 6425 struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net) 6426 { 6427 struct ctl_table *table; 6428 6429 table = kmemdup(ipv6_route_table_template, 6430 sizeof(ipv6_route_table_template), 6431 GFP_KERNEL); 6432 6433 if (table) { 6434 table[0].data = &net->ipv6.sysctl.ip6_rt_max_size; 6435 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh; 6436 table[2].data = &net->ipv6.sysctl.flush_delay; 6437 table[2].extra1 = net; 6438 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6439 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout; 6440 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval; 6441 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity; 6442 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires; 6443 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss; 6444 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval; 6445 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down; 6446 6447 /* Don't export sysctls to unprivileged users */ 6448 if (net->user_ns != &init_user_ns) 6449 table[1].procname = NULL; 6450 } 6451 6452 return table; 6453 } 6454 #endif 6455 6456 static int __net_init ip6_route_net_init(struct net *net) 6457 { 6458 int ret = -ENOMEM; 6459 6460 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template, 6461 sizeof(net->ipv6.ip6_dst_ops)); 6462 6463 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0) 6464 goto out_ip6_dst_ops; 6465 6466 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true); 6467 if (!net->ipv6.fib6_null_entry) 6468 goto out_ip6_dst_entries; 6469 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template, 6470 sizeof(*net->ipv6.fib6_null_entry)); 6471 6472 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template, 6473 sizeof(*net->ipv6.ip6_null_entry), 6474 GFP_KERNEL); 6475 if (!net->ipv6.ip6_null_entry) 6476 goto out_fib6_null_entry; 6477 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6478 dst_init_metrics(&net->ipv6.ip6_null_entry->dst, 6479 ip6_template_metrics, true); 6480 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->rt6i_uncached); 6481 6482 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6483 net->ipv6.fib6_has_custom_rules = false; 6484 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template, 6485 sizeof(*net->ipv6.ip6_prohibit_entry), 6486 GFP_KERNEL); 6487 if (!net->ipv6.ip6_prohibit_entry) 6488 goto out_ip6_null_entry; 6489 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6490 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst, 6491 ip6_template_metrics, true); 6492 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->rt6i_uncached); 6493 6494 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template, 6495 sizeof(*net->ipv6.ip6_blk_hole_entry), 6496 GFP_KERNEL); 6497 if (!net->ipv6.ip6_blk_hole_entry) 6498 goto out_ip6_prohibit_entry; 6499 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops; 6500 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst, 6501 ip6_template_metrics, true); 6502 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->rt6i_uncached); 6503 #ifdef CONFIG_IPV6_SUBTREES 6504 net->ipv6.fib6_routes_require_src = 0; 6505 #endif 6506 #endif 6507 6508 net->ipv6.sysctl.flush_delay = 0; 6509 net->ipv6.sysctl.ip6_rt_max_size = INT_MAX; 6510 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2; 6511 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ; 6512 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ; 6513 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9; 6514 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ; 6515 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40; 6516 net->ipv6.sysctl.skip_notify_on_dev_down = 0; 6517 6518 atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ); 6519 6520 ret = 0; 6521 out: 6522 return ret; 6523 6524 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6525 out_ip6_prohibit_entry: 6526 kfree(net->ipv6.ip6_prohibit_entry); 6527 out_ip6_null_entry: 6528 kfree(net->ipv6.ip6_null_entry); 6529 #endif 6530 out_fib6_null_entry: 6531 kfree(net->ipv6.fib6_null_entry); 6532 out_ip6_dst_entries: 6533 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6534 out_ip6_dst_ops: 6535 goto out; 6536 } 6537 6538 static void __net_exit ip6_route_net_exit(struct net *net) 6539 { 6540 kfree(net->ipv6.fib6_null_entry); 6541 kfree(net->ipv6.ip6_null_entry); 6542 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6543 kfree(net->ipv6.ip6_prohibit_entry); 6544 kfree(net->ipv6.ip6_blk_hole_entry); 6545 #endif 6546 dst_entries_destroy(&net->ipv6.ip6_dst_ops); 6547 } 6548 6549 static int __net_init ip6_route_net_init_late(struct net *net) 6550 { 6551 #ifdef CONFIG_PROC_FS 6552 if (!proc_create_net("ipv6_route", 0, net->proc_net, 6553 &ipv6_route_seq_ops, 6554 sizeof(struct ipv6_route_iter))) 6555 return -ENOMEM; 6556 6557 if (!proc_create_net_single("rt6_stats", 0444, net->proc_net, 6558 rt6_stats_seq_show, NULL)) { 6559 remove_proc_entry("ipv6_route", net->proc_net); 6560 return -ENOMEM; 6561 } 6562 #endif 6563 return 0; 6564 } 6565 6566 static void __net_exit ip6_route_net_exit_late(struct net *net) 6567 { 6568 #ifdef CONFIG_PROC_FS 6569 remove_proc_entry("ipv6_route", net->proc_net); 6570 remove_proc_entry("rt6_stats", net->proc_net); 6571 #endif 6572 } 6573 6574 static struct pernet_operations ip6_route_net_ops = { 6575 .init = ip6_route_net_init, 6576 .exit = ip6_route_net_exit, 6577 }; 6578 6579 static int __net_init ipv6_inetpeer_init(struct net *net) 6580 { 6581 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL); 6582 6583 if (!bp) 6584 return -ENOMEM; 6585 inet_peer_base_init(bp); 6586 net->ipv6.peers = bp; 6587 return 0; 6588 } 6589 6590 static void __net_exit ipv6_inetpeer_exit(struct net *net) 6591 { 6592 struct inet_peer_base *bp = net->ipv6.peers; 6593 6594 net->ipv6.peers = NULL; 6595 inetpeer_invalidate_tree(bp); 6596 kfree(bp); 6597 } 6598 6599 static struct pernet_operations ipv6_inetpeer_ops = { 6600 .init = ipv6_inetpeer_init, 6601 .exit = ipv6_inetpeer_exit, 6602 }; 6603 6604 static struct pernet_operations ip6_route_net_late_ops = { 6605 .init = ip6_route_net_init_late, 6606 .exit = ip6_route_net_exit_late, 6607 }; 6608 6609 static struct notifier_block ip6_route_dev_notifier = { 6610 .notifier_call = ip6_route_dev_notify, 6611 .priority = ADDRCONF_NOTIFY_PRIORITY - 10, 6612 }; 6613 6614 void __init ip6_route_init_special_entries(void) 6615 { 6616 /* Registering of the loopback is done before this portion of code, 6617 * the loopback reference in rt6_info will not be taken, do it 6618 * manually for init_net */ 6619 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev; 6620 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev; 6621 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6622 #ifdef CONFIG_IPV6_MULTIPLE_TABLES 6623 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev; 6624 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6625 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev; 6626 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev); 6627 #endif 6628 } 6629 6630 #if IS_BUILTIN(CONFIG_IPV6) 6631 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6632 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt) 6633 6634 BTF_ID_LIST(btf_fib6_info_id) 6635 BTF_ID(struct, fib6_info) 6636 6637 static const struct bpf_iter_seq_info ipv6_route_seq_info = { 6638 .seq_ops = &ipv6_route_seq_ops, 6639 .init_seq_private = bpf_iter_init_seq_net, 6640 .fini_seq_private = bpf_iter_fini_seq_net, 6641 .seq_priv_size = sizeof(struct ipv6_route_iter), 6642 }; 6643 6644 static struct bpf_iter_reg ipv6_route_reg_info = { 6645 .target = "ipv6_route", 6646 .ctx_arg_info_size = 1, 6647 .ctx_arg_info = { 6648 { offsetof(struct bpf_iter__ipv6_route, rt), 6649 PTR_TO_BTF_ID_OR_NULL }, 6650 }, 6651 .seq_info = &ipv6_route_seq_info, 6652 }; 6653 6654 static int __init bpf_iter_register(void) 6655 { 6656 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id; 6657 return bpf_iter_reg_target(&ipv6_route_reg_info); 6658 } 6659 6660 static void bpf_iter_unregister(void) 6661 { 6662 bpf_iter_unreg_target(&ipv6_route_reg_info); 6663 } 6664 #endif 6665 #endif 6666 6667 int __init ip6_route_init(void) 6668 { 6669 int ret; 6670 int cpu; 6671 6672 ret = -ENOMEM; 6673 ip6_dst_ops_template.kmem_cachep = 6674 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0, 6675 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL); 6676 if (!ip6_dst_ops_template.kmem_cachep) 6677 goto out; 6678 6679 ret = dst_entries_init(&ip6_dst_blackhole_ops); 6680 if (ret) 6681 goto out_kmem_cache; 6682 6683 ret = register_pernet_subsys(&ipv6_inetpeer_ops); 6684 if (ret) 6685 goto out_dst_entries; 6686 6687 ret = register_pernet_subsys(&ip6_route_net_ops); 6688 if (ret) 6689 goto out_register_inetpeer; 6690 6691 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep; 6692 6693 ret = fib6_init(); 6694 if (ret) 6695 goto out_register_subsys; 6696 6697 ret = xfrm6_init(); 6698 if (ret) 6699 goto out_fib6_init; 6700 6701 ret = fib6_rules_init(); 6702 if (ret) 6703 goto xfrm6_init; 6704 6705 ret = register_pernet_subsys(&ip6_route_net_late_ops); 6706 if (ret) 6707 goto fib6_rules_init; 6708 6709 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE, 6710 inet6_rtm_newroute, NULL, 0); 6711 if (ret < 0) 6712 goto out_register_late_subsys; 6713 6714 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE, 6715 inet6_rtm_delroute, NULL, 0); 6716 if (ret < 0) 6717 goto out_register_late_subsys; 6718 6719 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE, 6720 inet6_rtm_getroute, NULL, 6721 RTNL_FLAG_DOIT_UNLOCKED); 6722 if (ret < 0) 6723 goto out_register_late_subsys; 6724 6725 ret = register_netdevice_notifier(&ip6_route_dev_notifier); 6726 if (ret) 6727 goto out_register_late_subsys; 6728 6729 #if IS_BUILTIN(CONFIG_IPV6) 6730 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6731 ret = bpf_iter_register(); 6732 if (ret) 6733 goto out_register_late_subsys; 6734 #endif 6735 #endif 6736 6737 for_each_possible_cpu(cpu) { 6738 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); 6739 6740 INIT_LIST_HEAD(&ul->head); 6741 INIT_LIST_HEAD(&ul->quarantine); 6742 spin_lock_init(&ul->lock); 6743 } 6744 6745 out: 6746 return ret; 6747 6748 out_register_late_subsys: 6749 rtnl_unregister_all(PF_INET6); 6750 unregister_pernet_subsys(&ip6_route_net_late_ops); 6751 fib6_rules_init: 6752 fib6_rules_cleanup(); 6753 xfrm6_init: 6754 xfrm6_fini(); 6755 out_fib6_init: 6756 fib6_gc_cleanup(); 6757 out_register_subsys: 6758 unregister_pernet_subsys(&ip6_route_net_ops); 6759 out_register_inetpeer: 6760 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6761 out_dst_entries: 6762 dst_entries_destroy(&ip6_dst_blackhole_ops); 6763 out_kmem_cache: 6764 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6765 goto out; 6766 } 6767 6768 void ip6_route_cleanup(void) 6769 { 6770 #if IS_BUILTIN(CONFIG_IPV6) 6771 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 6772 bpf_iter_unregister(); 6773 #endif 6774 #endif 6775 unregister_netdevice_notifier(&ip6_route_dev_notifier); 6776 unregister_pernet_subsys(&ip6_route_net_late_ops); 6777 fib6_rules_cleanup(); 6778 xfrm6_fini(); 6779 fib6_gc_cleanup(); 6780 unregister_pernet_subsys(&ipv6_inetpeer_ops); 6781 unregister_pernet_subsys(&ip6_route_net_ops); 6782 dst_entries_destroy(&ip6_dst_blackhole_ops); 6783 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep); 6784 } 6785